From c1effc5e75029e23ccfe3aa8817a9b33cbcf406f Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Tue, 25 Feb 2020 22:27:22 -0500 Subject: [PATCH 1/6] Install GDAL/rasterio/GEOS via pip --- requirements.txt | 15 +++++++++------ webodm/settings.py | 5 ++++- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/requirements.txt b/requirements.txt index 93b22aee..94f4c3a4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -24,7 +24,7 @@ futures==3.0.5 geojson==2.3.0 gunicorn==19.7.1 itypes==1.1.0 -kombu==4.1.0 +kombu==4.6.7 libsass==0.13.3 Markdown==2.6.7 olefile==0.44 @@ -32,10 +32,10 @@ openapi-codec==1.1.7 packaging==16.8 piexif==1.0.13 pilkit==2.0 -Pillow==6.2.0 +Pillow==6.2.2 pip-autoremove==0.9.0 -psycopg2==2.7.4 -psycopg2-binary==2.7.4 +psycopg2==2.8.4 +psycopg2-binary==2.8.4 PyJWT==1.5.3 pyodm==1.5.3b1 pyparsing==2.1.10 @@ -53,7 +53,10 @@ tzlocal==1.3 uritemplate==3.0.0 vine==1.1.4 webcolors==1.5 -rasterio==1.1.0 -e git://github.com/OpenDroneMap/rio-tiler.git#egg=rio-tiler rio-color==1.0.0 -rio-cogeo==1.1.8 \ No newline at end of file +rio-cogeo==1.1.8 +rasterio==1.1.0 ; sys_platform == 'linux' or sys_platform == 'darwin' +https://download.lfd.uci.edu/pythonlibs/s2jqpv5t/rasterio-1.1.3-cp38-cp38-win_amd64.whl ; sys_platform == "win32" +https://download.lfd.uci.edu/pythonlibs/s2jqpv5t/GDAL-3.0.4-cp38-cp38-win_amd64.whl ; sys_platform == "win32" +Shapely==1.7.0 ; sys_platform == "win32" \ No newline at end of file diff --git a/webodm/settings.py b/webodm/settings.py index 2e56ecb5..c3420acf 100644 --- a/webodm/settings.py +++ b/webodm/settings.py @@ -10,7 +10,7 @@ For the full list of settings and their values, see https://docs.djangoproject.com/en/1.10/ref/settings/ """ -import os, sys, json +import os, sys, json, platform import datetime @@ -335,6 +335,9 @@ CELERY_INCLUDE=['worker.tasks'] CELERY_WORKER_REDIRECT_STDOUTS = False CELERY_WORKER_HIJACK_ROOT_LOGGER = False +if platform.system() == "Windows": + GDAL_LIBRARY_PATH = ".venv/Lib/site-packages/osgeo/gdal300" + GEOS_LIBRARY_PATH = ".venv/Lib/site-packages/shapely/DLLs/geos_c" if TESTING: CELERY_TASK_ALWAYS_EAGER = True From fd771a2220660d6f6addcca03ff959d017bc859a Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Wed, 26 Feb 2020 22:47:49 -0500 Subject: [PATCH 2/6] More resiliant initialization of plugins, win32 requirements --- app/plugins/functions.py | 12 ++++++++++-- requirements.txt | 5 +++-- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/app/plugins/functions.py b/app/plugins/functions.py index 3910624d..89de3605 100644 --- a/app/plugins/functions.py +++ b/app/plugins/functions.py @@ -72,7 +72,12 @@ def build_plugins(): # and run npm install if needed if plugin.path_exists("public/package.json") and not plugin.path_exists("public/node_modules"): logger.info("Running npm install for {}".format(plugin)) - subprocess.call(['npm', 'install'], cwd=plugin.get_path("public")) + + try: + subprocess.call(['npm', 'install'], cwd=plugin.get_path("public")) + except FileNotFoundError: + logger.warn("npm is not installed, will skip this plugin") + continue # Check if we need to generate a webpack.config.js if len(plugin.build_jsx_components()) > 0 and plugin.path_exists('public'): @@ -105,8 +110,11 @@ def build_plugins(): subprocess.Popen(['webpack-cli', '--watch'], cwd=plugin.get_path("public")) elif not plugin.path_exists("public/build"): logger.info("Running webpack for {}".format(plugin.get_name())) - subprocess.call(['webpack-cli'], cwd=plugin.get_path("public")) + try: + subprocess.call(['webpack-cli'], cwd=plugin.get_path("public")) + except FileNotFoundError: + logger.warn("webpack-cli is not installed, plugin will not work") def webpack_watch_process_count(): count = 0 diff --git a/requirements.txt b/requirements.txt index 94f4c3a4..14a32f67 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ anyjson==0.3.3 appdirs==1.4.0 APScheduler==3.2.0 billiard==3.5.0.3 -celery==4.1.0 +celery==4.4.0 coreapi==2.0.9 Django==2.1.15 django-appconf==1.0.2 @@ -59,4 +59,5 @@ rio-cogeo==1.1.8 rasterio==1.1.0 ; sys_platform == 'linux' or sys_platform == 'darwin' https://download.lfd.uci.edu/pythonlibs/s2jqpv5t/rasterio-1.1.3-cp38-cp38-win_amd64.whl ; sys_platform == "win32" https://download.lfd.uci.edu/pythonlibs/s2jqpv5t/GDAL-3.0.4-cp38-cp38-win_amd64.whl ; sys_platform == "win32" -Shapely==1.7.0 ; sys_platform == "win32" \ No newline at end of file +Shapely==1.7.0 ; sys_platform == "win32" +waitress==1.4.3 ; sys_platform == "win32" \ No newline at end of file From 27e2442aab7beded1fd4f2e39e8fd766b014f2f7 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Fri, 28 Feb 2020 22:02:31 -0500 Subject: [PATCH 3/6] Find npm and webpack-cli on Windows --- app/plugins/functions.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/app/plugins/functions.py b/app/plugins/functions.py index 89de3605..dbaaac73 100644 --- a/app/plugins/functions.py +++ b/app/plugins/functions.py @@ -3,6 +3,7 @@ import logging import importlib import subprocess import traceback +import platform import django import json @@ -74,7 +75,10 @@ def build_plugins(): logger.info("Running npm install for {}".format(plugin)) try: - subprocess.call(['npm', 'install'], cwd=plugin.get_path("public")) + npm = "npm" + if platform.system() == "Windows": + npm = "npm.cmd" + subprocess.call([npm, 'install'], cwd=plugin.get_path("public")) except FileNotFoundError: logger.warn("npm is not installed, will skip this plugin") continue @@ -112,7 +116,11 @@ def build_plugins(): logger.info("Running webpack for {}".format(plugin.get_name())) try: - subprocess.call(['webpack-cli'], cwd=plugin.get_path("public")) + webpack = "webpack-cli" + if platform.system() == "Windows": + webpack = "webpack-cli.cmd" + + subprocess.call([webpack], cwd=plugin.get_path("public")) except FileNotFoundError: logger.warn("webpack-cli is not installed, plugin will not work") From fc66279eb445861814d74a38cd63c38ef193ceba Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Sat, 29 Feb 2020 15:28:21 -0500 Subject: [PATCH 4/6] Updated requirements.txt --- requirements.txt | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/requirements.txt b/requirements.txt index 14a32f67..90efaf19 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,8 @@ -amqp==2.2.2 +amqp==2.5.2 anyjson==0.3.3 appdirs==1.4.0 APScheduler==3.2.0 -billiard==3.5.0.3 +billiard==3.6.3.0 celery==4.4.0 coreapi==2.0.9 Django==2.1.15 @@ -39,7 +39,7 @@ psycopg2-binary==2.8.4 PyJWT==1.5.3 pyodm==1.5.3b1 pyparsing==2.1.10 -pytz==2018.3 +pytz==2019.3 rcssmin==1.0.6 redis==2.10.6 requests-toolbelt==0.9.1 @@ -51,7 +51,7 @@ six==1.11.0 strict-rfc3339==0.7 tzlocal==1.3 uritemplate==3.0.0 -vine==1.1.4 +vine==1.3.0 webcolors==1.5 -e git://github.com/OpenDroneMap/rio-tiler.git#egg=rio-tiler rio-color==1.0.0 @@ -60,4 +60,5 @@ rasterio==1.1.0 ; sys_platform == 'linux' or sys_platform == 'darwin' https://download.lfd.uci.edu/pythonlibs/s2jqpv5t/rasterio-1.1.3-cp38-cp38-win_amd64.whl ; sys_platform == "win32" https://download.lfd.uci.edu/pythonlibs/s2jqpv5t/GDAL-3.0.4-cp38-cp38-win_amd64.whl ; sys_platform == "win32" Shapely==1.7.0 ; sys_platform == "win32" -waitress==1.4.3 ; sys_platform == "win32" \ No newline at end of file +waitress==1.4.3 ; sys_platform == "win32" +gevent==1.4.0 ; sys_platform == "win32" \ No newline at end of file From c0899c96ffb3314eaa66642aceecfeba76f5f6c0 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Mon, 16 Mar 2020 14:46:22 -0400 Subject: [PATCH 5/6] GRASS engine working --- app/plugins/grass_engine.py | 30 ++++++++++++++++++++++++------ requirements.txt | 12 ++++++------ webodm/settings.py | 1 + 3 files changed, 31 insertions(+), 12 deletions(-) diff --git a/app/plugins/grass_engine.py b/app/plugins/grass_engine.py index 7695766b..3855a75f 100644 --- a/app/plugins/grass_engine.py +++ b/app/plugins/grass_engine.py @@ -3,6 +3,7 @@ import shutil import tempfile import subprocess import os +import platform from webodm import settings @@ -80,16 +81,33 @@ class GrassContext: # Create param list params = ["{}={}".format(opt,value) for opt,value in self.script_opts.items()] + # Track success, output + success = False + out = "" + err = "" + # Execute it logger.info("Executing grass script from {}: {} -c {} location --exec python {} {}".format(self.get_cwd(), self.grass_binary, self.location, script, " ".join(params))) - p = subprocess.Popen([self.grass_binary, '-c', self.location, 'location', '--exec', 'python', script] + params, - cwd=self.get_cwd(), stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out, err = p.communicate() + + command = [self.grass_binary, '-c', self.location, 'location', '--exec', 'python', script] + params + if platform.system() == "Windows": + # communicate() hangs on Windows so we use check_output instead + try: + out = subprocess.check_output(command, cwd=self.get_cwd()).decode('utf-8').strip() + success = True + except subprocess.CalledProcessError: + success = False + err = out + else: + p = subprocess.Popen(command, cwd=self.get_cwd(), stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + out, err = p.communicate() - out = out.decode('utf-8').strip() - err = err.decode('utf-8').strip() + out = out.decode('utf-8').strip() + err = err.decode('utf-8').strip() + success = p.returncode == 0 - if p.returncode == 0: + if success: return out else: raise GrassEngineException("Could not execute GRASS script {} from {}: {}".format(script, self.get_cwd(), err)) diff --git a/requirements.txt b/requirements.txt index 90efaf19..a877bc8a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -25,7 +25,7 @@ geojson==2.3.0 gunicorn==19.7.1 itypes==1.1.0 kombu==4.6.7 -libsass==0.13.3 +libsass==0.19.4 Markdown==2.6.7 olefile==0.44 openapi-codec==1.1.7 @@ -41,7 +41,7 @@ pyodm==1.5.3b1 pyparsing==2.1.10 pytz==2019.3 rcssmin==1.0.6 -redis==2.10.6 +redis==3.2.0 requests-toolbelt==0.9.1 requests==2.21.0 rfc3987==1.3.7 @@ -57,8 +57,8 @@ webcolors==1.5 rio-color==1.0.0 rio-cogeo==1.1.8 rasterio==1.1.0 ; sys_platform == 'linux' or sys_platform == 'darwin' -https://download.lfd.uci.edu/pythonlibs/s2jqpv5t/rasterio-1.1.3-cp38-cp38-win_amd64.whl ; sys_platform == "win32" -https://download.lfd.uci.edu/pythonlibs/s2jqpv5t/GDAL-3.0.4-cp38-cp38-win_amd64.whl ; sys_platform == "win32" +https://download.lfd.uci.edu/pythonlibs/s2jqpv5t/rasterio-1.1.3-cp37-cp37m-win_amd64.whl ; sys_platform == "win32" +https://download.lfd.uci.edu/pythonlibs/s2jqpv5t/GDAL-3.0.4-cp37-cp37m-win_amd64.whl ; sys_platform == "win32" Shapely==1.7.0 ; sys_platform == "win32" -waitress==1.4.3 ; sys_platform == "win32" -gevent==1.4.0 ; sys_platform == "win32" \ No newline at end of file +eventlet==0.25.1 ; sys_platform == "win32" +pyopenssl==19.1.0 ; sys_platform == "win32" \ No newline at end of file diff --git a/webodm/settings.py b/webodm/settings.py index c3420acf..33364ba6 100644 --- a/webodm/settings.py +++ b/webodm/settings.py @@ -338,6 +338,7 @@ CELERY_WORKER_HIJACK_ROOT_LOGGER = False if platform.system() == "Windows": GDAL_LIBRARY_PATH = ".venv/Lib/site-packages/osgeo/gdal300" GEOS_LIBRARY_PATH = ".venv/Lib/site-packages/shapely/DLLs/geos_c" + if TESTING: CELERY_TASK_ALWAYS_EAGER = True From 3caf313921573f8eb104a30be5800b0134e47f2d Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Mon, 16 Mar 2020 15:51:41 -0400 Subject: [PATCH 6/6] Removed platform dependent settings --- webodm/settings.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/webodm/settings.py b/webodm/settings.py index 33364ba6..2e56ecb5 100644 --- a/webodm/settings.py +++ b/webodm/settings.py @@ -10,7 +10,7 @@ For the full list of settings and their values, see https://docs.djangoproject.com/en/1.10/ref/settings/ """ -import os, sys, json, platform +import os, sys, json import datetime @@ -335,10 +335,6 @@ CELERY_INCLUDE=['worker.tasks'] CELERY_WORKER_REDIRECT_STDOUTS = False CELERY_WORKER_HIJACK_ROOT_LOGGER = False -if platform.system() == "Windows": - GDAL_LIBRARY_PATH = ".venv/Lib/site-packages/osgeo/gdal300" - GEOS_LIBRARY_PATH = ".venv/Lib/site-packages/shapely/DLLs/geos_c" - if TESTING: CELERY_TASK_ALWAYS_EAGER = True