From 11e3e42b060214f6c3ed541836cc9dcd30392b0b Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Tue, 9 Jul 2019 18:08:28 -0400 Subject: [PATCH] Support for requirements.txt, tweaks to elevationmap plugin --- app/admin.py | 19 ++++-- app/models/plugin.py | 11 +++- app/plugins/__init__.py | 2 - app/plugins/functions.py | 43 ++++++++++-- app/plugins/plugin_base.py | 66 ++++++++++++++++++- app/plugins/pyutils.py | 37 +++++++++++ app/static/app/js/classes/plugins/Map.js | 6 +- app/static/app/js/components/Map.jsx | 22 ++++--- package.json | 2 +- plugins/.gitignore | 1 - plugins/elevationmap/api.py | 4 ++ plugins/elevationmap/calc_elevation_map.grass | 3 +- plugins/elevationmap/disabled | 0 plugins/elevationmap/manifest.json | 2 +- plugins/elevationmap/plugin.py | 1 - plugins/elevationmap/public/main.js | 4 +- .../elevationmap/requirements.txt | 0 17 files changed, 186 insertions(+), 37 deletions(-) create mode 100644 app/plugins/pyutils.py create mode 100644 plugins/elevationmap/disabled rename requirements3.txt => plugins/elevationmap/requirements.txt (100%) diff --git a/app/admin.py b/app/admin.py index a64ff6ce..f32125d5 100644 --- a/app/admin.py +++ b/app/admin.py @@ -1,5 +1,6 @@ from django.conf.urls import url from django.contrib import admin +from django.contrib import messages from django.http import HttpResponseRedirect from django.urls import reverse from django.utils.html import format_html @@ -8,7 +9,7 @@ from guardian.admin import GuardedModelAdmin from app.models import PluginDatum from app.models import Preset from app.models import Plugin -from app.plugins import get_plugin_by_name +from app.plugins import get_plugin_by_name, enable_plugin, disable_plugin from .models import Project, Task, ImageUpload, Setting, Theme from django import forms from codemirror2.widgets import CodeMirrorEditor @@ -108,15 +109,19 @@ class PluginAdmin(admin.ModelAdmin): return custom_urls + urls def plugin_enable(self, request, plugin_name, *args, **kwargs): - p = Plugin.objects.get(pk=plugin_name) - p.enabled = True - p.save() + try: + enable_plugin(plugin_name) + except Exception as e: + messages.warning(request, "Cannot enable plugin {}: {}".format(plugin_name, str(e))) + return HttpResponseRedirect(reverse('admin:app_plugin_changelist')) def plugin_disable(self, request, plugin_name, *args, **kwargs): - p = Plugin.objects.get(pk=plugin_name) - p.enabled = False - p.save() + try: + disable_plugin(plugin_name) + except Exception as e: + messages.warning(request, "Cannot disable plugin {}: {}".format(plugin_name, str(e))) + return HttpResponseRedirect(reverse('admin:app_plugin_changelist')) def plugin_actions(self, obj): diff --git a/app/models/plugin.py b/app/models/plugin.py index f054f01b..7047369b 100644 --- a/app/models/plugin.py +++ b/app/models/plugin.py @@ -1,9 +1,16 @@ from django.db import models - class Plugin(models.Model): name = models.CharField(max_length=255, primary_key=True, blank=False, null=False, help_text="Plugin name") enabled = models.BooleanField(db_index=True, default=True, help_text="Whether this plugin is enabled.") def __str__(self): - return self.name \ No newline at end of file + return self.name + + def enable(self): + self.enabled = True + self.save() + + def disable(self): + self.enabled = False + self.save() \ No newline at end of file diff --git a/app/plugins/__init__.py b/app/plugins/__init__.py index cc82df2a..22666a83 100644 --- a/app/plugins/__init__.py +++ b/app/plugins/__init__.py @@ -3,5 +3,3 @@ from .plugin_base import PluginBase from .menu import Menu from .mount_point import MountPoint from .functions import * - - diff --git a/app/plugins/functions.py b/app/plugins/functions.py index d7313200..ff09985a 100644 --- a/app/plugins/functions.py +++ b/app/plugins/functions.py @@ -2,6 +2,7 @@ import os import logging import importlib import subprocess +import traceback import django import json @@ -18,6 +19,10 @@ from webodm import settings logger = logging.getLogger('app.logger') def init_plugins(): + # Make sure app/media/plugins exists + if not os.path.exists(get_plugins_persistent_path()): + os.mkdir(get_plugins_persistent_path()) + build_plugins() sync_plugin_db() register_plugins() @@ -105,8 +110,12 @@ def build_plugins(): def register_plugins(): for plugin in get_active_plugins(): - plugin.register() - logger.info("Registered {}".format(plugin)) + try: + plugin.register() + logger.info("Registered {}".format(plugin)) + except Exception as e: + disable_plugin(plugin.get_name()) + logger.warning("Cannot register {}: {}".format(plugin, str(e))) def get_app_url_patterns(): @@ -115,7 +124,7 @@ def get_app_url_patterns(): each mount point """ url_patterns = [] - for plugin in get_active_plugins(): + for plugin in get_plugins(): for mount_point in plugin.app_mount_points(): url_patterns.append(url('^plugins/{}/{}'.format(plugin.get_name(), mount_point.url), mount_point.view, @@ -134,7 +143,7 @@ def get_api_url_patterns(): @return the patterns to expose the plugin API mount points (if any) """ url_patterns = [] - for plugin in get_active_plugins(): + for plugin in get_plugins(): for mount_point in plugin.api_mount_points(): url_patterns.append(url('^plugins/{}/{}'.format(plugin.get_name(), mount_point.url), mount_point.view, @@ -228,10 +237,28 @@ def get_plugin_by_name(name, only_active=True, refresh_cache_if_none=False): else: return res +def get_current_plugin(): + """ + When called from a python module inside a plugin's directory, + it returns the plugin that this python module belongs to + :return: Plugin instance + """ + caller_filename = traceback.extract_stack()[-2][0] + + relp = os.path.relpath(caller_filename, get_plugins_path()) + parts = relp.split(os.sep) + if len(parts) > 0: + plugin_name = parts[0] + return get_plugin_by_name(plugin_name, only_active=False, refresh_cache_if_none=True) + + return None + def get_plugins_path(): current_path = os.path.dirname(os.path.realpath(__file__)) return os.path.abspath(os.path.join(current_path, "..", "..", "plugins")) +def get_plugins_persistent_path(*paths): + return os.path.join(settings.MEDIA_ROOT, "plugins", *paths) def get_dynamic_script_handler(script_path, callback=None, **kwargs): def handleRequest(request): @@ -251,11 +278,17 @@ def get_dynamic_script_handler(script_path, callback=None, **kwargs): return handleRequest +def enable_plugin(plugin_name): + p = get_plugin_by_name(plugin_name, only_active=False) + p.register() + Plugin.objects.get(pk=plugin_name).enable() + +def disable_plugin(plugin_name): + Plugin.objects.get(pk=plugin_name).disable() def get_site_settings(): return Setting.objects.first() - def versionToInt(version): """ Converts a WebODM version string (major.minor.build) to a integer value diff --git a/app/plugins/plugin_base.py b/app/plugins/plugin_base.py index a8a760a5..ad2fee3c 100644 --- a/app/plugins/plugin_base.py +++ b/app/plugins/plugin_base.py @@ -1,7 +1,12 @@ +import importlib import json -import logging, os, sys +import logging, os, sys, subprocess from abc import ABC from app.plugins import UserDataStore, GlobalDataStore +from app.plugins.functions import get_plugins_persistent_path +from contextlib import contextmanager + +from app.plugins.pyutils import requirements_installed, compute_file_md5 logger = logging.getLogger('app.logger') @@ -11,7 +16,64 @@ class PluginBase(ABC): self.manifest = None def register(self): - pass + self.check_requirements() + + + def check_requirements(self): + """ + Check if Python requirements need to be installed + """ + req_file = self.get_path("requirements.txt") + if os.path.exists(req_file): + reqs_installed = requirements_installed(req_file, self.get_python_packages_path()) + + md5_file = self.get_python_packages_path("install_md5") + md5_mismatch = False + req_md5 = compute_file_md5(req_file) + + if os.path.exists(md5_file): + with open(md5_file, 'r') as f: + md5_mismatch = f.read().strip() != req_md5 + + + if not reqs_installed or md5_mismatch: + logger.info("Installing requirements.txt for {}".format(self)) + + if not os.path.exists(self.get_python_packages_path()): + os.makedirs(self.get_python_packages_path(), exist_ok=True) + + p = subprocess.Popen(['pip', 'install', '-U', '-r', 'requirements.txt', + '--target', self.get_python_packages_path()], + cwd=self.get_path()) + p.wait() + + # Verify + if requirements_installed(self.get_path("requirements.txt"), self.get_python_packages_path()): + logger.info("Installed requirements.txt for {}".format(self)) + + # Write MD5 + if req_md5: + with open(md5_file, 'w') as f: + f.write(req_md5) + else: + logger.warning("Failed to install requirements.txt for {}".format(self)) + + def get_persistent_path(self, *paths): + return get_plugins_persistent_path(self.name, *paths) + + def get_python_packages_path(self, *paths): + return self.get_persistent_path("site-packages", *paths) + + @contextmanager + def python_imports(self): + # Add python path + sys.path.insert(0, self.get_python_packages_path()) + try: + yield + finally: + # Remove python path + sys.path.remove(self.get_python_packages_path()) + def get_path(self, *paths): """ diff --git a/app/plugins/pyutils.py b/app/plugins/pyutils.py new file mode 100644 index 00000000..3ac5f635 --- /dev/null +++ b/app/plugins/pyutils.py @@ -0,0 +1,37 @@ +import os +import re +import subprocess +import hashlib + +def parse_requirements(requirements_file): + """ + Parse a requirements.txt file + :param requirements_file: path to requirements.txt file + :return: package names + """ + if os.path.exists(requirements_file): + with open(requirements_file, 'r') as f: + deps = list(filter(lambda x: len(x) > 0, map(str.strip, f.read().split('\n')))) + return [re.split('==|<=|>=|<|>', d)[0] for d in deps] + + return [] + + +def requirements_installed(requirements_file, python_path): + """ + Checks if the packages in requirements.txt have been installed in the specified + python path. Note that this does NOT check for versions, just package names + :param requirements_file: path to requirements.txt + :param python_path: path to directory where packages are installed + :return: True if all requirements are installed, false otherwise + """ + env = os.environ.copy() + env["PYTHONPATH"] = env.get("PYTHONPATH", "") + ":" + python_path + reqs = subprocess.check_output(['pip', 'freeze'], env=env) + installed_packages = [r.decode().split('==')[0] for r in reqs.split()] + deps = parse_requirements(requirements_file) + + return set(deps) & set(installed_packages) == set(deps) + +def compute_file_md5(filename): + return hashlib.md5(open(filename, 'rb').read()).hexdigest() diff --git a/app/static/app/js/classes/plugins/Map.js b/app/static/app/js/classes/plugins/Map.js index 5126fc1f..d5abc807 100644 --- a/app/static/app/js/classes/plugins/Map.js +++ b/app/static/app/js/classes/plugins/Map.js @@ -9,7 +9,7 @@ const leafletPreCheck = (options) => { }; const layersControlPreCheck = (options) => { - assert(options.layersControl !== undefined); + assert(options.controls !== undefined); leafletPreCheck(options); } @@ -17,8 +17,8 @@ export default { namespace: "Map", endpoints: [ - ["willAddControls", layersControlPreCheck], - ["didAddControls", leafletPreCheck], + ["willAddControls", leafletPreCheck], + ["didAddControls", layersControlPreCheck], ["addActionButton", leafletPreCheck], ] }; diff --git a/app/static/app/js/components/Map.jsx b/app/static/app/js/components/Map.jsx index 2eaaf495..5303fe33 100644 --- a/app/static/app/js/components/Map.jsx +++ b/app/static/app/js/components/Map.jsx @@ -206,12 +206,17 @@ class Map extends React.Component { zoomControl: false }); - Leaflet.control.scale({ + PluginsAPI.Map.triggerWillAddControls({ + map: this.map, + tiles + }); + + let scaleControl = Leaflet.control.scale({ maxWidth: 250, }).addTo(this.map); //add zoom control with your options - Leaflet.control.zoom({ + let zoomControl = Leaflet.control.zoom({ position:'bottomleft' }).addTo(this.map); @@ -258,12 +263,6 @@ https://a.tile.openstreetmap.org/{z}/{x}/{y}.png selectedOverlays: [], baseLayers: this.basemaps }).addTo(this.map); - - PluginsAPI.Map.triggerWillAddControls({ - map: this.map, - layersControl: this.autolayers, - tiles - }); this.map.fitWorld(); this.map.attributionControl.setPrefix(""); @@ -316,7 +315,12 @@ https://a.tile.openstreetmap.org/{z}/{x}/{y}.png PluginsAPI.Map.triggerDidAddControls({ map: this.map, - tiles: tiles + tiles: tiles, + controls:{ + autolayers: this.autolayers, + scale: scaleControl, + zoom: zoomControl + } }); PluginsAPI.Map.triggerAddActionButton({ diff --git a/package.json b/package.json index 6b37b041..319c4d94 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "WebODM", - "version": "1.1.0", + "version": "1.1.1", "description": "Open Source Drone Image Processing", "main": "index.js", "scripts": { diff --git a/plugins/.gitignore b/plugins/.gitignore index ff22c659..6de001d8 100644 --- a/plugins/.gitignore +++ b/plugins/.gitignore @@ -1,2 +1 @@ webpack.config.js -disabled diff --git a/plugins/elevationmap/api.py b/plugins/elevationmap/api.py index 4ba613cf..0d63182d 100644 --- a/plugins/elevationmap/api.py +++ b/plugins/elevationmap/api.py @@ -10,10 +10,12 @@ from app.plugins.views import TaskView from worker.tasks import execute_grass_script from app.plugins.grass_engine import grass, GrassEngineException, cleanup_grass_context from worker.celery import app as celery +from app.plugins import get_current_plugin class TaskElevationMapGenerate(TaskView): def post(self, request, pk=None): task = self.get_and_check_task(request, pk) + plugin = get_current_plugin() if task.dsm_extent is None: return Response({'error': 'No DSM layer is available.'}, status=status.HTTP_400_BAD_REQUEST) @@ -41,6 +43,8 @@ class TaskElevationMapGenerate(TaskView): context.add_param('noise_filter_size', noise_filter_size) context.add_param('epsg', epsg) context.add_param('python_script_path', os.path.join(current_dir, "elevationmap.py")) + context.add_param('python_path', plugin.get_python_packages_path()) + if dtm != None: context.add_param('dtm', '--dtm {}'.format(dtm)) else: diff --git a/plugins/elevationmap/calc_elevation_map.grass b/plugins/elevationmap/calc_elevation_map.grass index 93d82a84..a23accd5 100755 --- a/plugins/elevationmap/calc_elevation_map.grass +++ b/plugins/elevationmap/calc_elevation_map.grass @@ -4,6 +4,7 @@ # noise_filter_size: Area in meters where we will clean up noise in the contours # epsg: target EPSG code # python_script_path: Path of the python script +# python_path: Path to python modules # dtm: Optional text to include the GeoTIFF DTM # # ------ @@ -20,7 +21,7 @@ elif [ "${format}" = "ESRI Shapefile" ]; then ext="shp" fi -${python_script_path} "${dsm_file}" ${interval} --epsg ${epsg} --noise_filter_size ${noise_filter_size} ${dtm} -o output.json +PYTHONPATH="${python_path}" "${python_script_path}" "${dsm_file}" ${interval} --epsg ${epsg} --noise_filter_size ${noise_filter_size} ${dtm} -o output.json if [ $$ext != "json" ]; then ogr2ogr -f "${format}" output.$$ext output.json > /dev/null diff --git a/plugins/elevationmap/disabled b/plugins/elevationmap/disabled new file mode 100644 index 00000000..e69de29b diff --git a/plugins/elevationmap/manifest.json b/plugins/elevationmap/manifest.json index d4ec8fed..ad883599 100644 --- a/plugins/elevationmap/manifest.json +++ b/plugins/elevationmap/manifest.json @@ -1,6 +1,6 @@ { "name": "ElevationMap", - "webodmMinVersion": "1.1.0", + "webodmMinVersion": "1.1.1", "description": "Calculate and draw an elevation map based on a task's DEMs", "version": "1.0.0", "author": "Nicolas Chamo", diff --git a/plugins/elevationmap/plugin.py b/plugins/elevationmap/plugin.py index 792e6f60..6f760abd 100644 --- a/plugins/elevationmap/plugin.py +++ b/plugins/elevationmap/plugin.py @@ -4,7 +4,6 @@ from .api import TaskElevationMapGenerate from .api import TaskElevationMapCheck from .api import TaskElevationMapDownload - class Plugin(PluginBase): def include_js_files(self): return ['main.js'] diff --git a/plugins/elevationmap/public/main.js b/plugins/elevationmap/public/main.js index 64ebe5d3..464a8d42 100644 --- a/plugins/elevationmap/public/main.js +++ b/plugins/elevationmap/public/main.js @@ -1,4 +1,4 @@ -PluginsAPI.Map.willAddControls([ +PluginsAPI.Map.didAddControls([ 'elevationmap/build/ElevationMap.js', 'elevationmap/build/ElevationMap.css' ], function(args, ElevationMap){ @@ -9,6 +9,6 @@ PluginsAPI.Map.willAddControls([ // TODO: add support for map view where multiple tasks are available? if (tasks.length === 1){ - args.map.addControl(new ElevationMap({map: args.map, layersControl: args.layersControl, tasks: tasks})); + args.map.addControl(new ElevationMap({map: args.map, layersControl: args.controls.autolayers, tasks: tasks})); } }); diff --git a/requirements3.txt b/plugins/elevationmap/requirements.txt similarity index 100% rename from requirements3.txt rename to plugins/elevationmap/requirements.txt