Support for requirements.txt, tweaks to elevationmap plugin

app/admin.py

@@ -1,5 +1,6 @@
 from django.conf.urls import url
 from django.contrib import admin
+from django.contrib import messages
 from django.http import HttpResponseRedirect
 from django.urls import reverse
 from django.utils.html import format_html
@@ -8,7 +9,7 @@ from guardian.admin import GuardedModelAdmin
 from app.models import PluginDatum
 from app.models import Preset
 from app.models import Plugin
-from app.plugins import get_plugin_by_name
+from app.plugins import get_plugin_by_name, enable_plugin, disable_plugin
 from .models import Project, Task, ImageUpload, Setting, Theme
 from django import forms
 from codemirror2.widgets import CodeMirrorEditor
@@ -108,15 +109,19 @@ class PluginAdmin(admin.ModelAdmin):
         return custom_urls + urls
 
     def plugin_enable(self, request, plugin_name, *args, **kwargs):
-        p = Plugin.objects.get(pk=plugin_name)
-        p.enabled = True
-        p.save()
+        try:
+            enable_plugin(plugin_name)
+        except Exception as e:
+            messages.warning(request, "Cannot enable plugin {}: {}".format(plugin_name, str(e)))
+
         return HttpResponseRedirect(reverse('admin:app_plugin_changelist'))
 
     def plugin_disable(self, request, plugin_name, *args, **kwargs):
-        p = Plugin.objects.get(pk=plugin_name)
-        p.enabled = False
-        p.save()
+        try:
+            disable_plugin(plugin_name)
+        except Exception as e:
+            messages.warning(request, "Cannot disable plugin {}: {}".format(plugin_name, str(e)))
+
         return HttpResponseRedirect(reverse('admin:app_plugin_changelist'))
 
     def plugin_actions(self, obj):

app/models/plugin.py

@@ -1,9 +1,16 @@
 from django.db import models
 
-
 class Plugin(models.Model):
     name = models.CharField(max_length=255, primary_key=True, blank=False, null=False, help_text="Plugin name")
     enabled = models.BooleanField(db_index=True, default=True, help_text="Whether this plugin is enabled.")
 
     def __str__(self):
-        return self.name
+        return self.name
+
+    def enable(self):
+        self.enabled = True
+        self.save()
+
+    def disable(self):
+        self.enabled = False
+        self.save()

app/plugins/__init__.py

@@ -3,5 +3,3 @@ from .plugin_base import PluginBase
 from .menu import Menu
 from .mount_point import MountPoint
 from .functions import *
-
-

app/plugins/functions.py

@@ -2,6 +2,7 @@ import os
 import logging
 import importlib
 import subprocess
+import traceback
 
 import django
 import json
@@ -18,6 +19,10 @@ from webodm import settings
 logger = logging.getLogger('app.logger')
 
 def init_plugins():
+    # Make sure app/media/plugins exists
+    if not os.path.exists(get_plugins_persistent_path()):
+        os.mkdir(get_plugins_persistent_path())
+
     build_plugins()
     sync_plugin_db()
     register_plugins()
@@ -105,8 +110,12 @@ def build_plugins():
 
 def register_plugins():
     for plugin in get_active_plugins():
-        plugin.register()
-        logger.info("Registered {}".format(plugin))
+        try:
+            plugin.register()
+            logger.info("Registered {}".format(plugin))
+        except Exception as e:
+            disable_plugin(plugin.get_name())
+            logger.warning("Cannot register {}: {}".format(plugin, str(e)))
 
 
 def get_app_url_patterns():
@@ -115,7 +124,7 @@ def get_app_url_patterns():
         each mount point
     """
     url_patterns = []
-    for plugin in get_active_plugins():
+    for plugin in get_plugins():
         for mount_point in plugin.app_mount_points():
             url_patterns.append(url('^plugins/{}/{}'.format(plugin.get_name(), mount_point.url),
                                 mount_point.view,
@@ -134,7 +143,7 @@ def get_api_url_patterns():
     @return the patterns to expose the plugin API mount points (if any)
     """
     url_patterns = []
-    for plugin in get_active_plugins():
+    for plugin in get_plugins():
         for mount_point in plugin.api_mount_points():
             url_patterns.append(url('^plugins/{}/{}'.format(plugin.get_name(), mount_point.url),
                                 mount_point.view,
@@ -228,10 +237,28 @@ def get_plugin_by_name(name, only_active=True, refresh_cache_if_none=False):
     else:
         return res
 
+def get_current_plugin():
+    """
+    When called from a python module inside a plugin's directory,
+    it returns the plugin that this python module belongs to
+    :return: Plugin instance
+    """
+    caller_filename = traceback.extract_stack()[-2][0]
+
+    relp = os.path.relpath(caller_filename, get_plugins_path())
+    parts = relp.split(os.sep)
+    if len(parts) > 0:
+        plugin_name = parts[0]
+        return get_plugin_by_name(plugin_name, only_active=False, refresh_cache_if_none=True)
+
+    return None
+
 def get_plugins_path():
     current_path = os.path.dirname(os.path.realpath(__file__))
     return os.path.abspath(os.path.join(current_path, "..", "..", "plugins"))
 
+def get_plugins_persistent_path(*paths):
+    return os.path.join(settings.MEDIA_ROOT, "plugins", *paths)
 
 def get_dynamic_script_handler(script_path, callback=None, **kwargs):
     def handleRequest(request):
@@ -251,11 +278,17 @@ def get_dynamic_script_handler(script_path, callback=None, **kwargs):
 
     return handleRequest
 
+def enable_plugin(plugin_name):
+    p = get_plugin_by_name(plugin_name, only_active=False)
+    p.register()
+    Plugin.objects.get(pk=plugin_name).enable()
+
+def disable_plugin(plugin_name):
+    Plugin.objects.get(pk=plugin_name).disable()
 
 def get_site_settings():
     return Setting.objects.first()
 
-
 def versionToInt(version):
     """
     Converts a WebODM version string (major.minor.build) to a integer value

app/plugins/plugin_base.py

@@ -1,7 +1,12 @@
+import importlib
 import json
-import logging, os, sys
+import logging, os, sys, subprocess
 from abc import ABC
 from app.plugins import UserDataStore, GlobalDataStore
+from app.plugins.functions import get_plugins_persistent_path
+from contextlib import contextmanager
+
+from app.plugins.pyutils import requirements_installed, compute_file_md5
 
 logger = logging.getLogger('app.logger')
 
@@ -11,7 +16,64 @@ class PluginBase(ABC):
         self.manifest = None
 
     def register(self):
-        pass
+        self.check_requirements()
+
+
+    def check_requirements(self):
+        """
+        Check if Python requirements need to be installed
+        """
+        req_file = self.get_path("requirements.txt")
+        if os.path.exists(req_file):
+            reqs_installed =  requirements_installed(req_file, self.get_python_packages_path())
+
+            md5_file = self.get_python_packages_path("install_md5")
+            md5_mismatch = False
+            req_md5 = compute_file_md5(req_file)
+
+            if os.path.exists(md5_file):
+                with open(md5_file, 'r') as f:
+                    md5_mismatch = f.read().strip() != req_md5
+
+
+            if not reqs_installed or md5_mismatch:
+                logger.info("Installing requirements.txt for {}".format(self))
+
+                if not os.path.exists(self.get_python_packages_path()):
+                    os.makedirs(self.get_python_packages_path(), exist_ok=True)
+
+                p = subprocess.Popen(['pip', 'install', '-U', '-r', 'requirements.txt',
+                                  '--target', self.get_python_packages_path()],
+                                     cwd=self.get_path())
+                p.wait()
+
+                # Verify
+                if requirements_installed(self.get_path("requirements.txt"), self.get_python_packages_path()):
+                    logger.info("Installed requirements.txt for {}".format(self))
+
+                    # Write MD5
+                    if req_md5:
+                        with open(md5_file, 'w') as f:
+                            f.write(req_md5)
+                else:
+                    logger.warning("Failed to install requirements.txt for {}".format(self))
+
+    def get_persistent_path(self, *paths):
+        return get_plugins_persistent_path(self.name, *paths)
+
+    def get_python_packages_path(self, *paths):
+        return self.get_persistent_path("site-packages", *paths)
+
+    @contextmanager
+    def python_imports(self):
+        # Add python path
+        sys.path.insert(0, self.get_python_packages_path())
+        try:
+            yield
+        finally:
+            # Remove python path
+            sys.path.remove(self.get_python_packages_path())
+
 
     def get_path(self, *paths):
         """

app/plugins/pyutils.py

@@ -0,0 +1,37 @@
+import os
+import re
+import subprocess
+import hashlib
+
+def parse_requirements(requirements_file):
+    """
+    Parse a requirements.txt file
+    :param requirements_file: path to requirements.txt file
+    :return: package names
+    """
+    if os.path.exists(requirements_file):
+        with open(requirements_file, 'r') as f:
+            deps = list(filter(lambda x: len(x) > 0, map(str.strip, f.read().split('\n'))))
+            return [re.split('==|<=|>=|<|>', d)[0] for d in deps]
+
+    return []
+
+
+def requirements_installed(requirements_file, python_path):
+    """
+    Checks if the packages in requirements.txt have been installed in the specified
+    python path. Note that this does NOT check for versions, just package names
+    :param requirements_file: path to requirements.txt
+    :param python_path: path to directory where packages are installed
+    :return: True if all requirements are installed, false otherwise
+    """
+    env = os.environ.copy()
+    env["PYTHONPATH"] = env.get("PYTHONPATH", "") + ":" + python_path
+    reqs = subprocess.check_output(['pip', 'freeze'], env=env)
+    installed_packages = [r.decode().split('==')[0] for r in reqs.split()]
+    deps = parse_requirements(requirements_file)
+
+    return set(deps) & set(installed_packages) == set(deps)
+
+def compute_file_md5(filename):
+    return hashlib.md5(open(filename, 'rb').read()).hexdigest()

app/static/app/js/classes/plugins/Map.js

@@ -9,7 +9,7 @@ const leafletPreCheck = (options) => {
 };
 
 const layersControlPreCheck = (options) => {
-  assert(options.layersControl !== undefined);
+  assert(options.controls !== undefined);
   leafletPreCheck(options);
 }
 
@@ -17,8 +17,8 @@ export default {
   namespace: "Map",
 
   endpoints: [
-    ["willAddControls", layersControlPreCheck],
-    ["didAddControls", leafletPreCheck],
+    ["willAddControls", leafletPreCheck],
+    ["didAddControls", layersControlPreCheck],
 	["addActionButton", leafletPreCheck],
   ]
 };

app/static/app/js/components/Map.jsx

@@ -206,12 +206,17 @@ class Map extends React.Component {
       zoomControl: false
     });
 
-    Leaflet.control.scale({
+    PluginsAPI.Map.triggerWillAddControls({
+        map: this.map,
+        tiles
+    });
+
+    let scaleControl = Leaflet.control.scale({
       maxWidth: 250,
     }).addTo(this.map);
 
     //add zoom control with your options
-    Leaflet.control.zoom({
+    let zoomControl = Leaflet.control.zoom({
          position:'bottomleft'
     }).addTo(this.map);
 
@@ -258,12 +263,6 @@ https://a.tile.openstreetmap.org/{z}/{x}/{y}.png
       selectedOverlays: [],
       baseLayers: this.basemaps
     }).addTo(this.map);
-    
-    PluginsAPI.Map.triggerWillAddControls({
-      map: this.map,
-      layersControl: this.autolayers,
-      tiles
-    });
 
     this.map.fitWorld();
     this.map.attributionControl.setPrefix("");
@@ -316,7 +315,12 @@ https://a.tile.openstreetmap.org/{z}/{x}/{y}.png
 
     PluginsAPI.Map.triggerDidAddControls({
       map: this.map,
-      tiles: tiles
+      tiles: tiles,
+      controls:{
+        autolayers: this.autolayers,
+        scale: scaleControl,
+        zoom: zoomControl
+      }
     });
 
     PluginsAPI.Map.triggerAddActionButton({

package.json

@@ -1,6 +1,6 @@
 {
   "name": "WebODM",
-  "version": "1.1.0",
+  "version": "1.1.1",
   "description": "Open Source Drone Image Processing",
   "main": "index.js",
   "scripts": {

plugins/.gitignore

@@ -1,2 +1 @@
 webpack.config.js
-disabled

plugins/elevationmap/api.py

@@ -10,10 +10,12 @@ from app.plugins.views import TaskView
 from worker.tasks import execute_grass_script
 from app.plugins.grass_engine import grass, GrassEngineException, cleanup_grass_context
 from worker.celery import app as celery
+from app.plugins import get_current_plugin
 
 class TaskElevationMapGenerate(TaskView):
     def post(self, request, pk=None):
         task = self.get_and_check_task(request, pk)
+        plugin = get_current_plugin()
         
         if task.dsm_extent is None:
             return Response({'error': 'No DSM layer is available.'}, status=status.HTTP_400_BAD_REQUEST)
@@ -41,6 +43,8 @@ class TaskElevationMapGenerate(TaskView):
             context.add_param('noise_filter_size', noise_filter_size)
             context.add_param('epsg', epsg)
             context.add_param('python_script_path', os.path.join(current_dir, "elevationmap.py"))
+            context.add_param('python_path', plugin.get_python_packages_path())
+
             if dtm != None:
                 context.add_param('dtm', '--dtm {}'.format(dtm))
             else:

plugins/elevationmap/calc_elevation_map.grass

@@ -4,6 +4,7 @@
 # noise_filter_size: Area in meters where we will clean up noise in the contours
 # epsg: target EPSG code
 # python_script_path: Path of the python script
+# python_path: Path to python modules
 # dtm: Optional text to include the GeoTIFF DTM
 #
 # ------
@@ -20,7 +21,7 @@ elif [ "${format}" = "ESRI Shapefile" ]; then
     ext="shp"
 fi
 
-${python_script_path} "${dsm_file}" ${interval} --epsg ${epsg} --noise_filter_size ${noise_filter_size} ${dtm} -o output.json
+PYTHONPATH="${python_path}" "${python_script_path}" "${dsm_file}" ${interval} --epsg ${epsg} --noise_filter_size ${noise_filter_size} ${dtm} -o output.json
 
 if [ $$ext != "json" ]; then
   ogr2ogr -f "${format}" output.$$ext output.json > /dev/null

plugins/elevationmap/manifest.json

@@ -1,6 +1,6 @@
 {
 	"name": "ElevationMap",
-	"webodmMinVersion": "1.1.0",
+	"webodmMinVersion": "1.1.1",
 	"description": "Calculate and draw an elevation map based on a task's DEMs",
 	"version": "1.0.0",
 	"author": "Nicolas Chamo",

plugins/elevationmap/plugin.py

@@ -4,7 +4,6 @@ from .api import TaskElevationMapGenerate
 from .api import TaskElevationMapCheck
 from .api import TaskElevationMapDownload
 
-
 class Plugin(PluginBase):
     def include_js_files(self):
         return ['main.js']

plugins/elevationmap/public/main.js

@@ -1,4 +1,4 @@
-PluginsAPI.Map.willAddControls([
+PluginsAPI.Map.didAddControls([
     	'elevationmap/build/ElevationMap.js',
     	'elevationmap/build/ElevationMap.css'
 	], function(args, ElevationMap){
@@ -9,6 +9,6 @@ PluginsAPI.Map.willAddControls([
 
 	// TODO: add support for map view where multiple tasks are available?
 	if (tasks.length === 1){
-		args.map.addControl(new ElevationMap({map: args.map, layersControl: args.layersControl, tasks: tasks}));
+		args.map.addControl(new ElevationMap({map: args.map, layersControl: args.controls.autolayers, tasks: tasks}));
 	}
 });