Plugins refactoring for dynamic URL patterns

app/admin.py

@@ -9,7 +9,7 @@ from guardian.admin import GuardedModelAdmin
 from app.models import PluginDatum
 from app.models import Preset
 from app.models import Plugin
-from app.plugins import get_plugin_by_name, enable_plugin, disable_plugin
+from app.plugins import get_plugin_by_name, enable_plugin, disable_plugin, delete_plugin
 from .models import Project, Task, ImageUpload, Setting, Theme
 from django import forms
 from codemirror2.widgets import CodeMirrorEditor
@@ -105,6 +105,11 @@ class PluginAdmin(admin.ModelAdmin):
                 self.admin_site.admin_view(self.plugin_disable),
                 name='plugin-disable',
             ),
+            url(
+                r'^(?P<plugin_name>.+)/delete/$',
+                self.admin_site.admin_view(self.plugin_delete),
+                name='plugin-delete',
+            ),
         ]
         return custom_urls + urls
 
@@ -124,14 +129,30 @@ class PluginAdmin(admin.ModelAdmin):
 
         return HttpResponseRedirect(reverse('admin:app_plugin_changelist'))
 
+    def plugin_delete(self, request, plugin_name, *args, **kwargs):
+        try:
+            delete_plugin(plugin_name)
+        except Exception as e:
+            messages.warning(request, "Cannot delete plugin {}: {}".format(plugin_name, str(e)))
+
+        return HttpResponseRedirect(reverse('admin:app_plugin_changelist'))
+
+
     def plugin_actions(self, obj):
+        plugin = get_plugin_by_name(obj.name, only_active=False)
         return format_html(
             '<a class="button" href="{}" {}>Disable</a>&nbsp;'
-            '<a class="button" href="{}" {}>Enable</a>',
+            '<a class="button" href="{}" {}>Enable</a>'
+            + ('&nbsp;<a class="button" href="{}" onclick="return confirm(\'Are you sure you want to delete {}?\')"><i class="fa fa-trash"></i></a>' if not plugin.is_persistent() else '&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;')
+            ,
             reverse('admin:plugin-disable', args=[obj.pk]) if obj.enabled else '#',
             'disabled' if not obj.enabled else '',
             reverse('admin:plugin-enable', args=[obj.pk]) if not obj.enabled else '#',
             'disabled' if obj.enabled else '',
+
+            # TODO
+            reverse('admin:plugin-delete', args=[obj.pk]),
+            obj.name
         )
 
     plugin_actions.short_description = 'Actions'

app/api/urls.py

@@ -1,7 +1,7 @@
 from django.conf.urls import url, include
 
 from app.api.presets import PresetViewSet
-from app.plugins import get_api_url_patterns
+from app.plugins.views import api_view_handler
 from .projects import ProjectViewSet
 from .tasks import TaskViewSet, TaskDownloads, TaskAssets, TaskAssetsImport
 from .processingnodes import ProcessingNodeViewSet, ProcessingNodeOptionsView
@@ -49,6 +49,6 @@ urlpatterns = [
 
     url(r'^auth/', include('rest_framework.urls')),
     url(r'^token-auth/', obtain_jwt_token),
-]
 
-urlpatterns += get_api_url_patterns()
+    url(r'^plugins/(?P<plugin_name>[^/.]+)/(.*)$', api_view_handler)
+]

app/plugins/functions.py

@@ -59,7 +59,7 @@ def sync_plugin_db():
             defaults={'enabled': not disabled},
         )
         if created:
-            logger.info("Added [{}] plugin to database".format(plugin.get_name()))
+            logger.info("Added [{}] plugin to database".format(plugin))
 
 
 def clear_plugins_cache():
@@ -150,40 +150,6 @@ def register_plugins():
             logger.warning("Cannot register {}: {}".format(plugin, str(e)))
 
 
-def get_app_url_patterns():
-    """
-    @return the patterns to expose the /public directory of each plugin (if needed) and
-        each mount point
-    """
-    url_patterns = []
-    for plugin in get_plugins():
-        for mount_point in plugin.app_mount_points():
-            url_patterns.append(url('^plugins/{}/{}'.format(plugin.get_name(), mount_point.url),
-                                mount_point.view,
-                                *mount_point.args,
-                                **mount_point.kwargs))
-
-        if plugin.path_exists("public"):
-            url_patterns.append(url('^plugins/{}/(.*)'.format(plugin.get_name()),
-                                    django.views.static.serve,
-                                    {'document_root': plugin.get_path("public")}))
-
-    return url_patterns
-
-def get_api_url_patterns():
-    """
-    @return the patterns to expose the plugin API mount points (if any)
-    """
-    url_patterns = []
-    for plugin in get_plugins():
-        for mount_point in plugin.api_mount_points():
-            url_patterns.append(url('^plugins/{}/{}'.format(plugin.get_name(), mount_point.url),
-                                mount_point.view,
-                                *mount_point.args,
-                                **mount_point.kwargs))
-
-    return url_patterns
-
 plugins = None
 def get_plugins():
     """
@@ -193,51 +159,60 @@ def get_plugins():
     global plugins
     if plugins != None: return plugins
 
-    plugins_path = get_plugins_path()
+    plugins_paths = get_plugins_paths()
     plugins = []
 
-    for dir in [d for d in os.listdir(plugins_path) if os.path.isdir(plugins_path)]:
-        # Each plugin must have a manifest.json and a plugin.py
-        plugin_path = os.path.join(plugins_path, dir)
-        pluginpy_path = os.path.join(plugin_path, "plugin.py")
-        manifest_path = os.path.join(plugin_path, "manifest.json")
+    for plugins_path in plugins_paths:
+        for dir in [d for d in os.listdir(plugins_path) if os.path.isdir(plugins_path)]:
+            # Each plugin must have a manifest.json and a plugin.py
+            plugin_path = os.path.join(plugins_path, dir)
+            pluginpy_path = os.path.join(plugin_path, "plugin.py")
+            manifest_path = os.path.join(plugin_path, "manifest.json")
 
-        # Do not load test plugin unless we're in test mode
-        if os.path.basename(plugin_path) == 'test' and not settings.TESTING:
-            continue
-
-        # Ignore .gitignore
-        if os.path.basename(plugin_path) == '.gitignore':
-            continue
-
-        # Check plugin required files
-        if not os.path.isfile(manifest_path) or not os.path.isfile(pluginpy_path):
-            logger.warning("Found invalid plugin in {}".format(plugin_path))
-            continue
-
-        # Instantiate the plugin
-        try:
-            module = importlib.import_module("plugins.{}".format(dir))
-            plugin = (getattr(module, "Plugin"))()
-
-            # Check version
-            manifest = plugin.get_manifest()
-            if 'webodmMinVersion' in manifest:
-                min_version = manifest['webodmMinVersion']
-
-                if versionToInt(min_version) > versionToInt(settings.VERSION):
-                    logger.warning(
-                        "In {} webodmMinVersion is set to {} but WebODM version is {}. Plugin will not be loaded. Update WebODM.".format(
-                            manifest_path, min_version, settings.VERSION))
-                    continue
-
-            # Skip plugins in blacklist
-            if plugin.get_name() in settings.PLUGINS_BLACKLIST:
+            # Do not load test plugin unless we're in test mode
+            if os.path.basename(plugin_path) == 'test' and not settings.TESTING:
                 continue
 
-            plugins.append(plugin)
-        except Exception as e:
-            logger.warning("Failed to instantiate plugin {}: {}".format(dir, e))
+            # Ignore .gitignore
+            if os.path.basename(plugin_path) == '.gitignore':
+                continue
+
+            # Check plugin required files
+            if not os.path.isfile(manifest_path) or not os.path.isfile(pluginpy_path):
+                continue
+
+            # Instantiate the plugin
+            try:
+                try:
+                    module = importlib.import_module("app.media.plugins.{}".format(dir))
+                    plugin = (getattr(module, "Plugin"))()
+                except (ModuleNotFoundError, AttributeError):
+                    module = importlib.import_module("plugins.{}".format(dir))
+                    plugin = (getattr(module, "Plugin"))()
+
+                # Check version
+                manifest = plugin.get_manifest()
+                if 'webodmMinVersion' in manifest:
+                    min_version = manifest['webodmMinVersion']
+
+                    if versionToInt(min_version) > versionToInt(settings.VERSION):
+                        logger.warning(
+                            "In {} webodmMinVersion is set to {} but WebODM version is {}. Plugin will not be loaded. Update WebODM.".format(
+                                manifest_path, min_version, settings.VERSION))
+                        continue
+
+                # Skip plugins in blacklist
+                if plugin.get_name() in settings.PLUGINS_BLACKLIST:
+                    continue
+
+                # Skip plugins already added
+                if plugin.get_name() in [p.get_name() for p in plugins]:
+                    logger.warning("Duplicate plugin name found in {}, skipping".format(plugin_path))
+                    continue
+
+                plugins.append(plugin)
+            except Exception as e:
+                logger.warning("Failed to instantiate plugin {}: {}".format(dir, e))
 
     return plugins
 
@@ -281,17 +256,24 @@ def get_current_plugin():
     """
     caller_filename = traceback.extract_stack()[-2][0]
 
-    relp = os.path.relpath(caller_filename, get_plugins_path())
-    parts = relp.split(os.sep)
-    if len(parts) > 0:
-        plugin_name = parts[0]
-        return get_plugin_by_name(plugin_name, only_active=False)
+    for p in get_plugins_paths():
+        relp = os.path.relpath(caller_filename, p)
+        if ".." in relp:
+            continue
+
+        parts = relp.split(os.sep)
+        if len(parts) > 0:
+            plugin_name = parts[0]
+            return get_plugin_by_name(plugin_name, only_active=False)
 
     return None
 
-def get_plugins_path():
+def get_plugins_paths():
     current_path = os.path.dirname(os.path.realpath(__file__))
-    return os.path.abspath(os.path.join(current_path, "..", "..", "plugins"))
+    return [
+        os.path.abspath(get_plugins_persistent_path()),
+        os.path.abspath(os.path.join(current_path, "..", "..", "plugins")),
+    ]
 
 def get_plugins_persistent_path(*paths):
     return os.path.join(settings.MEDIA_ROOT, "plugins", *paths)
@@ -322,6 +304,9 @@ def enable_plugin(plugin_name):
 def disable_plugin(plugin_name):
     Plugin.objects.get(pk=plugin_name).disable()
 
+def delete_plugin(plugin_name):
+    Plugin.objects.get(pk=plugin_name).disable()
+
 def get_site_settings():
     return Setting.objects.first()
 

app/plugins/plugin_base.py

@@ -1,4 +1,3 @@
-import importlib
 import json
 import logging, os, sys, subprocess
 from abc import ABC
@@ -120,12 +119,22 @@ class PluginBase(ABC):
         """
         return "/plugins/{}/{}".format(self.get_name(), path)
 
+    def is_persistent(self):
+        """
+        :return: whether this plugin is persistent (stored in the /plugins directory,
+                instead of /app/media/plugins which are transient)
+        """
+        return ".." in os.path.relpath(self.get_path(), get_plugins_persistent_path())
+
     def template_path(self, path):
         """
         :param path: unix-style path
         :return: path used to reference Django templates for a plugin
         """
-        return "plugins/{}/templates/{}".format(self.get_name(), path)
+        if self.is_persistent():
+            return "plugins/{}/templates/{}".format(self.get_name(), path)
+        else:
+            return "app/media/plugins/{}/templates/{}".format(self.get_name(), path)
 
     def path_exists(self, path):
         return os.path.exists(self.get_path(path))

app/plugins/templates/webpack.config.js.tmpl

@@ -1,5 +1,8 @@
 // Magic to include node_modules of root WebODM's directory
-process.env.NODE_PATH = "../../../node_modules";
+const fs = require('fs');
+let webodmRoot = "../../../"; // Assuming plugins/<name>/public
+if (!fs.existsSync(webodmRoot + "webodm.sh")) webodmRoot = "../../../../../"; // Assuming app/media/plugins/<name>/public
+process.env.NODE_PATH = webodmRoot + "node_modules";
 require("module").Module._initPaths();
 
 let path = require("path");
@@ -69,7 +72,7 @@ module.exports = {
     modules: ['node_modules', 'bower_components'],
     extensions: ['.js', '.jsx'],
     alias: {
-        webodm: path.resolve(__dirname, '../../../app/static/app/js')
+        webodm: path.resolve(__dirname, webodmRoot + 'app/static/app/js')
     }
   },
 

app/plugins/views.py

@@ -1,3 +1,60 @@
+import os
+
 from app.api.tasks import TaskNestedView as TaskView
 from app.api.workers import CheckTask as CheckTask
-from app.api.workers import GetTaskResult as GetTaskResult
+from app.api.workers import GetTaskResult as GetTaskResult
+
+from django.http import HttpResponse, Http404
+from .functions import get_plugin_by_name
+from django.conf.urls import url
+from django.views.static import serve
+
+
+def try_resolve_url(request, url):
+    res = url.resolve(request.get_full_path())
+    if res:
+        return res
+    else:
+        return (None, None, None)
+
+def app_view_handler(request, plugin_name=None):
+    plugin = get_plugin_by_name(plugin_name) # TODO: this pings the server, which might be bad for performance with very large amount of files
+    if plugin is None:
+        raise Http404("Plugin not found")
+
+    # Try mountpoints first
+    for mount_point in plugin.app_mount_points():
+        view, args, kwargs = try_resolve_url(request, url(r'^/plugins/{}/{}'.format(plugin_name, mount_point.url),
+                                                 mount_point.view,
+                                                 *mount_point.args,
+                                                 **mount_point.kwargs))
+
+        if view:
+            return view(request, *args, **kwargs)
+
+    # Try public assets
+    if os.path.exists(plugin.get_path("public")):
+        view, args, kwargs = try_resolve_url(request, url('^/plugins/{}/(.*)'.format(plugin_name),
+                                                            serve,
+                                                            {'document_root': plugin.get_path("public")}))
+        if view:
+            return view(request, *args, **kwargs)
+
+    raise Http404("No valid routes")
+
+
+def api_view_handler(request, plugin_name=None):
+    plugin = get_plugin_by_name(plugin_name) # TODO: this pings the server, which might be bad for performance with very large amount of files
+    if plugin is None:
+        raise Http404("Plugin not found")
+
+    for mount_point in plugin.api_mount_points():
+        view, args, kwargs = try_resolve_url(request, url(r'^/api/plugins/{}/{}'.format(plugin_name, mount_point.url),
+                                                 mount_point.view,
+                                                 *mount_point.args,
+                                                 **mount_point.kwargs))
+
+        if view:
+            return view(request, *args, **kwargs)
+
+    raise Http404("No valid routes")

app/urls.py

@@ -1,7 +1,7 @@
 from django.conf.urls import url, include
 
 from .views import app as app_views, public as public_views
-from .plugins import get_app_url_patterns
+from .plugins.views import app_view_handler
 
 from app.boot import boot
 from webodm import settings
@@ -35,11 +35,10 @@ urlpatterns = [
     url(r'^processingnode/([\d]+)/$', app_views.processing_node, name='processing_node'),
 
     url(r'^api/', include("app.api.urls")),
+
+    url(r'^plugins/(?P<plugin_name>[^/.]+)/(.*)$', app_view_handler)
 ]
 
 handler404 = app_views.handler404
 handler500 = app_views.handler500
 
-# TODO: is there a way to place plugins /public directories
-# into the static build directories and let nginx serve them?
-urlpatterns += get_app_url_patterns()