From 8c587d07c4899e55fa3caa52052cce4a3bf7c3a6 Mon Sep 17 00:00:00 2001 From: Agate Date: Wed, 17 Jun 2020 21:49:45 +0200 Subject: [PATCH] Moar plugins polishing and sugar --- api/config/plugins.py | 33 ++++++++- api/config/settings/common.py | 10 ++- api/funkwhale_api/cli/plugins.py | 73 +++++++++++++++++-- .../migrations/0008_auto_20200617_1902.py | 29 ++++++++ api/funkwhale_api/common/models.py | 15 ++++ api/funkwhale_api/common/serializers.py | 18 +++++ .../README.md | 0 .../__init__.py | 0 .../entrypoint.py | 5 +- .../setup.cfg | 4 +- .../setup.py | 0 api/tests/common/test_plugins.py | 34 +++++++++ api/tests/common/test_serializers.py | 19 +++++ api/tests/conftest.py | 11 +++ docs/developers/index.rst | 1 + docs/developers/plugins.rst | 8 ++ 16 files changed, 242 insertions(+), 18 deletions(-) create mode 100644 api/funkwhale_api/common/migrations/0008_auto_20200617_1902.py rename api/funkwhale_api/plugins/{prometheus_exporter => funkwhale_plugin_prometheus}/README.md (100%) rename api/funkwhale_api/plugins/{prometheus_exporter => funkwhale_plugin_prometheus}/__init__.py (100%) rename api/funkwhale_api/plugins/{prometheus_exporter => funkwhale_plugin_prometheus}/entrypoint.py (85%) rename api/funkwhale_api/plugins/{prometheus_exporter => funkwhale_plugin_prometheus}/setup.cfg (86%) rename api/funkwhale_api/plugins/{prometheus_exporter => funkwhale_plugin_prometheus}/setup.py (100%) create mode 100644 docs/developers/plugins.rst diff --git a/api/config/plugins.py b/api/config/plugins.py index eb5f8a520..c3ff57e64 100644 --- a/api/config/plugins.py +++ b/api/config/plugins.py @@ -14,9 +14,25 @@ class ConfigError(ValueError): class Plugin(AppConfig): conf = {} path = "noop" + conf_serializer = None def get_conf(self): - return {"enabled": self.plugin_settings.enabled} + return self.instance.conf + + def set_conf(self, data): + if self.conf_serializer: + s = self.conf_serializer(data=data) + s.is_valid(raise_exception=True) + data = s.validated_data + instance = self.instance() + instance.conf = data + instance.save(update_fields=["conf"]) + + def instance(self): + """Return the DB object that match the plugin""" + from funkwhale_api.common import models + + return models.PodPlugin.objects.get_or_create(code=self.name)[0] def plugin_settings(self): """ @@ -94,7 +110,13 @@ plugins_manager.add_hookspecs(HookSpec()) def register(plugin_class): - return plugins_manager.register(plugin_class("noop", "noop")) + return plugins_manager.register(plugin_class(plugin_class.name, "noop")) + + +def save(plugin_class): + from funkwhale_api.common.models import PodPlugin + + return PodPlugin.objects.get_or_create(code=plugin_class.name)[0] def trigger_hook(name, *args, **kwargs): @@ -104,6 +126,13 @@ def trigger_hook(name, *args, **kwargs): @register class DefaultPlugin(Plugin): + name = "default" + verbose_name = "Default plugin" + @plugin_hook def database_engine(self): return "django.db.backends.postgresql" + + @plugin_hook + def urls(self): + return [] diff --git a/api/config/settings/common.py b/api/config/settings/common.py index ff66ae8ef..db4b579c8 100644 --- a/api/config/settings/common.py +++ b/api/config/settings/common.py @@ -24,7 +24,11 @@ class Plugins(persisting_theory.Registry): look_into = "entrypoint" -PLUGINS = [p for p in env.list("FUNKWHALE_PLUGINS", default=[]) if p] +PLUGINS = [ + "funkwhale_plugin_{}".format(p) + for p in env.list("FUNKWHALE_PLUGINS", default=[]) + if p +] """ List of Funkwhale plugins to load. """ @@ -33,8 +37,6 @@ from config import plugins # noqa plugins_registry = Plugins() plugins_registry.autodiscover(PLUGINS) -# plugins.plugins_manager.register(Plugin("noop", "noop")) - LOGLEVEL = env("LOGLEVEL", default="info").upper() """ Default logging level for the Funkwhale processes""" # pylint: disable=W0105 @@ -272,7 +274,7 @@ List of Django apps to load in addition to Funkwhale plugins and apps. PLUGINS_APPS = tuple() for p in plugins.trigger_hook("register_apps"): - PLUGINS_APPS += (p,) + PLUGINS_APPS += tuple(p) INSTALLED_APPS = ( DJANGO_APPS diff --git a/api/funkwhale_api/cli/plugins.py b/api/funkwhale_api/cli/plugins.py index 16c1cf313..f1c9b593a 100644 --- a/api/funkwhale_api/cli/plugins.py +++ b/api/funkwhale_api/cli/plugins.py @@ -1,5 +1,8 @@ import os +import shutil import subprocess +import sys +import tempfile import click @@ -8,12 +11,27 @@ from django.conf import settings from . import base +PIP = os.path.join(sys.prefix, "bin", "pip") + + @base.cli.group() def plugins(): """Install, configure and remove plugins""" pass +def get_all_plugins(): + plugins = [ + f.path + for f in os.scandir(settings.FUNKWHALE_PLUGINS_PATH) + if "/funkwhale_plugin_" in f.path + ] + plugins = [ + p.split("-")[0].split("/")[-1].replace("funkwhale_plugin_", "") for p in plugins + ] + return plugins + + @plugins.command("install") @click.argument("name_or_url", nargs=-1) @click.option("--builtins", is_flag=True) @@ -21,17 +39,56 @@ def plugins(): def install(name_or_url, builtins, pip_args): """ Installed the specified plug using their name. - - If --builtins is provided, it will also install - plugins present at FUNKWHALE_PLUGINS_PATH """ pip_args = pip_args or "" - target_path = settings.FUNKWHALE_PLUGINS_PATH - builtins_path = os.path.join(settings.APPS_DIR, "plugins") - builtins_plugins = [f.path for f in os.scandir(builtins_path) if f.is_dir()] - command = "pip install {} --target={} {}".format( - pip_args, target_path, " ".join(builtins_plugins) + all_plugins = [] + for p in name_or_url: + builtin_path = os.path.join( + settings.APPS_DIR, "plugins", "funkwhale_plugin_{}".format(p) + ) + if os.path.exists(builtin_path): + all_plugins.append(builtin_path) + else: + all_plugins.append(p) + install_plugins(pip_args, all_plugins) + click.echo( + "Installation completed, ensure FUNKWHALE_PLUGINS={} is present in your .env file".format( + ",".join(get_all_plugins()) + ) ) + + +def install_plugins(pip_args, all_plugins): + with tempfile.TemporaryDirectory() as tmpdirname: + command = "{} install {} --target {} --build={} {}".format( + PIP, + pip_args, + settings.FUNKWHALE_PLUGINS_PATH, + tmpdirname, + " ".join(all_plugins), + ) + subprocess.run( + command, shell=True, check=True, + ) + + +@plugins.command("uninstall") +@click.argument("name", nargs=-1) +def uninstall(name): + """ + Remove plugins + """ + to_remove = ["funkwhale_plugin_{}".format(n) for n in name] + command = "{} uninstall -y {}".format(PIP, " ".join(to_remove)) subprocess.run( command, shell=True, check=True, ) + for f in os.scandir(settings.FUNKWHALE_PLUGINS_PATH): + for n in name: + if "/funkwhale_plugin_{}".format(n) in f.path: + shutil.rmtree(f.path) + click.echo( + "Removal completed, set FUNKWHALE_PLUGINS={} in your .env file".format( + ",".join(get_all_plugins()) + ) + ) diff --git a/api/funkwhale_api/common/migrations/0008_auto_20200617_1902.py b/api/funkwhale_api/common/migrations/0008_auto_20200617_1902.py new file mode 100644 index 000000000..315daf19b --- /dev/null +++ b/api/funkwhale_api/common/migrations/0008_auto_20200617_1902.py @@ -0,0 +1,29 @@ +# Generated by Django 3.0.6 on 2020-06-17 19:02 + +import django.contrib.postgres.fields.jsonb +from django.db import migrations, models +import django.utils.timezone + + +class Migration(migrations.Migration): + + dependencies = [ + ('common', '0007_auto_20200116_1610'), + ] + + operations = [ + migrations.CreateModel( + name='PodPlugin', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('conf', django.contrib.postgres.fields.jsonb.JSONField(default=None, null=True, blank=True)), + ('code', models.CharField(max_length=100, unique=True)), + ('creation_date', models.DateTimeField(default=django.utils.timezone.now)), + ], + ), + migrations.AlterField( + model_name='attachment', + name='url', + field=models.URLField(blank=True, max_length=500, null=True), + ), + ] diff --git a/api/funkwhale_api/common/models.py b/api/funkwhale_api/common/models.py index 1a31b2dcd..e78b4e2c6 100644 --- a/api/funkwhale_api/common/models.py +++ b/api/funkwhale_api/common/models.py @@ -18,6 +18,7 @@ from django.urls import reverse from versatileimagefield.fields import VersatileImageField from versatileimagefield.image_warmer import VersatileImageFieldWarmer +from config import plugins from funkwhale_api.federation import utils as federation_utils from . import utils @@ -363,3 +364,17 @@ def remove_attached_content(sender, instance, **kwargs): getattr(instance, field).delete() except Content.DoesNotExist: pass + + +class PodPlugin(models.Model): + conf = JSONField(default=None, null=True, blank=True) + code = models.CharField(max_length=100, unique=True) + creation_date = models.DateTimeField(default=timezone.now) + + @property + def plugin(self): + """Links to the Plugin instance in entryposint.py""" + candidates = plugins.plugins_manager.get_plugins() + for p in candidates: + if p.name == self.code: + return p diff --git a/api/funkwhale_api/common/serializers.py b/api/funkwhale_api/common/serializers.py index f5565f133..7063d85e4 100644 --- a/api/funkwhale_api/common/serializers.py +++ b/api/funkwhale_api/common/serializers.py @@ -339,3 +339,21 @@ class NullToEmptDict(object): if not v: return v return super().to_representation(v) + + +class PodPluginSerializer(serializers.Serializer): + code = serializers.CharField(read_only=True) + enabled = serializers.BooleanField() + conf = serializers.JSONField() + label = serializers.SerializerMethodField() + + class Meta: + fields = [ + "code", + "label", + "enabled", + "conf", + ] + + def get_label(self, o): + return o.plugin.verbose_name diff --git a/api/funkwhale_api/plugins/prometheus_exporter/README.md b/api/funkwhale_api/plugins/funkwhale_plugin_prometheus/README.md similarity index 100% rename from api/funkwhale_api/plugins/prometheus_exporter/README.md rename to api/funkwhale_api/plugins/funkwhale_plugin_prometheus/README.md diff --git a/api/funkwhale_api/plugins/prometheus_exporter/__init__.py b/api/funkwhale_api/plugins/funkwhale_plugin_prometheus/__init__.py similarity index 100% rename from api/funkwhale_api/plugins/prometheus_exporter/__init__.py rename to api/funkwhale_api/plugins/funkwhale_plugin_prometheus/__init__.py diff --git a/api/funkwhale_api/plugins/prometheus_exporter/entrypoint.py b/api/funkwhale_api/plugins/funkwhale_plugin_prometheus/entrypoint.py similarity index 85% rename from api/funkwhale_api/plugins/prometheus_exporter/entrypoint.py rename to api/funkwhale_api/plugins/funkwhale_plugin_prometheus/entrypoint.py index ccd6e0020..1a4760ec3 100644 --- a/api/funkwhale_api/plugins/prometheus_exporter/entrypoint.py +++ b/api/funkwhale_api/plugins/funkwhale_plugin_prometheus/entrypoint.py @@ -5,7 +5,8 @@ from config import plugins @plugins.register class Plugin(plugins.Plugin): - name = "prometheus_exporter" + name = "funkwhale_plugin_prometheus" + verbose_name = "Prometheus metrics exporter" @plugins.plugin_hook def database_engine(self): @@ -13,7 +14,7 @@ class Plugin(plugins.Plugin): @plugins.plugin_hook def register_apps(self): - return "django_prometheus" + return ["django_prometheus"] @plugins.plugin_hook def middlewares_before(self): diff --git a/api/funkwhale_api/plugins/prometheus_exporter/setup.cfg b/api/funkwhale_api/plugins/funkwhale_plugin_prometheus/setup.cfg similarity index 86% rename from api/funkwhale_api/plugins/prometheus_exporter/setup.cfg rename to api/funkwhale_api/plugins/funkwhale_plugin_prometheus/setup.cfg index 5c8e8a498..acf736476 100644 --- a/api/funkwhale_api/plugins/prometheus_exporter/setup.cfg +++ b/api/funkwhale_api/plugins/funkwhale_plugin_prometheus/setup.cfg @@ -1,6 +1,6 @@ [metadata] -name = funkwhale-prometheus -description = "A prometheus metric exporter for your Funkwhale pod" +name = funkwhale-plugin-prometheus +description = "A prometheus metrics exporter for your Funkwhale pod" version = 0.1.dev0 author = Agate Blue author_email = me@agate.blue diff --git a/api/funkwhale_api/plugins/prometheus_exporter/setup.py b/api/funkwhale_api/plugins/funkwhale_plugin_prometheus/setup.py similarity index 100% rename from api/funkwhale_api/plugins/prometheus_exporter/setup.py rename to api/funkwhale_api/plugins/funkwhale_plugin_prometheus/setup.py diff --git a/api/tests/common/test_plugins.py b/api/tests/common/test_plugins.py index e69de29bb..1fd740aab 100644 --- a/api/tests/common/test_plugins.py +++ b/api/tests/common/test_plugins.py @@ -0,0 +1,34 @@ +import pytest + +from rest_framework import serializers + +from config import plugins +from funkwhale_api.common import models + + +def test_plugin_validate_set_conf(): + class S(serializers.Serializer): + test = serializers.CharField() + foo = serializers.BooleanField() + + class P(plugins.Plugin): + conf_serializer = S + + p = P("noop", "noop") + with pytest.raises(serializers.ValidationError): + assert p.set_conf({"test": "hello", "foo": "bar"}) + + +def test_plugin_validate_set_conf_persists(): + class S(serializers.Serializer): + test = serializers.CharField() + foo = serializers.BooleanField() + + class P(plugins.Plugin): + name = "test_plugin" + conf_serializer = S + + p = P("noop", "noop") + p.set_conf({"test": "hello", "foo": False}) + assert p.instance() == models.PodPlugin.objects.latest("id") + assert p.instance().conf == {"test": "hello", "foo": False} diff --git a/api/tests/common/test_serializers.py b/api/tests/common/test_serializers.py index 8fdb21edb..4d677783c 100644 --- a/api/tests/common/test_serializers.py +++ b/api/tests/common/test_serializers.py @@ -6,6 +6,7 @@ from django.urls import reverse import django_filters +from config import plugins from funkwhale_api.common import serializers from funkwhale_api.common import utils from funkwhale_api.users import models @@ -267,3 +268,21 @@ def test_content_serializer(factories): serializer = serializers.ContentSerializer(content) assert serializer.data == expected + + +def test_plugin_serializer(): + class TestPlugin(plugins.Plugin): + name = "test_plugin" + verbose_name = "A test plugin" + + plugins.register(TestPlugin) + instance = plugins.save(TestPlugin) + assert isinstance(instance.plugin, TestPlugin) + expected = { + "code": "test_plugin", + "label": "A test plugin", + "enabled": True, + "conf": None, + } + + assert serializers.PodPluginSerializer(instance).data == expected diff --git a/api/tests/conftest.py b/api/tests/conftest.py index 8b1dddd48..2748076fc 100644 --- a/api/tests/conftest.py +++ b/api/tests/conftest.py @@ -24,6 +24,7 @@ from aioresponses import aioresponses from dynamic_preferences.registries import global_preferences_registry from rest_framework.test import APIClient, APIRequestFactory +from config import plugins from funkwhale_api.activity import record from funkwhale_api.federation import actors from funkwhale_api.moderation import mrf @@ -429,3 +430,13 @@ def clear_license_cache(db): @pytest.fixture def faker(): return factory.Faker._get_faker() + + +@pytest.fixture +def plugins_manager(): + return plugins.PluginManager("tests") + + +@pytest.fixture +def hook(plugins_manager): + return plugins.HookimplMarker("tests") diff --git a/docs/developers/index.rst b/docs/developers/index.rst index 966cac3af..f214c7819 100644 --- a/docs/developers/index.rst +++ b/docs/developers/index.rst @@ -13,5 +13,6 @@ Reference architecture ../api ./authentication + ./plugins ../federation/index subsonic diff --git a/docs/developers/plugins.rst b/docs/developers/plugins.rst new file mode 100644 index 000000000..97f57713d --- /dev/null +++ b/docs/developers/plugins.rst @@ -0,0 +1,8 @@ +Funkwhale Plugins +================= + +With version 1.0, Funkwhale makes it possible for third party to write plugins +and distribute them. + +Funkwhale plugins are regular django apps, that can register models, API +endpoints, and react to specific events (e.g a son was listened, a federation message was delivered, etc.)