Merge branch 'nodeinfo-autofetch' into 'develop'

Nodeinfo autofetch

See merge request funkwhale/funkwhale!714

api/config/settings/common.py

@@ -486,8 +486,15 @@ CELERY_BEAT_SCHEDULE = {
         "schedule": crontab(minute="0", hour="0"),
         "options": {"expires": 60 * 60 * 24},
     },
+    "federation.refresh_nodeinfo_known_nodes": {
+        "task": "federation.refresh_nodeinfo_known_nodes",
+        "schedule": crontab(minute="0", hour="*"),
+        "options": {"expires": 60 * 60},
+    },
 }
 
+NODEINFO_REFRESH_DELAY = env.int("NODEINFO_REFRESH_DELAY", default=3600 * 24)
+
 JWT_AUTH = {
     "JWT_ALLOW_REFRESH": True,
     "JWT_EXPIRATION_DELTA": datetime.timedelta(days=7),

api/funkwhale_api/federation/models.py

@@ -151,6 +151,10 @@ class Domain(models.Model):
         )
         return data
 
+    @property
+    def is_local(self):
+        return self.name == settings.FEDERATION_HOSTNAME
+
 
 class Actor(models.Model):
     ap_type = "Actor"

api/funkwhale_api/federation/serializers.py

@@ -11,7 +11,7 @@ from funkwhale_api.music import licenses
 from funkwhale_api.music import models as music_models
 from funkwhale_api.music import tasks as music_tasks
 
-from . import activity, actors, contexts, jsonld, models, utils
+from . import activity, actors, contexts, jsonld, models, tasks, utils
 
 AP_CONTEXT = jsonld.get_default_context()
 
@@ -152,7 +152,12 @@ class ActorSerializer(jsonld.JsonLdSerializer):
         if maf is not None:
             kwargs["manually_approves_followers"] = maf
         domain = urllib.parse.urlparse(kwargs["fid"]).netloc
-        kwargs["domain"] = models.Domain.objects.get_or_create(pk=domain)[0]
+        domain, domain_created = models.Domain.objects.get_or_create(pk=domain)
+        if domain_created and not domain.is_local:
+            # first time we see the domain, we trigger nodeinfo fetching
+            tasks.update_domain_nodeinfo(domain_name=domain.name)
+
+        kwargs["domain"] = domain
         for endpoint, url in self.validated_data.get("endpoints", {}).items():
             if endpoint == "sharedInbox":
                 kwargs["shared_inbox_url"] = url

api/funkwhale_api/federation/tasks.py

@@ -212,6 +212,22 @@ def update_domain_nodeinfo(domain):
     domain.save(update_fields=["nodeinfo", "nodeinfo_fetch_date", "service_actor"])
 
 
+@celery.app.task(name="federation.refresh_nodeinfo_known_nodes")
+def refresh_nodeinfo_known_nodes():
+    """
+    Trigger a node info refresh on all nodes that weren't refreshed since
+    settings.NODEINFO_REFRESH_DELAY
+    """
+    limit = timezone.now() - datetime.timedelta(seconds=settings.NODEINFO_REFRESH_DELAY)
+    candidates = models.Domain.objects.external().exclude(
+        nodeinfo_fetch_date__gte=limit
+    )
+    names = candidates.values_list("name", flat=True)
+    logger.info("Launching periodic nodeinfo refresh on %s domains", len(names))
+    for domain_name in names:
+        update_domain_nodeinfo.delay(domain_name=domain_name)
+
+
 def delete_qs(qs):
     label = qs.model._meta.label
     result = qs.delete()

api/funkwhale_api/manage/views.py

@@ -121,6 +121,10 @@ class ManageDomainViewSet(
         "instance_policy",
     ]
 
+    def perform_create(self, serializer):
+        domain = serializer.save()
+        federation_tasks.update_domain_nodeinfo(domain_name=domain.name)
+
     @rest_decorators.action(methods=["get"], detail=True)
     def nodeinfo(self, request, *args, **kwargs):
         domain = self.get_object()

api/tests/federation/test_actors.py

@@ -14,7 +14,10 @@ def test_actor_fetching(r_mock):
     assert r == payload
 
 
-def test_get_actor(factories, r_mock):
+def test_get_actor(factories, r_mock, mocker):
+    update_domain_nodeinfo = mocker.patch(
+        "funkwhale_api.federation.tasks.update_domain_nodeinfo"
+    )
     actor = factories["federation.Actor"].build()
     payload = serializers.ActorSerializer(actor).data
     r_mock.get(actor.fid, json=payload)
@@ -22,6 +25,7 @@ def test_get_actor(factories, r_mock):
 
     assert new_actor.pk is not None
     assert serializers.ActorSerializer(new_actor).data == payload
+    update_domain_nodeinfo.assert_called_once_with(domain_name=new_actor.domain_id)
 
 
 def test_get_actor_use_existing(factories, preferences, mocker):

api/tests/federation/test_tasks.py

@@ -216,6 +216,31 @@ def test_update_domain_nodeinfo_error(factories, r_mock, now):
     }
 
 
+def test_refresh_nodeinfo_known_nodes(settings, factories, mocker, now):
+    settings.NODEINFO_REFRESH_DELAY = 666
+
+    refreshed = [
+        factories["federation.Domain"](nodeinfo_fetch_date=None),
+        factories["federation.Domain"](
+            nodeinfo_fetch_date=now
+            - datetime.timedelta(seconds=settings.NODEINFO_REFRESH_DELAY + 1)
+        ),
+    ]
+    factories["federation.Domain"](
+        nodeinfo_fetch_date=now
+        - datetime.timedelta(seconds=settings.NODEINFO_REFRESH_DELAY - 1)
+    )
+
+    update_domain_nodeinfo = mocker.patch.object(tasks.update_domain_nodeinfo, "delay")
+
+    tasks.refresh_nodeinfo_known_nodes()
+
+    assert update_domain_nodeinfo.call_count == len(refreshed)
+
+    for d in refreshed:
+        update_domain_nodeinfo.assert_any_call(domain_name=d.name)
+
+
 def test_handle_purge_actors(factories, mocker):
     to_purge = factories["federation.Actor"]()
     keeped = [

api/tests/manage/test_views.py

@@ -77,12 +77,16 @@ def test_domain_detail(factories, superuser_api_client):
     assert response.data["name"] == d.pk
 
 
-def test_domain_create(superuser_api_client):
+def test_domain_create(superuser_api_client, mocker):
+    update_domain_nodeinfo = mocker.patch(
+        "funkwhale_api.federation.tasks.update_domain_nodeinfo"
+    )
     url = reverse("api:v1:manage:federation:domains-list")
     response = superuser_api_client.post(url, {"name": "test.federation"})
 
     assert response.status_code == 201
     assert federation_models.Domain.objects.filter(pk="test.federation").exists()
+    update_domain_nodeinfo.assert_called_once_with(domain_name="test.federation")
 
 
 def test_domain_nodeinfo(factories, superuser_api_client, mocker):