From be388870a331edb87f94d5cae739fb184d08a39e Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Thu, 27 Dec 2018 17:42:43 +0100 Subject: [PATCH] Can now fetch domain nodeinfo --- .../migrations/0016_auto_20181227_1605.py | 25 +++++++++ api/funkwhale_api/federation/models.py | 3 ++ api/funkwhale_api/federation/serializers.py | 12 +++++ api/funkwhale_api/federation/tasks.py | 39 ++++++++++++++ api/funkwhale_api/manage/serializers.py | 2 + api/funkwhale_api/manage/views.py | 11 +++- api/tests/federation/test_tasks.py | 52 +++++++++++++++++++ api/tests/manage/test_serializers.py | 2 + api/tests/manage/test_views.py | 26 ++++++++++ 9 files changed, 171 insertions(+), 1 deletion(-) create mode 100644 api/funkwhale_api/federation/migrations/0016_auto_20181227_1605.py diff --git a/api/funkwhale_api/federation/migrations/0016_auto_20181227_1605.py b/api/funkwhale_api/federation/migrations/0016_auto_20181227_1605.py new file mode 100644 index 000000000..8b705e72f --- /dev/null +++ b/api/funkwhale_api/federation/migrations/0016_auto_20181227_1605.py @@ -0,0 +1,25 @@ +# Generated by Django 2.0.9 on 2018-12-27 16:05 + +import django.contrib.postgres.fields.jsonb +from django.db import migrations, models +import funkwhale_api.federation.models + + +class Migration(migrations.Migration): + + dependencies = [("federation", "0015_populate_domains")] + + operations = [ + migrations.AddField( + model_name="domain", + name="nodeinfo", + field=django.contrib.postgres.fields.jsonb.JSONField( + default=funkwhale_api.federation.models.empty_dict, max_length=50000 + ), + ), + migrations.AddField( + model_name="domain", + name="nodeinfo_fetch_date", + field=models.DateTimeField(blank=True, default=None, null=True), + ), + ] diff --git a/api/funkwhale_api/federation/models.py b/api/funkwhale_api/federation/models.py index 48e5982da..ad4c0c7be 100644 --- a/api/funkwhale_api/federation/models.py +++ b/api/funkwhale_api/federation/models.py @@ -87,6 +87,9 @@ class DomainQuerySet(models.QuerySet): class Domain(models.Model): name = models.CharField(primary_key=True, max_length=255) creation_date = models.DateTimeField(default=timezone.now) + nodeinfo_fetch_date = models.DateTimeField(default=None, null=True, blank=True) + nodeinfo = JSONField(default=empty_dict, max_length=50000) + objects = DomainQuerySet.as_manager() def __str__(self): diff --git a/api/funkwhale_api/federation/serializers.py b/api/funkwhale_api/federation/serializers.py index 76ab5ba86..7d00476af 100644 --- a/api/funkwhale_api/federation/serializers.py +++ b/api/funkwhale_api/federation/serializers.py @@ -889,3 +889,15 @@ class CollectionSerializer(serializers.Serializer): if self.context.get("include_ap_context", True): d["@context"] = AP_CONTEXT return d + + +class NodeInfoLinkSerializer(serializers.Serializer): + href = serializers.URLField() + rel = serializers.URLField() + + +class NodeInfoSerializer(serializers.Serializer): + links = serializers.ListField( + child=NodeInfoLinkSerializer(), + min_length=1 + ) \ No newline at end of file diff --git a/api/funkwhale_api/federation/tasks.py b/api/funkwhale_api/federation/tasks.py index 33f94cad3..4ed07aa25 100644 --- a/api/funkwhale_api/federation/tasks.py +++ b/api/funkwhale_api/federation/tasks.py @@ -1,6 +1,7 @@ import datetime import logging import os +import requests from django.conf import settings from django.db.models import Q, F @@ -14,6 +15,7 @@ from funkwhale_api.music import models as music_models from funkwhale_api.taskapp import celery from . import models, signing +from . import serializers from . import routes logger = logging.getLogger(__name__) @@ -147,3 +149,40 @@ def deliver_to_remote(delivery): delivery.attempts = F("attempts") + 1 delivery.is_delivered = True delivery.save(update_fields=["last_attempt_date", "attempts", "is_delivered"]) + + +def fetch_nodeinfo(domain_name): + s = session.get_session() + wellknown_url = "https://{}/.well-known/nodeinfo".format(domain_name) + response = s.get( + url=wellknown_url, timeout=5, verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL + ) + response.raise_for_status() + serializer = serializers.NodeInfoSerializer(data=response.json()) + serializer.is_valid(raise_exception=True) + nodeinfo_url = None + for link in serializer.validated_data["links"]: + if link["rel"] == "http://nodeinfo.diaspora.software/ns/schema/2.0": + nodeinfo_url = link["href"] + break + + response = s.get( + url=nodeinfo_url, timeout=5, verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL + ) + response.raise_for_status() + return response.json() + + +@celery.app.task(name="federation.update_domain_nodeinfo") +@celery.require_instance( + models.Domain.objects.external(), "domain", id_kwarg_name="domain_name" +) +def update_domain_nodeinfo(domain): + now = timezone.now() + try: + nodeinfo = {"status": "ok", "payload": fetch_nodeinfo(domain.name)} + except (requests.RequestException, serializers.serializers.ValidationError) as e: + nodeinfo = {"status": "error", "error": str(e)} + domain.nodeinfo_fetch_date = now + domain.nodeinfo = nodeinfo + domain.save(update_fields=["nodeinfo", "nodeinfo_fetch_date"]) diff --git a/api/funkwhale_api/manage/serializers.py b/api/funkwhale_api/manage/serializers.py index 8686a99b9..a401381e6 100644 --- a/api/funkwhale_api/manage/serializers.py +++ b/api/funkwhale_api/manage/serializers.py @@ -184,6 +184,8 @@ class ManageDomainSerializer(serializers.ModelSerializer): "actors_count", "last_activity_date", "outbox_activities_count", + "nodeinfo", + "nodeinfo_fetch_date", ] def get_actors_count(self, o): diff --git a/api/funkwhale_api/manage/views.py b/api/funkwhale_api/manage/views.py index 30f7179e8..99b3b41c4 100644 --- a/api/funkwhale_api/manage/views.py +++ b/api/funkwhale_api/manage/views.py @@ -1,8 +1,9 @@ from rest_framework import mixins, response, viewsets -from rest_framework.decorators import list_route +from rest_framework.decorators import detail_route, list_route from funkwhale_api.common import preferences from funkwhale_api.federation import models as federation_models +from funkwhale_api.federation import tasks as federation_tasks from funkwhale_api.music import models as music_models from funkwhale_api.users import models as users_models from funkwhale_api.users.permissions import HasUserPermission @@ -98,6 +99,7 @@ class ManageInvitationViewSet( class ManageDomainViewSet( mixins.ListModelMixin, mixins.RetrieveModelMixin, viewsets.GenericViewSet ): + lookup_value_regex = "[a-zA-Z0-9\-\.]+" queryset = ( federation_models.Domain.objects.external() .with_last_activity_date() @@ -116,3 +118,10 @@ class ManageDomainViewSet( "actors_count", "outbox_activities_count", ] + + @detail_route(methods=["get"]) + def nodeinfo(self, request, *args, **kwargs): + domain = self.get_object() + federation_tasks.update_domain_nodeinfo(domain_name=domain.name) + domain.refresh_from_db() + return response.Response(domain.nodeinfo, status=200) diff --git a/api/tests/federation/test_tasks.py b/api/tests/federation/test_tasks.py index 1f58055a2..ad7a577ef 100644 --- a/api/tests/federation/test_tasks.py +++ b/api/tests/federation/test_tasks.py @@ -138,3 +138,55 @@ def test_deliver_to_remote_error(factories, r_mock, now): assert delivery.is_delivered is False assert delivery.attempts == 1 assert delivery.last_attempt_date == now + + +def test_fetch_nodeinfo(factories, r_mock, now): + wellknown_url = "https://test.test/.well-known/nodeinfo" + nodeinfo_url = "https://test.test/nodeinfo" + + r_mock.get( + wellknown_url, + json={ + "links": [ + { + "rel": "http://nodeinfo.diaspora.software/ns/schema/2.0", + "href": "https://test.test/nodeinfo", + } + ] + }, + ) + r_mock.get(nodeinfo_url, json={"hello": "world"}) + + assert tasks.fetch_nodeinfo("test.test") == {"hello": "world"} + + +def test_update_domain_nodeinfo(factories, mocker, now): + domain = factories["federation.Domain"]() + mocker.patch.object(tasks, "fetch_nodeinfo", return_value={"hello": "world"}) + + assert domain.nodeinfo == {} + assert domain.nodeinfo_fetch_date is None + + tasks.update_domain_nodeinfo(domain_name=domain.name) + + domain.refresh_from_db() + + assert domain.nodeinfo_fetch_date == now + assert domain.nodeinfo == {"status": "ok", "payload": {"hello": "world"}} + + +def test_update_domain_nodeinfo_error(factories, r_mock, now): + domain = factories["federation.Domain"]() + wellknown_url = "https://{}/.well-known/nodeinfo".format(domain.name) + + r_mock.get(wellknown_url, status_code=500) + + tasks.update_domain_nodeinfo(domain_name=domain.name) + + domain.refresh_from_db() + + assert domain.nodeinfo_fetch_date == now + assert domain.nodeinfo == { + "status": "error", + "error": "500 Server Error: None for url: {}".format(wellknown_url), + } diff --git a/api/tests/manage/test_serializers.py b/api/tests/manage/test_serializers.py index be02e6727..d3b96ec22 100644 --- a/api/tests/manage/test_serializers.py +++ b/api/tests/manage/test_serializers.py @@ -47,6 +47,8 @@ def test_manage_domain_serializer(factories, now): "last_activity_date": now, "actors_count": 42, "outbox_activities_count": 23, + "nodeinfo": {}, + "nodeinfo_fetch_date": None, } s = serializers.ManageDomainSerializer(domain) diff --git a/api/tests/manage/test_views.py b/api/tests/manage/test_views.py index 3d153073a..31e1075ed 100644 --- a/api/tests/manage/test_views.py +++ b/api/tests/manage/test_views.py @@ -1,6 +1,7 @@ import pytest from django.urls import reverse +from funkwhale_api.federation import tasks as federation_tasks from funkwhale_api.manage import serializers, views @@ -77,3 +78,28 @@ def test_domain_list(factories, superuser_api_client, settings): assert response.data["count"] == 1 assert response.data["results"][0]["name"] == d.pk + + +def test_domain_detail(factories, superuser_api_client): + d = factories["federation.Domain"]() + url = reverse("api:v1:manage:federation:domains-detail", kwargs={"pk": d.name}) + response = superuser_api_client.get(url) + + assert response.status_code == 200 + assert response.data["name"] == d.pk + + +def test_domain_nodeinfo(factories, superuser_api_client, mocker): + domain = factories["federation.Domain"]() + url = reverse( + "api:v1:manage:federation:domains-nodeinfo", kwargs={"pk": domain.name} + ) + mocker.patch.object( + federation_tasks, "fetch_nodeinfo", return_value={"hello": "world"} + ) + update_domain_nodeinfo = mocker.spy(federation_tasks, "update_domain_nodeinfo") + response = superuser_api_client.get(url) + assert response.status_code == 200 + assert response.data == {"status": "ok", "payload": {"hello": "world"}} + + update_domain_nodeinfo.assert_called_once_with(domain_name=domain.name)