kopia lustrzana https://dev.funkwhale.audio/funkwhale/funkwhale
Merge branch 'nodeinfo-autofetch' into 'develop'
Nodeinfo autofetch See merge request funkwhale/funkwhale!714merge-requests/757/head
commit
4d55c844f4
|
@ -486,8 +486,15 @@ CELERY_BEAT_SCHEDULE = {
|
||||||
"schedule": crontab(minute="0", hour="0"),
|
"schedule": crontab(minute="0", hour="0"),
|
||||||
"options": {"expires": 60 * 60 * 24},
|
"options": {"expires": 60 * 60 * 24},
|
||||||
},
|
},
|
||||||
|
"federation.refresh_nodeinfo_known_nodes": {
|
||||||
|
"task": "federation.refresh_nodeinfo_known_nodes",
|
||||||
|
"schedule": crontab(minute="0", hour="*"),
|
||||||
|
"options": {"expires": 60 * 60},
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
NODEINFO_REFRESH_DELAY = env.int("NODEINFO_REFRESH_DELAY", default=3600 * 24)
|
||||||
|
|
||||||
JWT_AUTH = {
|
JWT_AUTH = {
|
||||||
"JWT_ALLOW_REFRESH": True,
|
"JWT_ALLOW_REFRESH": True,
|
||||||
"JWT_EXPIRATION_DELTA": datetime.timedelta(days=7),
|
"JWT_EXPIRATION_DELTA": datetime.timedelta(days=7),
|
||||||
|
|
|
@ -151,6 +151,10 @@ class Domain(models.Model):
|
||||||
)
|
)
|
||||||
return data
|
return data
|
||||||
|
|
||||||
|
@property
|
||||||
|
def is_local(self):
|
||||||
|
return self.name == settings.FEDERATION_HOSTNAME
|
||||||
|
|
||||||
|
|
||||||
class Actor(models.Model):
|
class Actor(models.Model):
|
||||||
ap_type = "Actor"
|
ap_type = "Actor"
|
||||||
|
|
|
@ -11,7 +11,7 @@ from funkwhale_api.music import licenses
|
||||||
from funkwhale_api.music import models as music_models
|
from funkwhale_api.music import models as music_models
|
||||||
from funkwhale_api.music import tasks as music_tasks
|
from funkwhale_api.music import tasks as music_tasks
|
||||||
|
|
||||||
from . import activity, actors, contexts, jsonld, models, utils
|
from . import activity, actors, contexts, jsonld, models, tasks, utils
|
||||||
|
|
||||||
AP_CONTEXT = jsonld.get_default_context()
|
AP_CONTEXT = jsonld.get_default_context()
|
||||||
|
|
||||||
|
@ -152,7 +152,12 @@ class ActorSerializer(jsonld.JsonLdSerializer):
|
||||||
if maf is not None:
|
if maf is not None:
|
||||||
kwargs["manually_approves_followers"] = maf
|
kwargs["manually_approves_followers"] = maf
|
||||||
domain = urllib.parse.urlparse(kwargs["fid"]).netloc
|
domain = urllib.parse.urlparse(kwargs["fid"]).netloc
|
||||||
kwargs["domain"] = models.Domain.objects.get_or_create(pk=domain)[0]
|
domain, domain_created = models.Domain.objects.get_or_create(pk=domain)
|
||||||
|
if domain_created and not domain.is_local:
|
||||||
|
# first time we see the domain, we trigger nodeinfo fetching
|
||||||
|
tasks.update_domain_nodeinfo(domain_name=domain.name)
|
||||||
|
|
||||||
|
kwargs["domain"] = domain
|
||||||
for endpoint, url in self.validated_data.get("endpoints", {}).items():
|
for endpoint, url in self.validated_data.get("endpoints", {}).items():
|
||||||
if endpoint == "sharedInbox":
|
if endpoint == "sharedInbox":
|
||||||
kwargs["shared_inbox_url"] = url
|
kwargs["shared_inbox_url"] = url
|
||||||
|
|
|
@ -212,6 +212,22 @@ def update_domain_nodeinfo(domain):
|
||||||
domain.save(update_fields=["nodeinfo", "nodeinfo_fetch_date", "service_actor"])
|
domain.save(update_fields=["nodeinfo", "nodeinfo_fetch_date", "service_actor"])
|
||||||
|
|
||||||
|
|
||||||
|
@celery.app.task(name="federation.refresh_nodeinfo_known_nodes")
|
||||||
|
def refresh_nodeinfo_known_nodes():
|
||||||
|
"""
|
||||||
|
Trigger a node info refresh on all nodes that weren't refreshed since
|
||||||
|
settings.NODEINFO_REFRESH_DELAY
|
||||||
|
"""
|
||||||
|
limit = timezone.now() - datetime.timedelta(seconds=settings.NODEINFO_REFRESH_DELAY)
|
||||||
|
candidates = models.Domain.objects.external().exclude(
|
||||||
|
nodeinfo_fetch_date__gte=limit
|
||||||
|
)
|
||||||
|
names = candidates.values_list("name", flat=True)
|
||||||
|
logger.info("Launching periodic nodeinfo refresh on %s domains", len(names))
|
||||||
|
for domain_name in names:
|
||||||
|
update_domain_nodeinfo.delay(domain_name=domain_name)
|
||||||
|
|
||||||
|
|
||||||
def delete_qs(qs):
|
def delete_qs(qs):
|
||||||
label = qs.model._meta.label
|
label = qs.model._meta.label
|
||||||
result = qs.delete()
|
result = qs.delete()
|
||||||
|
|
|
@ -121,6 +121,10 @@ class ManageDomainViewSet(
|
||||||
"instance_policy",
|
"instance_policy",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
def perform_create(self, serializer):
|
||||||
|
domain = serializer.save()
|
||||||
|
federation_tasks.update_domain_nodeinfo(domain_name=domain.name)
|
||||||
|
|
||||||
@rest_decorators.action(methods=["get"], detail=True)
|
@rest_decorators.action(methods=["get"], detail=True)
|
||||||
def nodeinfo(self, request, *args, **kwargs):
|
def nodeinfo(self, request, *args, **kwargs):
|
||||||
domain = self.get_object()
|
domain = self.get_object()
|
||||||
|
|
|
@ -14,7 +14,10 @@ def test_actor_fetching(r_mock):
|
||||||
assert r == payload
|
assert r == payload
|
||||||
|
|
||||||
|
|
||||||
def test_get_actor(factories, r_mock):
|
def test_get_actor(factories, r_mock, mocker):
|
||||||
|
update_domain_nodeinfo = mocker.patch(
|
||||||
|
"funkwhale_api.federation.tasks.update_domain_nodeinfo"
|
||||||
|
)
|
||||||
actor = factories["federation.Actor"].build()
|
actor = factories["federation.Actor"].build()
|
||||||
payload = serializers.ActorSerializer(actor).data
|
payload = serializers.ActorSerializer(actor).data
|
||||||
r_mock.get(actor.fid, json=payload)
|
r_mock.get(actor.fid, json=payload)
|
||||||
|
@ -22,6 +25,7 @@ def test_get_actor(factories, r_mock):
|
||||||
|
|
||||||
assert new_actor.pk is not None
|
assert new_actor.pk is not None
|
||||||
assert serializers.ActorSerializer(new_actor).data == payload
|
assert serializers.ActorSerializer(new_actor).data == payload
|
||||||
|
update_domain_nodeinfo.assert_called_once_with(domain_name=new_actor.domain_id)
|
||||||
|
|
||||||
|
|
||||||
def test_get_actor_use_existing(factories, preferences, mocker):
|
def test_get_actor_use_existing(factories, preferences, mocker):
|
||||||
|
|
|
@ -216,6 +216,31 @@ def test_update_domain_nodeinfo_error(factories, r_mock, now):
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_refresh_nodeinfo_known_nodes(settings, factories, mocker, now):
|
||||||
|
settings.NODEINFO_REFRESH_DELAY = 666
|
||||||
|
|
||||||
|
refreshed = [
|
||||||
|
factories["federation.Domain"](nodeinfo_fetch_date=None),
|
||||||
|
factories["federation.Domain"](
|
||||||
|
nodeinfo_fetch_date=now
|
||||||
|
- datetime.timedelta(seconds=settings.NODEINFO_REFRESH_DELAY + 1)
|
||||||
|
),
|
||||||
|
]
|
||||||
|
factories["federation.Domain"](
|
||||||
|
nodeinfo_fetch_date=now
|
||||||
|
- datetime.timedelta(seconds=settings.NODEINFO_REFRESH_DELAY - 1)
|
||||||
|
)
|
||||||
|
|
||||||
|
update_domain_nodeinfo = mocker.patch.object(tasks.update_domain_nodeinfo, "delay")
|
||||||
|
|
||||||
|
tasks.refresh_nodeinfo_known_nodes()
|
||||||
|
|
||||||
|
assert update_domain_nodeinfo.call_count == len(refreshed)
|
||||||
|
|
||||||
|
for d in refreshed:
|
||||||
|
update_domain_nodeinfo.assert_any_call(domain_name=d.name)
|
||||||
|
|
||||||
|
|
||||||
def test_handle_purge_actors(factories, mocker):
|
def test_handle_purge_actors(factories, mocker):
|
||||||
to_purge = factories["federation.Actor"]()
|
to_purge = factories["federation.Actor"]()
|
||||||
keeped = [
|
keeped = [
|
||||||
|
|
|
@ -77,12 +77,16 @@ def test_domain_detail(factories, superuser_api_client):
|
||||||
assert response.data["name"] == d.pk
|
assert response.data["name"] == d.pk
|
||||||
|
|
||||||
|
|
||||||
def test_domain_create(superuser_api_client):
|
def test_domain_create(superuser_api_client, mocker):
|
||||||
|
update_domain_nodeinfo = mocker.patch(
|
||||||
|
"funkwhale_api.federation.tasks.update_domain_nodeinfo"
|
||||||
|
)
|
||||||
url = reverse("api:v1:manage:federation:domains-list")
|
url = reverse("api:v1:manage:federation:domains-list")
|
||||||
response = superuser_api_client.post(url, {"name": "test.federation"})
|
response = superuser_api_client.post(url, {"name": "test.federation"})
|
||||||
|
|
||||||
assert response.status_code == 201
|
assert response.status_code == 201
|
||||||
assert federation_models.Domain.objects.filter(pk="test.federation").exists()
|
assert federation_models.Domain.objects.filter(pk="test.federation").exists()
|
||||||
|
update_domain_nodeinfo.assert_called_once_with(domain_name="test.federation")
|
||||||
|
|
||||||
|
|
||||||
def test_domain_nodeinfo(factories, superuser_api_client, mocker):
|
def test_domain_nodeinfo(factories, superuser_api_client, mocker):
|
||||||
|
|
Ładowanie…
Reference in New Issue