From 12c9db396096c07632b6e3045b856e786f3d9f67 Mon Sep 17 00:00:00 2001 From: Eliot Berriot Date: Tue, 5 Mar 2019 15:15:37 +0100 Subject: [PATCH] Ensures federation urls can answer to application/ld+json, application/activity+json and application/json requests --- api/funkwhale_api/common/utils.py | 8 + api/funkwhale_api/federation/activity.py | 18 +- api/funkwhale_api/federation/contexts.py | 333 +++++++ api/funkwhale_api/federation/jsonld.py | 276 ++++++ api/funkwhale_api/federation/renderers.py | 13 +- api/funkwhale_api/federation/serializers.py | 211 +++-- api/funkwhale_api/federation/utils.py | 2 +- api/funkwhale_api/federation/views.py | 16 +- api/requirements/base.txt | 3 + api/requirements/local.txt | 3 + api/tests/conftest.py | 10 + api/tests/federation/test_activity.py | 20 +- api/tests/federation/test_authentication.py | 5 +- api/tests/federation/test_contexts.py | 32 + api/tests/federation/test_jsonld.py | 361 ++++++++ api/tests/federation/test_routes.py | 3 +- api/tests/federation/test_serializers.py | 941 +------------------- api/tests/federation/test_views.py | 29 + api/tests/music/test_tasks.py | 11 + 19 files changed, 1294 insertions(+), 1001 deletions(-) create mode 100644 api/funkwhale_api/federation/contexts.py create mode 100644 api/funkwhale_api/federation/jsonld.py create mode 100644 api/tests/federation/test_contexts.py create mode 100644 api/tests/federation/test_jsonld.py diff --git a/api/funkwhale_api/common/utils.py b/api/funkwhale_api/common/utils.py index 15d70d5b4..6a21517f6 100644 --- a/api/funkwhale_api/common/utils.py +++ b/api/funkwhale_api/common/utils.py @@ -193,3 +193,11 @@ def replace_prefix(queryset, field, old, new): models.functions.Substr(field, len(old) + 1, output_field=models.CharField()), ) return qs.update(**{field: update}) + + +def concat_dicts(*dicts): + n = {} + for d in dicts: + n.update(d) + + return n diff --git a/api/funkwhale_api/federation/activity.py b/api/funkwhale_api/federation/activity.py index 94b7fd54e..488d92cd6 100644 --- a/api/funkwhale_api/federation/activity.py +++ b/api/funkwhale_api/federation/activity.py @@ -9,11 +9,13 @@ from django.db.models import Q from funkwhale_api.common import channels from funkwhale_api.common import utils as funkwhale_utils +from . import contexts + recursive_getattr = funkwhale_utils.recursive_getattr logger = logging.getLogger(__name__) -PUBLIC_ADDRESS = "https://www.w3.org/ns/activitystreams#Public" +PUBLIC_ADDRESS = contexts.AS.Public ACTIVITY_TYPES = [ "Accept", @@ -84,7 +86,10 @@ OBJECT_TYPES = ( BROADCAST_TO_USER_ACTIVITIES = ["Follow", "Accept"] -def should_reject(id, actor_id=None, payload={}): +def should_reject(fid, actor_id=None, payload={}): + if fid is None and actor_id is None: + return False + from funkwhale_api.moderation import models as moderation_models policies = moderation_models.InstancePolicy.objects.active() @@ -102,9 +107,12 @@ def should_reject(id, actor_id=None, payload={}): else: policy_type = Q(block_all=True) - query = policies.matching_url_query(id) & policy_type - if actor_id: + if fid: + query = policies.matching_url_query(fid) & policy_type + if fid and actor_id: query |= policies.matching_url_query(actor_id) & policy_type + elif actor_id: + query = policies.matching_url_query(actor_id) & policy_type return policies.filter(query).exists() @@ -121,7 +129,7 @@ def receive(activity, on_behalf_of): ) serializer.is_valid(raise_exception=True) if should_reject( - id=serializer.validated_data["id"], + fid=serializer.validated_data.get("id"), actor_id=serializer.validated_data["actor"].fid, payload=activity, ): diff --git a/api/funkwhale_api/federation/contexts.py b/api/funkwhale_api/federation/contexts.py new file mode 100644 index 000000000..0873bcd46 --- /dev/null +++ b/api/funkwhale_api/federation/contexts.py @@ -0,0 +1,333 @@ +CONTEXTS = [ + { + "shortId": "LDP", + "contextUrl": None, + "documentUrl": "http://www.w3.org/ns/ldp", + "document": { + "@context": { + "ldp": "http://www.w3.org/ns/ldp#", + "id": "@id", + "type": "@type", + "Container": "ldp:Container", + "BasicContainer": "ldp:BasicContainer", + "DirectContainer": "ldp:DirectContainer", + "IndirectContainer": "ldp:IndirectContainer", + "hasMemberRelation": {"@id": "ldp:hasMemberRelation", "@type": "@id"}, + "isMemberOfRelation": {"@id": "ldp:isMemberOfRelation", "@type": "@id"}, + "membershipResource": {"@id": "ldp:membershipResource", "@type": "@id"}, + "insertedContentRelation": { + "@id": "ldp:insertedContentRelation", + "@type": "@id", + }, + "contains": {"@id": "ldp:contains", "@type": "@id"}, + "member": {"@id": "ldp:member", "@type": "@id"}, + "constrainedBy": {"@id": "ldp:constrainedBy", "@type": "@id"}, + "Resource": "ldp:Resource", + "RDFSource": "ldp:RDFSource", + "NonRDFSource": "ldp:NonRDFSource", + "MemberSubject": "ldp:MemberSubject", + "PreferContainment": "ldp:PreferContainment", + "PreferMembership": "ldp:PreferMembership", + "PreferMinimalContainer": "ldp:PreferMinimalContainer", + "PageSortCriterion": "ldp:PageSortCriterion", + "pageSortCriteria": { + "@id": "ldp:pageSortCriteria", + "@type": "@id", + "@container": "@list", + }, + "pageSortPredicate": {"@id": "ldp:pageSortPredicate", "@type": "@id"}, + "pageSortOrder": {"@id": "ldp:pageSortOrder", "@type": "@id"}, + "pageSortCollation": {"@id": "ldp:pageSortCollation", "@type": "@id"}, + "Ascending": "ldp:Ascending", + "Descending": "ldp:Descending", + "Page": "ldp:Page", + "pageSequence": {"@id": "ldp:pageSequence", "@type": "@id"}, + "inbox": {"@id": "ldp:inbox", "@type": "@id"}, + } + }, + }, + { + "shortId": "AS", + "contextUrl": None, + "documentUrl": "https://www.w3.org/ns/activitystreams", + "document": { + "@context": { + "@vocab": "_:", + "xsd": "http://www.w3.org/2001/XMLSchema#", + "as": "https://www.w3.org/ns/activitystreams#", + "ldp": "http://www.w3.org/ns/ldp#", + "id": "@id", + "type": "@type", + "Accept": "as:Accept", + "Activity": "as:Activity", + "IntransitiveActivity": "as:IntransitiveActivity", + "Add": "as:Add", + "Announce": "as:Announce", + "Application": "as:Application", + "Arrive": "as:Arrive", + "Article": "as:Article", + "Audio": "as:Audio", + "Block": "as:Block", + "Collection": "as:Collection", + "CollectionPage": "as:CollectionPage", + "Relationship": "as:Relationship", + "Create": "as:Create", + "Delete": "as:Delete", + "Dislike": "as:Dislike", + "Document": "as:Document", + "Event": "as:Event", + "Follow": "as:Follow", + "Flag": "as:Flag", + "Group": "as:Group", + "Ignore": "as:Ignore", + "Image": "as:Image", + "Invite": "as:Invite", + "Join": "as:Join", + "Leave": "as:Leave", + "Like": "as:Like", + "Link": "as:Link", + "Mention": "as:Mention", + "Note": "as:Note", + "Object": "as:Object", + "Offer": "as:Offer", + "OrderedCollection": "as:OrderedCollection", + "OrderedCollectionPage": "as:OrderedCollectionPage", + "Organization": "as:Organization", + "Page": "as:Page", + "Person": "as:Person", + "Place": "as:Place", + "Profile": "as:Profile", + "Question": "as:Question", + "Reject": "as:Reject", + "Remove": "as:Remove", + "Service": "as:Service", + "TentativeAccept": "as:TentativeAccept", + "TentativeReject": "as:TentativeReject", + "Tombstone": "as:Tombstone", + "Undo": "as:Undo", + "Update": "as:Update", + "Video": "as:Video", + "View": "as:View", + "Listen": "as:Listen", + "Read": "as:Read", + "Move": "as:Move", + "Travel": "as:Travel", + "IsFollowing": "as:IsFollowing", + "IsFollowedBy": "as:IsFollowedBy", + "IsContact": "as:IsContact", + "IsMember": "as:IsMember", + "subject": {"@id": "as:subject", "@type": "@id"}, + "relationship": {"@id": "as:relationship", "@type": "@id"}, + "actor": {"@id": "as:actor", "@type": "@id"}, + "attributedTo": {"@id": "as:attributedTo", "@type": "@id"}, + "attachment": {"@id": "as:attachment", "@type": "@id"}, + "bcc": {"@id": "as:bcc", "@type": "@id"}, + "bto": {"@id": "as:bto", "@type": "@id"}, + "cc": {"@id": "as:cc", "@type": "@id"}, + "context": {"@id": "as:context", "@type": "@id"}, + "current": {"@id": "as:current", "@type": "@id"}, + "first": {"@id": "as:first", "@type": "@id"}, + "generator": {"@id": "as:generator", "@type": "@id"}, + "icon": {"@id": "as:icon", "@type": "@id"}, + "image": {"@id": "as:image", "@type": "@id"}, + "inReplyTo": {"@id": "as:inReplyTo", "@type": "@id"}, + "items": {"@id": "as:items", "@type": "@id"}, + "instrument": {"@id": "as:instrument", "@type": "@id"}, + "orderedItems": { + "@id": "as:items", + "@type": "@id", + "@container": "@list", + }, + "last": {"@id": "as:last", "@type": "@id"}, + "location": {"@id": "as:location", "@type": "@id"}, + "next": {"@id": "as:next", "@type": "@id"}, + "object": {"@id": "as:object", "@type": "@id"}, + "oneOf": {"@id": "as:oneOf", "@type": "@id"}, + "anyOf": {"@id": "as:anyOf", "@type": "@id"}, + "closed": {"@id": "as:closed", "@type": "xsd:dateTime"}, + "origin": {"@id": "as:origin", "@type": "@id"}, + "accuracy": {"@id": "as:accuracy", "@type": "xsd:float"}, + "prev": {"@id": "as:prev", "@type": "@id"}, + "preview": {"@id": "as:preview", "@type": "@id"}, + "replies": {"@id": "as:replies", "@type": "@id"}, + "result": {"@id": "as:result", "@type": "@id"}, + "audience": {"@id": "as:audience", "@type": "@id"}, + "partOf": {"@id": "as:partOf", "@type": "@id"}, + "tag": {"@id": "as:tag", "@type": "@id"}, + "target": {"@id": "as:target", "@type": "@id"}, + "to": {"@id": "as:to", "@type": "@id"}, + "url": {"@id": "as:url", "@type": "@id"}, + "altitude": {"@id": "as:altitude", "@type": "xsd:float"}, + "content": "as:content", + "contentMap": {"@id": "as:content", "@container": "@language"}, + "name": "as:name", + "nameMap": {"@id": "as:name", "@container": "@language"}, + "duration": {"@id": "as:duration", "@type": "xsd:duration"}, + "endTime": {"@id": "as:endTime", "@type": "xsd:dateTime"}, + "height": {"@id": "as:height", "@type": "xsd:nonNegativeInteger"}, + "href": {"@id": "as:href", "@type": "@id"}, + "hreflang": "as:hreflang", + "latitude": {"@id": "as:latitude", "@type": "xsd:float"}, + "longitude": {"@id": "as:longitude", "@type": "xsd:float"}, + "mediaType": "as:mediaType", + "published": {"@id": "as:published", "@type": "xsd:dateTime"}, + "radius": {"@id": "as:radius", "@type": "xsd:float"}, + "rel": "as:rel", + "startIndex": { + "@id": "as:startIndex", + "@type": "xsd:nonNegativeInteger", + }, + "startTime": {"@id": "as:startTime", "@type": "xsd:dateTime"}, + "summary": "as:summary", + "summaryMap": {"@id": "as:summary", "@container": "@language"}, + "totalItems": { + "@id": "as:totalItems", + "@type": "xsd:nonNegativeInteger", + }, + "units": "as:units", + "updated": {"@id": "as:updated", "@type": "xsd:dateTime"}, + "width": {"@id": "as:width", "@type": "xsd:nonNegativeInteger"}, + "describes": {"@id": "as:describes", "@type": "@id"}, + "formerType": {"@id": "as:formerType", "@type": "@id"}, + "deleted": {"@id": "as:deleted", "@type": "xsd:dateTime"}, + "inbox": {"@id": "ldp:inbox", "@type": "@id"}, + "outbox": {"@id": "as:outbox", "@type": "@id"}, + "following": {"@id": "as:following", "@type": "@id"}, + "followers": {"@id": "as:followers", "@type": "@id"}, + "streams": {"@id": "as:streams", "@type": "@id"}, + "preferredUsername": "as:preferredUsername", + "endpoints": {"@id": "as:endpoints", "@type": "@id"}, + "uploadMedia": {"@id": "as:uploadMedia", "@type": "@id"}, + "proxyUrl": {"@id": "as:proxyUrl", "@type": "@id"}, + "liked": {"@id": "as:liked", "@type": "@id"}, + "oauthAuthorizationEndpoint": { + "@id": "as:oauthAuthorizationEndpoint", + "@type": "@id", + }, + "oauthTokenEndpoint": {"@id": "as:oauthTokenEndpoint", "@type": "@id"}, + "provideClientKey": {"@id": "as:provideClientKey", "@type": "@id"}, + "signClientKey": {"@id": "as:signClientKey", "@type": "@id"}, + "sharedInbox": {"@id": "as:sharedInbox", "@type": "@id"}, + "Public": {"@id": "as:Public", "@type": "@id"}, + "source": "as:source", + "likes": {"@id": "as:likes", "@type": "@id"}, + "shares": {"@id": "as:shares", "@type": "@id"}, + # Added manually + "manuallyApprovesFollowers": "as:manuallyApprovesFollowers", + } + }, + }, + { + "shortId": "SEC", + "contextUrl": None, + "documentUrl": "https://w3id.org/security/v1", + "document": { + "@context": { + "id": "@id", + "type": "@type", + "dc": "http://purl.org/dc/terms/", + "sec": "https://w3id.org/security#", + "xsd": "http://www.w3.org/2001/XMLSchema#", + "EcdsaKoblitzSignature2016": "sec:EcdsaKoblitzSignature2016", + "Ed25519Signature2018": "sec:Ed25519Signature2018", + "EncryptedMessage": "sec:EncryptedMessage", + "GraphSignature2012": "sec:GraphSignature2012", + "LinkedDataSignature2015": "sec:LinkedDataSignature2015", + "LinkedDataSignature2016": "sec:LinkedDataSignature2016", + "CryptographicKey": "sec:Key", + "authenticationTag": "sec:authenticationTag", + "canonicalizationAlgorithm": "sec:canonicalizationAlgorithm", + "cipherAlgorithm": "sec:cipherAlgorithm", + "cipherData": "sec:cipherData", + "cipherKey": "sec:cipherKey", + "created": {"@id": "dc:created", "@type": "xsd:dateTime"}, + "creator": {"@id": "dc:creator", "@type": "@id"}, + "digestAlgorithm": "sec:digestAlgorithm", + "digestValue": "sec:digestValue", + "domain": "sec:domain", + "encryptionKey": "sec:encryptionKey", + "expiration": {"@id": "sec:expiration", "@type": "xsd:dateTime"}, + "expires": {"@id": "sec:expiration", "@type": "xsd:dateTime"}, + "initializationVector": "sec:initializationVector", + "iterationCount": "sec:iterationCount", + "nonce": "sec:nonce", + "normalizationAlgorithm": "sec:normalizationAlgorithm", + "owner": {"@id": "sec:owner", "@type": "@id"}, + "password": "sec:password", + "privateKey": {"@id": "sec:privateKey", "@type": "@id"}, + "privateKeyPem": "sec:privateKeyPem", + "publicKey": {"@id": "sec:publicKey", "@type": "@id"}, + "publicKeyBase58": "sec:publicKeyBase58", + "publicKeyPem": "sec:publicKeyPem", + "publicKeyWif": "sec:publicKeyWif", + "publicKeyService": {"@id": "sec:publicKeyService", "@type": "@id"}, + "revoked": {"@id": "sec:revoked", "@type": "xsd:dateTime"}, + "salt": "sec:salt", + "signature": "sec:signature", + "signatureAlgorithm": "sec:signingAlgorithm", + "signatureValue": "sec:signatureValue", + } + }, + }, + { + "shortId": "FW", + "contextUrl": None, + "documentUrl": "https://funkwhale.audio/ns", + "document": { + "@context": { + "id": "@id", + "type": "@type", + "as": "https://www.w3.org/ns/activitystreams#", + "fw": "https://funkwhale.audio/ns#", + "xsd": "http://www.w3.org/2001/XMLSchema#", + "Album": "fw:Album", + "Track": "fw:Track", + "Artist": "fw:Artist", + "Library": "fw:Library", + "bitrate": {"@id": "fw:bitrate", "@type": "xsd:nonNegativeInteger"}, + "size": {"@id": "fw:size", "@type": "xsd:nonNegativeInteger"}, + "position": {"@id": "fw:position", "@type": "xsd:nonNegativeInteger"}, + "disc": {"@id": "fw:disc", "@type": "xsd:nonNegativeInteger"}, + "library": {"@id": "fw:library", "@type": "@id"}, + "track": {"@id": "fw:track", "@type": "@id"}, + "cover": {"@id": "fw:cover", "@type": "as:Link"}, + "album": {"@id": "fw:album", "@type": "@id"}, + "artists": {"@id": "fw:artists", "@type": "@id", "@container": "@list"}, + "released": {"@id": "fw:released", "@type": "xsd:date"}, + "musicbrainzId": "fw:musicbrainzId", + "license": {"@id": "fw:license", "@type": "@id"}, + "copyright": "fw:copyright", + } + }, + }, +] + +CONTEXTS_BY_ID = {c["shortId"]: c for c in CONTEXTS} + + +class NS: + def __init__(self, conf): + self.conf = conf + self.baseUrl = self.conf["document"]["@context"][self.conf["shortId"].lower()] + + def __repr__(self): + return "<{}: {}>".format(self.conf["shortId"], self.baseUrl) + + def __getattr__(self, key): + if key not in self.conf["document"]["@context"]: + raise AttributeError( + "{} is not a valid property of context {}".format(key, self.baseUrl) + ) + return self.baseUrl + key + + +class NoopContext: + def __getattr__(self, key): + return "_:{}".format(key) + + +NOOP = NoopContext() +AS = NS(CONTEXTS_BY_ID["AS"]) +LDP = NS(CONTEXTS_BY_ID["LDP"]) +SEC = NS(CONTEXTS_BY_ID["SEC"]) +FW = NS(CONTEXTS_BY_ID["FW"]) diff --git a/api/funkwhale_api/federation/jsonld.py b/api/funkwhale_api/federation/jsonld.py new file mode 100644 index 000000000..319ab3b6b --- /dev/null +++ b/api/funkwhale_api/federation/jsonld.py @@ -0,0 +1,276 @@ +import aiohttp +import asyncio +import functools + +import pyld.jsonld +from django.conf import settings +import pyld.documentloader.requests +from rest_framework import serializers +from rest_framework.fields import empty +from . import contexts + + +def cached_contexts(loader): + functools.wraps(loader) + + def load(url, *args, **kwargs): + for cached in contexts.CONTEXTS: + if url == cached["documentUrl"]: + return cached + return loader(url, *args, **kwargs) + + return load + + +def get_document_loader(): + loader = pyld.documentloader.requests.requests_document_loader( + verify=settings.EXTERNAL_REQUESTS_VERIFY_SSL + ) + return cached_contexts(loader) + + +def expand(doc, options=None, insert_fw_context=True): + options = options or {} + options.setdefault("documentLoader", get_document_loader()) + if isinstance(doc, str): + doc = options["documentLoader"](doc)["document"] + if insert_fw_context: + fw = contexts.CONTEXTS_BY_ID["FW"]["documentUrl"] + try: + insert_context(fw, doc) + except KeyError: + # probably an already expanded document + pass + result = pyld.jsonld.expand(doc, options=options) + try: + # jsonld.expand returns a list, which is useless for us + return result[0] + except IndexError: + raise ValueError("Impossible to expand this jsonld document") + + +def insert_context(ctx, doc): + """ + In some situations, we may want to add a default context to an existing document. + This function enable that (this will mutate the original document) + """ + existing = doc["@context"] + if isinstance(existing, list): + if ctx not in existing: + existing.append(ctx) + else: + doc["@context"] = [existing, ctx] + return doc + + +def get_session(): + return aiohttp.ClientSession(raise_for_status=True) + + +async def fetch_json(url, session, cache=None, lock=None): + async with session.get(url) as response: + response.raise_for_status() + return url, await response.json() + + +async def fetch_many(*ids, references=None): + """ + Given a list of object ids, will fetch the remote + representations for those objects, expand them + and return a dictionnary with id as the key and expanded document as the values + """ + ids = set(ids) + results = references if references is not None else {} + + if not ids: + return results + + async with get_session() as session: + tasks = [fetch_json(url, session) for url in ids if url not in results] + tasks_results = await asyncio.gather(*tasks) + + for url, payload in tasks_results: + results[url] = payload + + return results + + +DEFAULT_PREPARE_CONFIG = { + "type": {"property": "@type", "keep": "first"}, + "id": {"property": "@id"}, +} + + +def dereference(value, references): + """ + Given a payload and a dictonary containing ids and objects, will replace + all the matching objects in the payload by the one in the references dictionary. + """ + + def replace(obj, id): + try: + matching = references[id] + except KeyError: + return + # we clear the current dict, and replace its content by the matching obj + obj.clear() + obj.update(matching) + + if isinstance(value, dict): + if "@id" in value: + replace(value, value["@id"]) + else: + for attr in value.values(): + dereference(attr, references) + + elif isinstance(value, list): + # we loop on nested objects and trigger dereferencing + for obj in value: + dereference(obj, references) + + return value + + +def get_value(value, keep=None, attr=None): + + if keep == "first": + value = value[0] + if attr: + value = value[attr] + + elif attr: + value = [obj[attr] for obj in value if attr in obj] + + return value + + +def prepare_for_serializer(payload, config, fallbacks={}): + """ + Json-ld payloads, as returned by expand are quite complex to handle, because + every attr is basically a list of dictionnaries. To make code simpler, + we use this function to clean the payload a little bit, base on the config object. + + Config is a dictionnary, with keys being serializer field names, and values + being dictionaries describing how to handle this field. + """ + final_payload = {} + final_config = {} + final_config.update(DEFAULT_PREPARE_CONFIG) + final_config.update(config) + for field, field_config in final_config.items(): + try: + value = get_value( + payload[field_config["property"]], + keep=field_config.get("keep"), + attr=field_config.get("attr"), + ) + except (IndexError, KeyError): + aliases = field_config.get("aliases", []) + noop = object() + value = noop + if not aliases: + continue + + for a in aliases: + try: + value = get_value( + payload[a], + keep=field_config.get("keep"), + attr=field_config.get("attr"), + ) + except (IndexError, KeyError): + continue + + break + + if value is noop: + continue + + final_payload[field] = value + + for key, choices in fallbacks.items(): + if key in final_payload: + # initial attr was found, no need to rely on fallbacks + continue + + for choice in choices: + if choice not in final_payload: + continue + + final_payload[key] = final_payload[choice] + + return final_payload + + +def get_ids(v): + if isinstance(v, dict) and "@id" in v: + yield v["@id"] + + if isinstance(v, list): + for obj in v: + yield from get_ids(obj) + + +def get_default_context(): + return ["https://www.w3.org/ns/activitystreams", "https://w3id.org/security/v1", {}] + + +class JsonLdSerializer(serializers.Serializer): + def run_validation(self, data=empty): + if data and data is not empty and self.context.get("expand", True): + try: + data = expand(data) + except ValueError: + raise serializers.ValidationError( + "{} is not a valid jsonld document".format(data) + ) + try: + config = self.Meta.jsonld_mapping + except AttributeError: + config = {} + try: + fallbacks = self.Meta.jsonld_fallbacks + except AttributeError: + fallbacks = {} + data = prepare_for_serializer(data, config, fallbacks=fallbacks) + dereferenced_fields = [ + k + for k, c in config.items() + if k in data and c.get("dereference", False) + ] + dereferenced_ids = set() + for field in dereferenced_fields: + for i in get_ids(data[field]): + dereferenced_ids.add(i) + + if dereferenced_ids: + try: + loop = asyncio.get_event_loop() + except RuntimeError: + loop = asyncio.new_event_loop() + references = self.context.setdefault("references", {}) + loop.run_until_complete( + fetch_many(*dereferenced_ids, references=references) + ) + data = dereference(data, references) + return super().run_validation(data) + + +def first_attr(property, attr, aliases=[]): + return {"property": property, "keep": "first", "attr": attr, "aliases": aliases} + + +def first_val(property, aliases=[]): + return first_attr(property, "@value", aliases=aliases) + + +def first_id(property, aliases=[]): + return first_attr(property, "@id", aliases=aliases) + + +def first_obj(property, aliases=[]): + return {"property": property, "keep": "first", "aliases": aliases} + + +def raw(property, aliases=[]): + return {"property": property, "aliases": aliases} diff --git a/api/funkwhale_api/federation/renderers.py b/api/funkwhale_api/federation/renderers.py index d72c4c06a..a92658e59 100644 --- a/api/funkwhale_api/federation/renderers.py +++ b/api/funkwhale_api/federation/renderers.py @@ -1,8 +1,17 @@ from rest_framework.renderers import JSONRenderer -class ActivityPubRenderer(JSONRenderer): - media_type = "application/activity+json" +def get_ap_renderers(): + MEDIA_TYPES = [ + ("APActivity", "application/activity+json"), + ("APLD", "application/ld+json"), + ("APJSON", "application/json"), + ] + + return [ + type(name, (JSONRenderer,), {"media_type": media_type}) + for name, media_type in MEDIA_TYPES + ] class WebfingerRenderer(JSONRenderer): diff --git a/api/funkwhale_api/federation/serializers.py b/api/funkwhale_api/federation/serializers.py index d0e07cd85..6ee219288 100644 --- a/api/funkwhale_api/federation/serializers.py +++ b/api/funkwhale_api/federation/serializers.py @@ -9,22 +9,24 @@ from rest_framework import serializers from funkwhale_api.common import utils as funkwhale_utils from funkwhale_api.music import models as music_models -from . import activity, models, utils +from . import activity, contexts, jsonld, models, utils -AP_CONTEXT = [ - "https://www.w3.org/ns/activitystreams", - "https://w3id.org/security/v1", - {}, -] +AP_CONTEXT = jsonld.get_default_context() logger = logging.getLogger(__name__) -class LinkSerializer(serializers.Serializer): - type = serializers.ChoiceField(choices=["Link"]) +class LinkSerializer(jsonld.JsonLdSerializer): + type = serializers.ChoiceField(choices=[contexts.AS.Link]) href = serializers.URLField(max_length=500) mediaType = serializers.CharField() + class Meta: + jsonld_mapping = { + "href": jsonld.first_id(contexts.AS.href), + "mediaType": jsonld.first_val(contexts.AS.mediaType), + } + def __init__(self, *args, **kwargs): self.allowed_mimetypes = kwargs.pop("allowed_mimetypes", []) super().__init__(*args, **kwargs) @@ -45,18 +47,52 @@ class LinkSerializer(serializers.Serializer): ) -class ActorSerializer(serializers.Serializer): +class EndpointsSerializer(jsonld.JsonLdSerializer): + sharedInbox = serializers.URLField(max_length=500, required=False) + + class Meta: + jsonld_mapping = {"sharedInbox": jsonld.first_id(contexts.AS.sharedInbox)} + + +class PublicKeySerializer(jsonld.JsonLdSerializer): + publicKeyPem = serializers.CharField(trim_whitespace=False) + + class Meta: + jsonld_mapping = {"publicKeyPem": jsonld.first_val(contexts.SEC.publicKeyPem)} + + +class ActorSerializer(jsonld.JsonLdSerializer): id = serializers.URLField(max_length=500) outbox = serializers.URLField(max_length=500) inbox = serializers.URLField(max_length=500) - type = serializers.ChoiceField(choices=models.TYPE_CHOICES) + type = serializers.ChoiceField( + choices=[getattr(contexts.AS, c[0]) for c in models.TYPE_CHOICES] + ) preferredUsername = serializers.CharField() manuallyApprovesFollowers = serializers.NullBooleanField(required=False) name = serializers.CharField(required=False, max_length=200) summary = serializers.CharField(max_length=None, required=False) followers = serializers.URLField(max_length=500) following = serializers.URLField(max_length=500, required=False, allow_null=True) - publicKey = serializers.JSONField(required=False) + publicKey = PublicKeySerializer(required=False) + endpoints = EndpointsSerializer(required=False) + + class Meta: + jsonld_mapping = { + "outbox": jsonld.first_id(contexts.AS.outbox), + "inbox": jsonld.first_id(contexts.LDP.inbox), + "following": jsonld.first_id(contexts.AS.following), + "followers": jsonld.first_id(contexts.AS.followers), + "preferredUsername": jsonld.first_val(contexts.AS.preferredUsername), + "summary": jsonld.first_val(contexts.AS.summary), + "name": jsonld.first_val(contexts.AS.name), + "publicKey": jsonld.first_obj(contexts.SEC.publicKey), + "manuallyApprovesFollowers": jsonld.first_val( + contexts.AS.manuallyApprovesFollowers + ), + "mediaType": jsonld.first_val(contexts.AS.mediaType), + "endpoints": jsonld.first_obj(contexts.AS.endpoints), + } def to_representation(self, instance): ret = { @@ -115,16 +151,19 @@ class ActorSerializer(serializers.Serializer): kwargs["manually_approves_followers"] = maf domain = urllib.parse.urlparse(kwargs["fid"]).netloc kwargs["domain"] = models.Domain.objects.get_or_create(pk=domain)[0] - for endpoint, url in self.initial_data.get("endpoints", {}).items(): + for endpoint, url in self.validated_data.get("endpoints", {}).items(): if endpoint == "sharedInbox": kwargs["shared_inbox_url"] = url break try: - kwargs["public_key"] = self.initial_data["publicKey"]["publicKeyPem"] + kwargs["public_key"] = self.validated_data["publicKey"]["publicKeyPem"] except KeyError: pass return kwargs + def validate_type(self, v): + return v.split("#")[-1] + def build(self): d = self.prepare_missing_fields() return models.Actor(**d) @@ -507,14 +546,26 @@ def get_additional_fields(data): return additional_fields -class PaginatedCollectionSerializer(serializers.Serializer): - type = serializers.ChoiceField(choices=["Collection"]) +PAGINATED_COLLECTION_JSONLD_MAPPING = { + "totalItems": jsonld.first_val(contexts.AS.totalItems), + "actor": jsonld.first_id(contexts.AS.actor), + "first": jsonld.first_id(contexts.AS.first), + "last": jsonld.first_id(contexts.AS.last), + "partOf": jsonld.first_id(contexts.AS.partOf), +} + + +class PaginatedCollectionSerializer(jsonld.JsonLdSerializer): + type = serializers.ChoiceField(choices=[contexts.AS.Collection]) totalItems = serializers.IntegerField(min_value=0) actor = serializers.URLField(max_length=500) id = serializers.URLField(max_length=500) first = serializers.URLField(max_length=500) last = serializers.URLField(max_length=500) + class Meta: + jsonld_mapping = PAGINATED_COLLECTION_JSONLD_MAPPING + def to_representation(self, conf): paginator = Paginator(conf["items"], conf.get("page_size", 20)) first = funkwhale_utils.set_query_parameter(conf["id"], page=1) @@ -536,17 +587,30 @@ class PaginatedCollectionSerializer(serializers.Serializer): class LibrarySerializer(PaginatedCollectionSerializer): - type = serializers.ChoiceField(choices=["Library"]) + type = serializers.ChoiceField( + choices=[contexts.AS.Collection, contexts.FW.Library] + ) name = serializers.CharField() summary = serializers.CharField(allow_blank=True, allow_null=True, required=False) followers = serializers.URLField(max_length=500) audience = serializers.ChoiceField( - choices=["", None, "https://www.w3.org/ns/activitystreams#Public"], + choices=["", "./", None, "https://www.w3.org/ns/activitystreams#Public"], required=False, allow_null=True, allow_blank=True, ) + class Meta: + jsonld_mapping = funkwhale_utils.concat_dicts( + PAGINATED_COLLECTION_JSONLD_MAPPING, + { + "name": jsonld.first_val(contexts.AS.name), + "summary": jsonld.first_val(contexts.AS.summary), + "audience": jsonld.first_id(contexts.AS.audience), + "followers": jsonld.first_id(contexts.AS.followers), + }, + ) + def to_representation(self, library): conf = { "id": library.fid, @@ -559,9 +623,7 @@ class LibrarySerializer(PaginatedCollectionSerializer): } r = super().to_representation(conf) r["audience"] = ( - "https://www.w3.org/ns/activitystreams#Public" - if library.privacy_level == "everyone" - else "" + contexts.AS.Public if library.privacy_level == "everyone" else "" ) r["followers"] = library.followers_url return r @@ -572,6 +634,7 @@ class LibrarySerializer(PaginatedCollectionSerializer): queryset=models.Actor, serializer_class=ActorSerializer, ) + privacy = {"": "me", "./": "me", None: "me", contexts.AS.Public: "everyone"} library, created = music_models.Library.objects.update_or_create( fid=validated_data["id"], actor=actor, @@ -580,17 +643,14 @@ class LibrarySerializer(PaginatedCollectionSerializer): "name": validated_data["name"], "description": validated_data["summary"], "followers_url": validated_data["followers"], - "privacy_level": "everyone" - if validated_data["audience"] - == "https://www.w3.org/ns/activitystreams#Public" - else "me", + "privacy_level": privacy[validated_data["audience"]], }, ) return library -class CollectionPageSerializer(serializers.Serializer): - type = serializers.ChoiceField(choices=["CollectionPage"]) +class CollectionPageSerializer(jsonld.JsonLdSerializer): + type = serializers.ChoiceField(choices=[contexts.AS.CollectionPage]) totalItems = serializers.IntegerField(min_value=0) items = serializers.ListField() actor = serializers.URLField(max_length=500) @@ -601,6 +661,18 @@ class CollectionPageSerializer(serializers.Serializer): prev = serializers.URLField(max_length=500, required=False) partOf = serializers.URLField(max_length=500) + class Meta: + jsonld_mapping = { + "totalItems": jsonld.first_val(contexts.AS.totalItems), + "items": jsonld.raw(contexts.AS.items), + "actor": jsonld.first_id(contexts.AS.actor), + "first": jsonld.first_id(contexts.AS.first), + "last": jsonld.first_id(contexts.AS.last), + "next": jsonld.first_id(contexts.AS.next), + "prev": jsonld.first_id(contexts.AS.next), + "partOf": jsonld.first_id(contexts.AS.partOf), + } + def validate_items(self, v): item_serializer = self.context.get("item_serializer") if not item_serializer: @@ -654,7 +726,14 @@ class CollectionPageSerializer(serializers.Serializer): return d -class MusicEntitySerializer(serializers.Serializer): +MUSIC_ENTITY_JSONLD_MAPPING = { + "name": jsonld.first_val(contexts.AS.name), + "published": jsonld.first_val(contexts.AS.published), + "musicbrainzId": jsonld.first_val(contexts.FW.musicbrainzId), +} + + +class MusicEntitySerializer(jsonld.JsonLdSerializer): id = serializers.URLField(max_length=500) published = serializers.DateTimeField() musicbrainzId = serializers.UUIDField(allow_null=True, required=False) @@ -662,6 +741,9 @@ class MusicEntitySerializer(serializers.Serializer): class ArtistSerializer(MusicEntitySerializer): + class Meta: + jsonld_mapping = MUSIC_ENTITY_JSONLD_MAPPING + def to_representation(self, instance): d = { "type": "Artist", @@ -683,6 +765,16 @@ class AlbumSerializer(MusicEntitySerializer): allowed_mimetypes=["image/*"], allow_null=True, required=False ) + class Meta: + jsonld_mapping = funkwhale_utils.concat_dicts( + MUSIC_ENTITY_JSONLD_MAPPING, + { + "released": jsonld.first_val(contexts.FW.released), + "artists": jsonld.first_attr(contexts.FW.artists, "@list"), + "cover": jsonld.first_obj(contexts.FW.cover), + }, + ) + def to_representation(self, instance): d = { "type": "Album", @@ -710,22 +802,6 @@ class AlbumSerializer(MusicEntitySerializer): d["@context"] = AP_CONTEXT return d - def get_create_data(self, validated_data): - artist_data = validated_data["artists"][0] - artist = ArtistSerializer( - context={"activity": self.context.get("activity")} - ).create(artist_data) - - return { - "mbid": validated_data.get("musicbrainzId"), - "fid": validated_data["id"], - "title": validated_data["name"], - "creation_date": validated_data["published"], - "artist": artist, - "release_date": validated_data.get("released"), - "from_activity": self.context.get("activity"), - } - class TrackSerializer(MusicEntitySerializer): position = serializers.IntegerField(min_value=0, allow_null=True, required=False) @@ -735,6 +811,19 @@ class TrackSerializer(MusicEntitySerializer): license = serializers.URLField(allow_null=True, required=False) copyright = serializers.CharField(allow_null=True, required=False) + class Meta: + jsonld_mapping = funkwhale_utils.concat_dicts( + MUSIC_ENTITY_JSONLD_MAPPING, + { + "album": jsonld.first_obj(contexts.FW.album), + "artists": jsonld.first_attr(contexts.FW.artists, "@list"), + "copyright": jsonld.first_val(contexts.FW.copyright), + "disc": jsonld.first_val(contexts.FW.disc), + "license": jsonld.first_id(contexts.FW.license), + "position": jsonld.first_val(contexts.FW.position), + }, + ) + def to_representation(self, instance): d = { "type": "Track", @@ -773,8 +862,8 @@ class TrackSerializer(MusicEntitySerializer): return track -class UploadSerializer(serializers.Serializer): - type = serializers.ChoiceField(choices=["Audio"]) +class UploadSerializer(jsonld.JsonLdSerializer): + type = serializers.ChoiceField(choices=[contexts.AS.Audio]) id = serializers.URLField(max_length=500) library = serializers.URLField(max_length=500) url = LinkSerializer(allowed_mimetypes=["audio/*"]) @@ -786,6 +875,18 @@ class UploadSerializer(serializers.Serializer): track = TrackSerializer(required=True) + class Meta: + jsonld_mapping = { + "track": jsonld.first_obj(contexts.FW.track), + "library": jsonld.first_id(contexts.FW.library), + "url": jsonld.first_obj(contexts.AS.url), + "published": jsonld.first_val(contexts.AS.published), + "updated": jsonld.first_val(contexts.AS.updated), + "duration": jsonld.first_val(contexts.AS.duration), + "bitrate": jsonld.first_val(contexts.FW.bitrate), + "size": jsonld.first_val(contexts.FW.size), + } + def validate_url(self, v): try: v["href"] @@ -870,26 +971,6 @@ class UploadSerializer(serializers.Serializer): return d -class CollectionSerializer(serializers.Serializer): - def to_representation(self, conf): - d = { - "id": conf["id"], - "actor": conf["actor"].fid, - "totalItems": len(conf["items"]), - "type": "Collection", - "items": [ - conf["item_serializer"]( - i, context={"actor": conf["actor"], "include_ap_context": False} - ).data - for i in conf["items"] - ], - } - - if self.context.get("include_ap_context", True): - d["@context"] = AP_CONTEXT - return d - - class NodeInfoLinkSerializer(serializers.Serializer): href = serializers.URLField() rel = serializers.URLField() diff --git a/api/funkwhale_api/federation/utils.py b/api/funkwhale_api/federation/utils.py index e49a4dd63..7351763e3 100644 --- a/api/funkwhale_api/federation/utils.py +++ b/api/funkwhale_api/federation/utils.py @@ -100,7 +100,7 @@ def retrieve_ap_object( except KeyError: pass else: - if apply_instance_policies and activity.should_reject(id=id, payload=data): + if apply_instance_policies and activity.should_reject(fid=id, payload=data): raise exceptions.BlockedActorOrDomain() if not serializer_class: return data diff --git a/api/funkwhale_api/federation/views.py b/api/funkwhale_api/federation/views.py index 665a08b17..97bcebbfb 100644 --- a/api/funkwhale_api/federation/views.py +++ b/api/funkwhale_api/federation/views.py @@ -22,7 +22,7 @@ class FederationMixin(object): class SharedViewSet(FederationMixin, viewsets.GenericViewSet): permission_classes = [] authentication_classes = [authentication.SignatureAuthentication] - renderer_classes = [renderers.ActivityPubRenderer] + renderer_classes = renderers.get_ap_renderers() @action(methods=["post"], detail=False) def inbox(self, request, *args, **kwargs): @@ -39,7 +39,7 @@ class ActorViewSet(FederationMixin, mixins.RetrieveModelMixin, viewsets.GenericV lookup_field = "preferred_username" authentication_classes = [authentication.SignatureAuthentication] permission_classes = [] - renderer_classes = [renderers.ActivityPubRenderer] + renderer_classes = renderers.get_ap_renderers() queryset = models.Actor.objects.local().select_related("user") serializer_class = serializers.ActorSerializer @@ -74,7 +74,7 @@ class EditViewSet(FederationMixin, mixins.RetrieveModelMixin, viewsets.GenericVi lookup_field = "uuid" authentication_classes = [authentication.SignatureAuthentication] permission_classes = [] - renderer_classes = [renderers.ActivityPubRenderer] + renderer_classes = renderers.get_ap_renderers() # queryset = common_models.Mutation.objects.local().select_related() # serializer_class = serializers.ActorSerializer @@ -147,7 +147,7 @@ class MusicLibraryViewSet( ): authentication_classes = [authentication.SignatureAuthentication] permission_classes = [] - renderer_classes = [renderers.ActivityPubRenderer] + renderer_classes = renderers.get_ap_renderers() serializer_class = serializers.LibrarySerializer queryset = music_models.Library.objects.all().select_related("actor") lookup_field = "uuid" @@ -202,7 +202,7 @@ class MusicUploadViewSet( ): authentication_classes = [authentication.SignatureAuthentication] permission_classes = [] - renderer_classes = [renderers.ActivityPubRenderer] + renderer_classes = renderers.get_ap_renderers() queryset = music_models.Upload.objects.local().select_related( "library__actor", "track__artist", "track__album__artist" ) @@ -220,7 +220,7 @@ class MusicArtistViewSet( ): authentication_classes = [authentication.SignatureAuthentication] permission_classes = [] - renderer_classes = [renderers.ActivityPubRenderer] + renderer_classes = renderers.get_ap_renderers() queryset = music_models.Artist.objects.local() serializer_class = serializers.ArtistSerializer lookup_field = "uuid" @@ -231,7 +231,7 @@ class MusicAlbumViewSet( ): authentication_classes = [authentication.SignatureAuthentication] permission_classes = [] - renderer_classes = [renderers.ActivityPubRenderer] + renderer_classes = renderers.get_ap_renderers() queryset = music_models.Album.objects.local().select_related("artist") serializer_class = serializers.AlbumSerializer lookup_field = "uuid" @@ -242,7 +242,7 @@ class MusicTrackViewSet( ): authentication_classes = [authentication.SignatureAuthentication] permission_classes = [] - renderer_classes = [renderers.ActivityPubRenderer] + renderer_classes = renderers.get_ap_renderers() queryset = music_models.Track.objects.local().select_related( "album__artist", "artist" ) diff --git a/api/requirements/base.txt b/api/requirements/base.txt index c2fc95a6f..663f15ad6 100644 --- a/api/requirements/base.txt +++ b/api/requirements/base.txt @@ -64,3 +64,6 @@ django-cleanup==2.1.0 python-ldap==3.1.0 django-auth-ldap==1.7.0 pydub==0.23.0 + +pyld==1.0.4 +aiohttp==3.5.4 diff --git a/api/requirements/local.txt b/api/requirements/local.txt index 60724fc95..dcedb43e7 100644 --- a/api/requirements/local.txt +++ b/api/requirements/local.txt @@ -11,3 +11,6 @@ django-debug-toolbar>=1.11,<1.12 ipdb==0.11 black profiling + +asynctest==0.12.2 +aioresponses==0.6.0 diff --git a/api/tests/conftest.py b/api/tests/conftest.py index 03dbdfa4e..7a15c6d38 100644 --- a/api/tests/conftest.py +++ b/api/tests/conftest.py @@ -22,6 +22,7 @@ from django.db import connection from django.db.migrations.executor import MigrationExecutor from django.db.models import QuerySet +from aioresponses import aioresponses from dynamic_preferences.registries import global_preferences_registry from rest_framework import fields as rest_fields from rest_framework.test import APIClient, APIRequestFactory @@ -30,6 +31,9 @@ from funkwhale_api.activity import record from funkwhale_api.users.permissions import HasUserPermission +pytest_plugins = "aiohttp.pytest_plugin" + + class FunkwhaleProvider(internet_provider.Provider): """ Our own faker data generator, since built-in ones are sometimes @@ -416,3 +420,9 @@ def migrator(transactional_db): def rsa_small_key(settings): # smaller size for faster generation, since it's CPU hungry settings.RSA_KEY_SIZE = 512 + + +@pytest.fixture(autouse=True) +def a_responses(): + with aioresponses() as m: + yield m diff --git a/api/tests/federation/test_activity.py b/api/tests/federation/test_activity.py index fa83ed1f4..e3388a478 100644 --- a/api/tests/federation/test_activity.py +++ b/api/tests/federation/test_activity.py @@ -60,7 +60,7 @@ def test_receive_calls_should_reject(factories, now, mocker): copy = activity.receive(activity=a, on_behalf_of=remote_actor) should_reject.assert_called_once_with( - id=a["id"], actor_id=remote_actor.fid, payload=a + fid=a["id"], actor_id=remote_actor.fid, payload=a ) assert copy is None @@ -68,22 +68,28 @@ def test_receive_calls_should_reject(factories, now, mocker): @pytest.mark.parametrize( "params, policy_kwargs, expected", [ - ({"id": "https://ok.test"}, {"target_domain__name": "notok.test"}, False), + ({"fid": "https://ok.test"}, {"target_domain__name": "notok.test"}, False), ( - {"id": "https://ok.test"}, + {"fid": "https://ok.test"}, {"target_domain__name": "ok.test", "is_active": False}, False, ), ( - {"id": "https://ok.test"}, + {"fid": "https://ok.test"}, {"target_domain__name": "ok.test", "block_all": False}, False, ), # id match blocked domain - ({"id": "http://notok.test"}, {"target_domain__name": "notok.test"}, True), + ({"fid": "http://notok.test"}, {"target_domain__name": "notok.test"}, True), # actor id match blocked domain ( - {"id": "http://ok.test", "actor_id": "https://notok.test"}, + {"fid": "http://ok.test", "actor_id": "https://notok.test"}, + {"target_domain__name": "notok.test"}, + True, + ), + # actor id match blocked domain + ( + {"fid": None, "actor_id": "https://notok.test"}, {"target_domain__name": "notok.test"}, True, ), @@ -91,7 +97,7 @@ def test_receive_calls_should_reject(factories, now, mocker): ( { "payload": {"type": "Library"}, - "id": "http://ok.test", + "fid": "http://ok.test", "actor_id": "http://notok.test", }, { diff --git a/api/tests/federation/test_authentication.py b/api/tests/federation/test_authentication.py index 3298f9543..4e837e641 100644 --- a/api/tests/federation/test_authentication.py +++ b/api/tests/federation/test_authentication.py @@ -1,6 +1,6 @@ import pytest -from funkwhale_api.federation import authentication, exceptions, keys +from funkwhale_api.federation import authentication, exceptions, keys, jsonld def test_authenticate(factories, mocker, api_request): @@ -10,6 +10,7 @@ def test_authenticate(factories, mocker, api_request): mocker.patch( "funkwhale_api.federation.actors.get_actor_data", return_value={ + "@context": jsonld.get_default_context(), "id": actor_url, "type": "Person", "outbox": "https://test.com", @@ -105,6 +106,7 @@ def test_authenticate_ignore_inactive_policy(factories, api_request, mocker): mocker.patch( "funkwhale_api.federation.actors.get_actor_data", return_value={ + "@context": jsonld.get_default_context(), "id": actor_url, "type": "Person", "outbox": "https://test.com", @@ -142,6 +144,7 @@ def test_autenthicate_supports_blind_key_rotation(factories, mocker, api_request mocker.patch( "funkwhale_api.federation.actors.get_actor_data", return_value={ + "@context": jsonld.get_default_context(), "id": actor_url, "type": "Person", "outbox": "https://test.com", diff --git a/api/tests/federation/test_contexts.py b/api/tests/federation/test_contexts.py new file mode 100644 index 000000000..a0134d909 --- /dev/null +++ b/api/tests/federation/test_contexts.py @@ -0,0 +1,32 @@ +import pytest + +from funkwhale_api.federation import contexts + + +@pytest.mark.parametrize( + "ns, property, expected", + [ + ("AS", "followers", "https://www.w3.org/ns/activitystreams#followers"), + ("AS", "following", "https://www.w3.org/ns/activitystreams#following"), + ("SEC", "owner", "https://w3id.org/security#owner"), + ("SEC", "publicKey", "https://w3id.org/security#publicKey"), + ], +) +def test_context_ns(ns, property, expected): + ns = getattr(contexts, ns) + id = getattr(ns, property) + assert id == expected + + +def test_raise_on_wrong_attr(): + ns = contexts.AS + with pytest.raises(AttributeError): + ns.noop + + +@pytest.mark.parametrize( + "property, expected", + [("publicKey", "_:publicKey"), ("cover", "_:cover"), ("hello", "_:hello")], +) +def test_noop_context(property, expected): + assert getattr(contexts.NOOP, property) == expected diff --git a/api/tests/federation/test_jsonld.py b/api/tests/federation/test_jsonld.py new file mode 100644 index 000000000..ad201b748 --- /dev/null +++ b/api/tests/federation/test_jsonld.py @@ -0,0 +1,361 @@ +import pytest + +from rest_framework import serializers + +from funkwhale_api.federation import contexts +from funkwhale_api.federation import jsonld + + +def test_expand_no_external_request(): + payload = { + "id": "https://noop/federation/actors/demo", + "outbox": "https://noop/federation/actors/demo/outbox", + "inbox": "https://noop/federation/actors/demo/inbox", + "preferredUsername": "demo", + "type": "Person", + "name": "demo", + "followers": "https://noop/federation/actors/demo/followers", + "following": "https://noop/federation/actors/demo/following", + "manuallyApprovesFollowers": False, + "@context": [ + "https://www.w3.org/ns/activitystreams", + "https://w3id.org/security/v1", + {}, + ], + "publicKey": { + "owner": "https://noop/federation/actors/demo", + "publicKeyPem": "-----BEGIN RSA PUBLIC KEY-----\nMIIBCgKCAQEAxPDd/oXx0ClJ2BuBZ937AiERjvoroEpNebg34Cdl6FYsb2Auib8b\nCQjdjLjK/1ag35lmqmsECqtoDYWOo4tGilZJW47TWmXfcvCMH2Sw9FqdOlzpV1RI\nm8kc0Lu1CC2xOTctqIwSH7kDDnS4+S5hSxRdMTeNQNoirncY1CXa9TmJR1lE2HWz\n+B05ewzMrSen3l3fJLQFoI2GVbbjj+tvILKBL1oG5MtYieYqjt2sqtqy/OpWUAC7\nlRERRzd4t5xPBKykWkBCAOh80pvPue5V4s+xUMr7ioKTcm6pq+pNBta5w0hUYIcT\nMefQOnNuR4J0meIqiDLcrglGAmM6AVFwYwIDAQAB\n-----END RSA PUBLIC KEY-----\n", # noqa + "id": "https://noop/federation/actors/demo#main-key", + }, + "endpoints": {"sharedInbox": "https://noop/federation/shared/inbox"}, + } + + expected = { + contexts.AS.endpoints: [ + {contexts.AS.sharedInbox: [{"@id": "https://noop/federation/shared/inbox"}]} + ], + contexts.AS.followers: [ + {"@id": "https://noop/federation/actors/demo/followers"} + ], + contexts.AS.following: [ + {"@id": "https://noop/federation/actors/demo/following"} + ], + "@id": "https://noop/federation/actors/demo", + "http://www.w3.org/ns/ldp#inbox": [ + {"@id": "https://noop/federation/actors/demo/inbox"} + ], + contexts.AS.manuallyApprovesFollowers: [{"@value": False}], + contexts.AS.name: [{"@value": "demo"}], + contexts.AS.outbox: [{"@id": "https://noop/federation/actors/demo/outbox"}], + contexts.AS.preferredUsername: [{"@value": "demo"}], + contexts.SEC.publicKey: [ + { + "@id": "https://noop/federation/actors/demo#main-key", + contexts.SEC.owner: [{"@id": "https://noop/federation/actors/demo"}], + contexts.SEC.publicKeyPem: [ + { + "@value": "-----BEGIN RSA PUBLIC KEY-----\nMIIBCgKCAQEAxPDd/oXx0ClJ2BuBZ937AiERjvoroEpNebg34Cdl6FYsb2Auib8b\nCQjdjLjK/1ag35lmqmsECqtoDYWOo4tGilZJW47TWmXfcvCMH2Sw9FqdOlzpV1RI\nm8kc0Lu1CC2xOTctqIwSH7kDDnS4+S5hSxRdMTeNQNoirncY1CXa9TmJR1lE2HWz\n+B05ewzMrSen3l3fJLQFoI2GVbbjj+tvILKBL1oG5MtYieYqjt2sqtqy/OpWUAC7\nlRERRzd4t5xPBKykWkBCAOh80pvPue5V4s+xUMr7ioKTcm6pq+pNBta5w0hUYIcT\nMefQOnNuR4J0meIqiDLcrglGAmM6AVFwYwIDAQAB\n-----END RSA PUBLIC KEY-----\n" # noqa + } + ], + } + ], + "@type": [contexts.AS.Person], + } + + doc = jsonld.expand(payload) + + assert doc == expected + + +def test_expand_remote_doc(r_mock): + url = "https://noop/federation/actors/demo" + payload = { + "id": url, + "outbox": "https://noop/federation/actors/demo/outbox", + "inbox": "https://noop/federation/actors/demo/inbox", + "preferredUsername": "demo", + "type": "Person", + "name": "demo", + "followers": "https://noop/federation/actors/demo/followers", + "following": "https://noop/federation/actors/demo/following", + "manuallyApprovesFollowers": False, + "@context": [ + "https://www.w3.org/ns/activitystreams", + "https://w3id.org/security/v1", + {}, + ], + "publicKey": { + "owner": "https://noop/federation/actors/demo", + "publicKeyPem": "-----BEGIN RSA PUBLIC KEY-----\nMIIBCgKCAQEAxPDd/oXx0ClJ2BuBZ937AiERjvoroEpNebg34Cdl6FYsb2Auib8b\nCQjdjLjK/1ag35lmqmsECqtoDYWOo4tGilZJW47TWmXfcvCMH2Sw9FqdOlzpV1RI\nm8kc0Lu1CC2xOTctqIwSH7kDDnS4+S5hSxRdMTeNQNoirncY1CXa9TmJR1lE2HWz\n+B05ewzMrSen3l3fJLQFoI2GVbbjj+tvILKBL1oG5MtYieYqjt2sqtqy/OpWUAC7\nlRERRzd4t5xPBKykWkBCAOh80pvPue5V4s+xUMr7ioKTcm6pq+pNBta5w0hUYIcT\nMefQOnNuR4J0meIqiDLcrglGAmM6AVFwYwIDAQAB\n-----END RSA PUBLIC KEY-----\n", # noqa + "id": "https://noop/federation/actors/demo#main-key", + }, + "endpoints": {"sharedInbox": "https://noop/federation/shared/inbox"}, + } + r_mock.get(url, json=payload) + + expected = { + contexts.AS.endpoints: [ + {contexts.AS.sharedInbox: [{"@id": "https://noop/federation/shared/inbox"}]} + ], + contexts.AS.followers: [ + {"@id": "https://noop/federation/actors/demo/followers"} + ], + contexts.AS.following: [ + {"@id": "https://noop/federation/actors/demo/following"} + ], + "@id": "https://noop/federation/actors/demo", + "http://www.w3.org/ns/ldp#inbox": [ + {"@id": "https://noop/federation/actors/demo/inbox"} + ], + contexts.AS.manuallyApprovesFollowers: [{"@value": False}], + contexts.AS.name: [{"@value": "demo"}], + contexts.AS.outbox: [{"@id": "https://noop/federation/actors/demo/outbox"}], + contexts.AS.preferredUsername: [{"@value": "demo"}], + contexts.SEC.publicKey: [ + { + "@id": "https://noop/federation/actors/demo#main-key", + contexts.SEC.owner: [{"@id": "https://noop/federation/actors/demo"}], + contexts.SEC.publicKeyPem: [ + { + "@value": "-----BEGIN RSA PUBLIC KEY-----\nMIIBCgKCAQEAxPDd/oXx0ClJ2BuBZ937AiERjvoroEpNebg34Cdl6FYsb2Auib8b\nCQjdjLjK/1ag35lmqmsECqtoDYWOo4tGilZJW47TWmXfcvCMH2Sw9FqdOlzpV1RI\nm8kc0Lu1CC2xOTctqIwSH7kDDnS4+S5hSxRdMTeNQNoirncY1CXa9TmJR1lE2HWz\n+B05ewzMrSen3l3fJLQFoI2GVbbjj+tvILKBL1oG5MtYieYqjt2sqtqy/OpWUAC7\nlRERRzd4t5xPBKykWkBCAOh80pvPue5V4s+xUMr7ioKTcm6pq+pNBta5w0hUYIcT\nMefQOnNuR4J0meIqiDLcrglGAmM6AVFwYwIDAQAB\n-----END RSA PUBLIC KEY-----\n" # noqa + } + ], + } + ], + "@type": [contexts.AS.Person], + } + + doc = jsonld.expand(url) + + assert doc == expected + + +async def test_fetch_many(a_responses): + doc = { + "@context": ["https://www.w3.org/ns/activitystreams", {}], + "id": "https://noop/federation/actors/demo", + "type": "Person", + "followers": "https://noop/federation/actors/demo/followers", + } + followers_doc = { + "@context": ["https://www.w3.org/ns/activitystreams", {}], + "id": "https://noop/federation/actors/demo/followers", + "type": "Collection", + } + + a_responses.get(doc["id"], payload=doc) + a_responses.get(followers_doc["id"], payload=followers_doc) + fetched = await jsonld.fetch_many(doc["id"], followers_doc["id"]) + assert fetched == {followers_doc["id"]: followers_doc, doc["id"]: doc} + + +def test_dereference(): + + followers_doc = { + "@context": ["https://www.w3.org/ns/activitystreams", {}], + "id": "https://noop/federation/actors/demo/followers", + "type": "Collection", + } + + actor_doc = { + "@context": ["https://www.w3.org/ns/activitystreams", {}], + "id": "https://noop/federation/actors/demo", + "type": "Person", + "followers": "https://noop/federation/actors/demo/followers", + } + + store = {followers_doc["id"]: followers_doc, actor_doc["id"]: actor_doc} + + payload = { + "followers": {"@id": followers_doc["id"]}, + "actor": [ + {"@id": actor_doc["id"], "hello": "world"}, + {"somethingElse": [{"@id": actor_doc["id"]}]}, + ], + } + expected = { + "followers": followers_doc, + "actor": [actor_doc, {"somethingElse": [actor_doc]}], + } + + assert jsonld.dereference(payload, store) == expected + + +def test_prepare_for_serializer(): + config = { + "followers": { + "property": contexts.AS.followers, + "keep": "first", + "attr": "@id", + }, + "name": {"property": contexts.AS.name, "keep": "first", "attr": "@value"}, + "keys": {"property": contexts.SEC.publicKey, "type": "raw"}, + } + + payload = { + "@id": "https://noop/federation/actors/demo", + "@type": [contexts.AS.Person], + contexts.AS.followers: [ + {"@id": "https://noop/federation/actors/demo/followers"} + ], + contexts.AS.name: [{"@value": "demo"}], + contexts.SEC.publicKey: [ + {"@id": "https://noop/federation/actors/demo#main-key1"}, + {"@id": "https://noop/federation/actors/demo#main-key2"}, + ], + } + + expected = { + "id": "https://noop/federation/actors/demo", + "type": contexts.AS.Person, + "followers": "https://noop/federation/actors/demo/followers", + "name": "demo", + "keys": [ + {"@id": "https://noop/federation/actors/demo#main-key1"}, + {"@id": "https://noop/federation/actors/demo#main-key2"}, + ], + } + + assert jsonld.prepare_for_serializer(payload, config) == expected + + +def test_prepare_for_serializer_fallback(): + config = { + "name": {"property": contexts.AS.name, "keep": "first", "attr": "@value"}, + "album": {"property": contexts.FW.Album, "keep": "first"}, + "noop_album": {"property": contexts.NOOP.Album, "keep": "first"}, + } + fallbacks = {"album": ["noop_album"]} + + payload = { + "@id": "https://noop/federation/actors/demo", + "@type": [contexts.AS.Person], + contexts.AS.name: [{"@value": "demo"}], + contexts.NOOP.Album: [{"@id": "https://noop/federation/album/demo"}], + } + + expected = { + "id": "https://noop/federation/actors/demo", + "type": contexts.AS.Person, + "name": "demo", + "album": {"@id": "https://noop/federation/album/demo"}, + "noop_album": {"@id": "https://noop/federation/album/demo"}, + } + + assert ( + jsonld.prepare_for_serializer(payload, config, fallbacks=fallbacks) == expected + ) + + +def test_jsonld_serializer_fallback(): + class TestSerializer(jsonld.JsonLdSerializer): + id = serializers.URLField() + type = serializers.CharField() + name = serializers.CharField() + username = serializers.CharField() + total = serializers.IntegerField() + + class Meta: + jsonld_fallbacks = {"total": ["total_fallback"]} + jsonld_mapping = { + "name": { + "property": contexts.AS.name, + "keep": "first", + "attr": "@value", + }, + "username": { + "property": contexts.AS.preferredUsername, + "keep": "first", + "attr": "@value", + }, + "total": { + "property": contexts.AS.totalItems, + "keep": "first", + "attr": "@value", + }, + "total_fallback": { + "property": contexts.NOOP.count, + "keep": "first", + "attr": "@value", + }, + } + + payload = { + "@context": ["https://www.w3.org/ns/activitystreams", {}], + "id": "https://noop.url/federation/actors/demo", + "type": "Person", + "name": "Hello", + "preferredUsername": "World", + "count": 42, + } + + serializer = TestSerializer(data=payload) + assert serializer.is_valid(raise_exception=True) + + assert serializer.validated_data == { + "type": contexts.AS.Person, + "id": payload["id"], + "name": payload["name"], + "username": payload["preferredUsername"], + "total": 42, + } + + +def test_jsonld_serializer_dereference(a_responses): + class TestSerializer(jsonld.JsonLdSerializer): + id = serializers.URLField() + type = serializers.CharField() + followers = serializers.JSONField() + + class Meta: + jsonld_mapping = { + "followers": {"property": contexts.AS.followers, "dereference": True} + } + + payload = { + "@context": ["https://www.w3.org/ns/activitystreams", {}], + "id": "https://noop.url/federation/actors/demo", + "type": "Person", + "followers": "https://noop.url/federation/actors/demo/followers", + } + + followers_doc = { + "@context": ["https://www.w3.org/ns/activitystreams", {}], + "id": "https://noop.url/federation/actors/demo/followers", + "type": "Collection", + } + + a_responses.get(followers_doc["id"], payload=followers_doc) + serializer = TestSerializer(data=payload) + + assert serializer.is_valid(raise_exception=True) + assert serializer.validated_data == { + "type": contexts.AS.Person, + "id": payload["id"], + "followers": [followers_doc], + } + + +@pytest.mark.parametrize( + "doc, ctx, expected", + [ + ( + {"@context": [{}], "hello": "world"}, + "http://test", + {"@context": [{}, "http://test"], "hello": "world"}, + ), + ( + {"@context": {"key": "value"}, "hello": "world"}, + "http://test", + {"@context": [{"key": "value"}, "http://test"], "hello": "world"}, + ), + ( + {"@context": "http://as", "hello": "world"}, + "http://test", + {"@context": ["http://as", "http://test"], "hello": "world"}, + ), + ], +) +def test_insert_context(doc, ctx, expected): + jsonld.insert_context(ctx, doc) + assert doc == expected diff --git a/api/tests/federation/test_routes.py b/api/tests/federation/test_routes.py index 7232b746c..438f45c22 100644 --- a/api/tests/federation/test_routes.py +++ b/api/tests/federation/test_routes.py @@ -1,6 +1,6 @@ import pytest -from funkwhale_api.federation import routes, serializers +from funkwhale_api.federation import jsonld, routes, serializers @pytest.mark.parametrize( @@ -190,6 +190,7 @@ def test_inbox_create_audio(factories, mocker): activity = factories["federation.Activity"]() upload = factories["music.Upload"](bitrate=42, duration=55) payload = { + "@context": jsonld.get_default_context(), "type": "Create", "actor": upload.library.actor.fid, "object": serializers.UploadSerializer(upload).data, diff --git a/api/tests/federation/test_serializers.py b/api/tests/federation/test_serializers.py index 207a8fbe5..bde9128cb 100644 --- a/api/tests/federation/test_serializers.py +++ b/api/tests/federation/test_serializers.py @@ -1,930 +1,49 @@ -import io -import pytest -import uuid - -from django.core.paginator import Paginator -from django.utils import timezone - -from funkwhale_api.federation import models, serializers, utils +from funkwhale_api.federation import keys +from funkwhale_api.federation import jsonld +from funkwhale_api.federation import serializers def test_actor_serializer_from_ap(db): + private, public = keys.get_key_pair() + actor_url = "https://test.federation/actor" payload = { - "id": "https://test.federation/user", + "@context": jsonld.get_default_context(), + "id": actor_url, "type": "Person", - "following": "https://test.federation/user/following", - "followers": "https://test.federation/user/followers", - "inbox": "https://test.federation/user/inbox", - "outbox": "https://test.federation/user/outbox", - "preferredUsername": "user", - "name": "Real User", + "outbox": "https://test.com/outbox", + "inbox": "https://test.com/inbox", + "following": "https://test.com/following", + "followers": "https://test.com/followers", + "preferredUsername": "test", + "name": "Test", "summary": "Hello world", - "url": "https://test.federation/@user", - "manuallyApprovesFollowers": False, + "manuallyApprovesFollowers": True, "publicKey": { - "id": "https://test.federation/user#main-key", - "owner": "https://test.federation/user", - "publicKeyPem": "yolo", + "publicKeyPem": public.decode("utf-8"), + "owner": actor_url, + "id": actor_url + "#main-key", }, - "endpoints": {"sharedInbox": "https://test.federation/inbox"}, + "endpoints": {"sharedInbox": "https://noop.url/federation/shared/inbox"}, } serializer = serializers.ActorSerializer(data=payload) assert serializer.is_valid(raise_exception=True) + actor = serializer.save() - actor = serializer.build() - - assert actor.fid == payload["id"] + assert actor.fid == actor_url + assert actor.url is None assert actor.inbox_url == payload["inbox"] - assert actor.outbox_url == payload["outbox"] assert actor.shared_inbox_url == payload["endpoints"]["sharedInbox"] - assert actor.followers_url == payload["followers"] + assert actor.outbox_url == payload["outbox"] assert actor.following_url == payload["following"] - assert actor.public_key == payload["publicKey"]["publicKeyPem"] + assert actor.followers_url == payload["followers"] + assert actor.followers_url == payload["followers"] + assert actor.type == "Person" assert actor.preferred_username == payload["preferredUsername"] assert actor.name == payload["name"] - assert actor.domain.pk == "test.federation" assert actor.summary == payload["summary"] - assert actor.type == "Person" - assert actor.manually_approves_followers == payload["manuallyApprovesFollowers"] - - -def test_actor_serializer_only_mandatory_field_from_ap(db): - payload = { - "id": "https://test.federation/user", - "type": "Person", - "following": "https://test.federation/user/following", - "followers": "https://test.federation/user/followers", - "inbox": "https://test.federation/user/inbox", - "outbox": "https://test.federation/user/outbox", - "preferredUsername": "user", - } - - serializer = serializers.ActorSerializer(data=payload) - assert serializer.is_valid(raise_exception=True) - - actor = serializer.build() - - assert actor.fid == payload["id"] - assert actor.inbox_url == payload["inbox"] - assert actor.outbox_url == payload["outbox"] - assert actor.followers_url == payload["followers"] - assert actor.following_url == payload["following"] - assert actor.preferred_username == payload["preferredUsername"] - assert actor.domain.pk == "test.federation" - assert actor.type == "Person" - assert actor.manually_approves_followers is None - - -def test_actor_serializer_to_ap(): - expected = { - "@context": [ - "https://www.w3.org/ns/activitystreams", - "https://w3id.org/security/v1", - {}, - ], - "id": "https://test.federation/user", - "type": "Person", - "following": "https://test.federation/user/following", - "followers": "https://test.federation/user/followers", - "inbox": "https://test.federation/user/inbox", - "outbox": "https://test.federation/user/outbox", - "preferredUsername": "user", - "name": "Real User", - "summary": "Hello world", - "manuallyApprovesFollowers": False, - "publicKey": { - "id": "https://test.federation/user#main-key", - "owner": "https://test.federation/user", - "publicKeyPem": "yolo", - }, - "endpoints": {"sharedInbox": "https://test.federation/inbox"}, - } - ac = models.Actor( - fid=expected["id"], - inbox_url=expected["inbox"], - outbox_url=expected["outbox"], - shared_inbox_url=expected["endpoints"]["sharedInbox"], - followers_url=expected["followers"], - following_url=expected["following"], - public_key=expected["publicKey"]["publicKeyPem"], - preferred_username=expected["preferredUsername"], - name=expected["name"], - domain=models.Domain(pk="test.federation"), - summary=expected["summary"], - type="Person", - manually_approves_followers=False, - ) - serializer = serializers.ActorSerializer(ac) - - assert serializer.data == expected - - -def test_webfinger_serializer(): - expected = { - "subject": "acct:service@test.federation", - "links": [ - { - "rel": "self", - "href": "https://test.federation/federation/instance/actor", - "type": "application/activity+json", - } - ], - "aliases": ["https://test.federation/federation/instance/actor"], - } - actor = models.Actor( - fid=expected["links"][0]["href"], - preferred_username="service", - domain=models.Domain(pk="test.federation"), - ) - serializer = serializers.ActorWebfingerSerializer(actor) - - assert serializer.data == expected - - -def test_follow_serializer_to_ap(factories): - follow = factories["federation.Follow"](local=True) - serializer = serializers.FollowSerializer(follow) - - expected = { - "@context": [ - "https://www.w3.org/ns/activitystreams", - "https://w3id.org/security/v1", - {}, - ], - "id": follow.get_federation_id(), - "type": "Follow", - "actor": follow.actor.fid, - "object": follow.target.fid, - } - - assert serializer.data == expected - - -def test_follow_serializer_save(factories): - actor = factories["federation.Actor"]() - target = factories["federation.Actor"]() - - data = { - "id": "https://test.follow", - "type": "Follow", - "actor": actor.fid, - "object": target.fid, - } - serializer = serializers.FollowSerializer(data=data) - - assert serializer.is_valid(raise_exception=True) - - follow = serializer.save() - - assert follow.pk is not None - assert follow.actor == actor - assert follow.target == target - assert follow.approved is None - - -def test_follow_serializer_save_validates_on_context(factories): - actor = factories["federation.Actor"]() - target = factories["federation.Actor"]() - impostor = factories["federation.Actor"]() - - data = { - "id": "https://test.follow", - "type": "Follow", - "actor": actor.fid, - "object": target.fid, - } - serializer = serializers.FollowSerializer( - data=data, context={"follow_actor": impostor, "follow_target": impostor} - ) - - assert serializer.is_valid() is False - - assert "actor" in serializer.errors - assert "object" in serializer.errors - - -def test_accept_follow_serializer_representation(factories): - follow = factories["federation.Follow"](approved=None) - - expected = { - "@context": [ - "https://www.w3.org/ns/activitystreams", - "https://w3id.org/security/v1", - {}, - ], - "id": follow.get_federation_id() + "/accept", - "type": "Accept", - "actor": follow.target.fid, - "object": serializers.FollowSerializer(follow).data, - } - - serializer = serializers.AcceptFollowSerializer(follow) - - assert serializer.data == expected - - -def test_accept_follow_serializer_save(factories): - follow = factories["federation.Follow"](approved=None) - - data = { - "@context": [ - "https://www.w3.org/ns/activitystreams", - "https://w3id.org/security/v1", - {}, - ], - "id": follow.get_federation_id() + "/accept", - "type": "Accept", - "actor": follow.target.fid, - "object": serializers.FollowSerializer(follow).data, - } - - serializer = serializers.AcceptFollowSerializer(data=data) - assert serializer.is_valid(raise_exception=True) - serializer.save() - - follow.refresh_from_db() - - assert follow.approved is True - - -def test_accept_follow_serializer_validates_on_context(factories): - follow = factories["federation.Follow"](approved=None) - impostor = factories["federation.Actor"]() - data = { - "@context": [ - "https://www.w3.org/ns/activitystreams", - "https://w3id.org/security/v1", - {}, - ], - "id": follow.get_federation_id() + "/accept", - "type": "Accept", - "actor": impostor.url, - "object": serializers.FollowSerializer(follow).data, - } - - serializer = serializers.AcceptFollowSerializer( - data=data, context={"follow_actor": impostor, "follow_target": impostor} - ) - - assert serializer.is_valid() is False - assert "actor" in serializer.errors["object"] - assert "object" in serializer.errors["object"] - - -def test_undo_follow_serializer_representation(factories): - follow = factories["federation.Follow"](approved=True) - - expected = { - "@context": [ - "https://www.w3.org/ns/activitystreams", - "https://w3id.org/security/v1", - {}, - ], - "id": follow.get_federation_id() + "/undo", - "type": "Undo", - "actor": follow.actor.fid, - "object": serializers.FollowSerializer(follow).data, - } - - serializer = serializers.UndoFollowSerializer(follow) - - assert serializer.data == expected - - -def test_undo_follow_serializer_save(factories): - follow = factories["federation.Follow"](approved=True) - - data = { - "@context": [ - "https://www.w3.org/ns/activitystreams", - "https://w3id.org/security/v1", - {}, - ], - "id": follow.get_federation_id() + "/undo", - "type": "Undo", - "actor": follow.actor.fid, - "object": serializers.FollowSerializer(follow).data, - } - - serializer = serializers.UndoFollowSerializer(data=data) - assert serializer.is_valid(raise_exception=True) - serializer.save() - - with pytest.raises(models.Follow.DoesNotExist): - follow.refresh_from_db() - - -def test_undo_follow_serializer_validates_on_context(factories): - follow = factories["federation.Follow"](approved=True) - impostor = factories["federation.Actor"]() - data = { - "@context": [ - "https://www.w3.org/ns/activitystreams", - "https://w3id.org/security/v1", - {}, - ], - "id": follow.get_federation_id() + "/undo", - "type": "Undo", - "actor": impostor.url, - "object": serializers.FollowSerializer(follow).data, - } - - serializer = serializers.UndoFollowSerializer( - data=data, context={"follow_actor": impostor, "follow_target": impostor} - ) - - assert serializer.is_valid() is False - assert "actor" in serializer.errors["object"] - assert "object" in serializer.errors["object"] - - -def test_paginated_collection_serializer(factories): - uploads = factories["music.Upload"].create_batch(size=5) - actor = factories["federation.Actor"](local=True) - - conf = { - "id": "https://test.federation/test", - "items": uploads, - "item_serializer": serializers.UploadSerializer, - "actor": actor, - "page_size": 2, - } - expected = { - "@context": [ - "https://www.w3.org/ns/activitystreams", - "https://w3id.org/security/v1", - {}, - ], - "type": "Collection", - "id": conf["id"], - "actor": actor.fid, - "totalItems": len(uploads), - "current": conf["id"] + "?page=1", - "last": conf["id"] + "?page=3", - "first": conf["id"] + "?page=1", - } - - serializer = serializers.PaginatedCollectionSerializer(conf) - - assert serializer.data == expected - - -def test_paginated_collection_serializer_validation(): - data = { - "type": "Collection", - "id": "https://test.federation/test", - "totalItems": 5, - "actor": "http://test.actor", - "first": "https://test.federation/test?page=1", - "last": "https://test.federation/test?page=1", - "items": [], - } - - serializer = serializers.PaginatedCollectionSerializer(data=data) - - assert serializer.is_valid(raise_exception=True) is True - assert serializer.validated_data["totalItems"] == 5 - assert serializer.validated_data["id"] == data["id"] - assert serializer.validated_data["actor"] == data["actor"] - - -def test_collection_page_serializer_validation(): - base = "https://test.federation/test" - data = { - "type": "CollectionPage", - "id": base + "?page=2", - "totalItems": 5, - "actor": "https://test.actor", - "items": [], - "first": "https://test.federation/test?page=1", - "last": "https://test.federation/test?page=3", - "prev": base + "?page=1", - "next": base + "?page=3", - "partOf": base, - } - - serializer = serializers.CollectionPageSerializer(data=data) - - assert serializer.is_valid(raise_exception=True) is True - assert serializer.validated_data["totalItems"] == 5 - assert serializer.validated_data["id"] == data["id"] - assert serializer.validated_data["actor"] == data["actor"] - assert serializer.validated_data["items"] == [] - assert serializer.validated_data["prev"] == data["prev"] - assert serializer.validated_data["next"] == data["next"] - assert serializer.validated_data["partOf"] == data["partOf"] - - -def test_collection_page_serializer_can_validate_child(): - data = { - "type": "CollectionPage", - "id": "https://test.page?page=2", - "actor": "https://test.actor", - "first": "https://test.page?page=1", - "last": "https://test.page?page=3", - "partOf": "https://test.page", - "totalItems": 1, - "items": [{"in": "valid"}], - } - - serializer = serializers.CollectionPageSerializer( - data=data, context={"item_serializer": serializers.UploadSerializer} - ) - - # child are validated but not included in data if not valid - assert serializer.is_valid(raise_exception=True) is True - assert len(serializer.validated_data["items"]) == 0 - - -def test_collection_page_serializer(factories): - uploads = factories["music.Upload"].create_batch(size=5) - actor = factories["federation.Actor"](local=True) - - conf = { - "id": "https://test.federation/test", - "item_serializer": serializers.UploadSerializer, - "actor": actor, - "page": Paginator(uploads, 2).page(2), - } - expected = { - "@context": [ - "https://www.w3.org/ns/activitystreams", - "https://w3id.org/security/v1", - {}, - ], - "type": "CollectionPage", - "id": conf["id"] + "?page=2", - "actor": actor.fid, - "totalItems": len(uploads), - "partOf": conf["id"], - "prev": conf["id"] + "?page=1", - "next": conf["id"] + "?page=3", - "first": conf["id"] + "?page=1", - "last": conf["id"] + "?page=3", - "items": [ - conf["item_serializer"]( - i, context={"actor": actor, "include_ap_context": False} - ).data - for i in conf["page"].object_list - ], - } - - serializer = serializers.CollectionPageSerializer(conf) - - assert serializer.data == expected - - -def test_music_library_serializer_to_ap(factories): - library = factories["music.Library"](privacy_level="everyone") - # pending, errored and skippednot included - factories["music.Upload"](import_status="pending") - factories["music.Upload"](import_status="errored") - factories["music.Upload"](import_status="finished") - serializer = serializers.LibrarySerializer(library) - expected = { - "@context": [ - "https://www.w3.org/ns/activitystreams", - "https://w3id.org/security/v1", - {}, - ], - "audience": "https://www.w3.org/ns/activitystreams#Public", - "type": "Library", - "id": library.fid, - "name": library.name, - "summary": library.description, - "actor": library.actor.fid, - "totalItems": 0, - "current": library.fid + "?page=1", - "last": library.fid + "?page=1", - "first": library.fid + "?page=1", - "followers": library.followers_url, - } - - assert serializer.data == expected - - -def test_music_library_serializer_from_public(factories, mocker): - actor = factories["federation.Actor"]() - retrieve = mocker.patch( - "funkwhale_api.federation.utils.retrieve_ap_object", return_value=actor - ) - data = { - "@context": [ - "https://www.w3.org/ns/activitystreams", - "https://w3id.org/security/v1", - {}, - ], - "audience": "https://www.w3.org/ns/activitystreams#Public", - "name": "Hello", - "summary": "World", - "type": "Library", - "id": "https://library.id", - "followers": "https://library.id/followers", - "actor": actor.fid, - "totalItems": 12, - "first": "https://library.id?page=1", - "last": "https://library.id?page=2", - } - serializer = serializers.LibrarySerializer(data=data) - - assert serializer.is_valid(raise_exception=True) - - library = serializer.save() - - assert library.actor == actor - assert library.fid == data["id"] - assert library.uploads_count == data["totalItems"] - assert library.privacy_level == "everyone" - assert library.name == "Hello" - assert library.description == "World" - assert library.followers_url == data["followers"] - - retrieve.assert_called_once_with( - actor.fid, - queryset=actor.__class__, - serializer_class=serializers.ActorSerializer, - ) - - -def test_music_library_serializer_from_private(factories, mocker): - actor = factories["federation.Actor"]() - retrieve = mocker.patch( - "funkwhale_api.federation.utils.retrieve_ap_object", return_value=actor - ) - data = { - "@context": [ - "https://www.w3.org/ns/activitystreams", - "https://w3id.org/security/v1", - {}, - ], - "audience": "", - "name": "Hello", - "summary": "World", - "type": "Library", - "id": "https://library.id", - "followers": "https://library.id/followers", - "actor": actor.fid, - "totalItems": 12, - "first": "https://library.id?page=1", - "last": "https://library.id?page=2", - } - serializer = serializers.LibrarySerializer(data=data) - - assert serializer.is_valid(raise_exception=True) - - library = serializer.save() - - assert library.actor == actor - assert library.fid == data["id"] - assert library.uploads_count == data["totalItems"] - assert library.privacy_level == "me" - assert library.name == "Hello" - assert library.description == "World" - assert library.followers_url == data["followers"] - retrieve.assert_called_once_with( - actor.fid, - queryset=actor.__class__, - serializer_class=serializers.ActorSerializer, - ) - - -def test_activity_pub_artist_serializer_to_ap(factories): - artist = factories["music.Artist"]() - expected = { - "@context": serializers.AP_CONTEXT, - "type": "Artist", - "id": artist.fid, - "name": artist.name, - "musicbrainzId": artist.mbid, - "published": artist.creation_date.isoformat(), - } - serializer = serializers.ArtistSerializer(artist) - - assert serializer.data == expected - - -def test_activity_pub_album_serializer_to_ap(factories): - album = factories["music.Album"]() - - expected = { - "@context": serializers.AP_CONTEXT, - "type": "Album", - "id": album.fid, - "name": album.title, - "cover": { - "type": "Link", - "mediaType": "image/jpeg", - "href": utils.full_url(album.cover.url), - }, - "musicbrainzId": album.mbid, - "published": album.creation_date.isoformat(), - "released": album.release_date.isoformat(), - "artists": [ - serializers.ArtistSerializer( - album.artist, context={"include_ap_context": False} - ).data - ], - } - serializer = serializers.AlbumSerializer(album) - - assert serializer.data == expected - - -def test_activity_pub_track_serializer_to_ap(factories): - track = factories["music.Track"]( - license="cc-by-4.0", copyright="test", disc_number=3 - ) - expected = { - "@context": serializers.AP_CONTEXT, - "published": track.creation_date.isoformat(), - "type": "Track", - "musicbrainzId": track.mbid, - "id": track.fid, - "name": track.title, - "position": track.position, - "disc": track.disc_number, - "license": track.license.conf["identifiers"][0], - "copyright": "test", - "artists": [ - serializers.ArtistSerializer( - track.artist, context={"include_ap_context": False} - ).data - ], - "album": serializers.AlbumSerializer( - track.album, context={"include_ap_context": False} - ).data, - } - serializer = serializers.TrackSerializer(track) - - assert serializer.data == expected - - -def test_activity_pub_track_serializer_from_ap(factories, r_mock): - activity = factories["federation.Activity"]() - published = timezone.now() - released = timezone.now().date() - data = { - "type": "Track", - "id": "http://hello.track", - "published": published.isoformat(), - "musicbrainzId": str(uuid.uuid4()), - "name": "Black in back", - "position": 5, - "disc": 1, - "album": { - "type": "Album", - "id": "http://hello.album", - "name": "Purple album", - "musicbrainzId": str(uuid.uuid4()), - "published": published.isoformat(), - "released": released.isoformat(), - "cover": { - "type": "Link", - "href": "https://cover.image/test.png", - "mediaType": "image/png", - }, - "artists": [ - { - "type": "Artist", - "id": "http://hello.artist", - "name": "John Smith", - "musicbrainzId": str(uuid.uuid4()), - "published": published.isoformat(), - } - ], - }, - "artists": [ - { - "type": "Artist", - "id": "http://hello.trackartist", - "name": "Bob Smith", - "musicbrainzId": str(uuid.uuid4()), - "published": published.isoformat(), - } - ], - } - r_mock.get(data["album"]["cover"]["href"], body=io.BytesIO(b"coucou")) - serializer = serializers.TrackSerializer(data=data, context={"activity": activity}) - assert serializer.is_valid(raise_exception=True) - - track = serializer.save() - album = track.album - artist = track.artist - album_artist = track.album.artist - - assert track.from_activity == activity - assert track.fid == data["id"] - assert track.title == data["name"] - assert track.position == data["position"] - assert track.disc_number == data["disc"] - assert track.creation_date == published - assert str(track.mbid) == data["musicbrainzId"] - - assert album.from_activity == activity - assert album.cover.read() == b"coucou" - assert album.cover.path.endswith(".png") - assert album.title == data["album"]["name"] - assert album.fid == data["album"]["id"] - assert str(album.mbid) == data["album"]["musicbrainzId"] - assert album.creation_date == published - assert album.release_date == released - - assert artist.from_activity == activity - assert artist.name == data["artists"][0]["name"] - assert artist.fid == data["artists"][0]["id"] - assert str(artist.mbid) == data["artists"][0]["musicbrainzId"] - assert artist.creation_date == published - - assert album_artist.from_activity == activity - assert album_artist.name == data["album"]["artists"][0]["name"] - assert album_artist.fid == data["album"]["artists"][0]["id"] - assert str(album_artist.mbid) == data["album"]["artists"][0]["musicbrainzId"] - assert album_artist.creation_date == published - - -def test_activity_pub_upload_serializer_from_ap(factories, mocker, r_mock): - activity = factories["federation.Activity"]() - library = factories["music.Library"]() - - published = timezone.now() - updated = timezone.now() - released = timezone.now().date() - data = { - "@context": serializers.AP_CONTEXT, - "type": "Audio", - "id": "https://track.file", - "name": "Ignored", - "published": published.isoformat(), - "updated": updated.isoformat(), - "duration": 43, - "bitrate": 42, - "size": 66, - "url": {"href": "https://audio.file", "type": "Link", "mediaType": "audio/mp3"}, - "library": library.fid, - "track": { - "type": "Track", - "id": "http://hello.track", - "published": published.isoformat(), - "musicbrainzId": str(uuid.uuid4()), - "name": "Black in back", - "position": 5, - "album": { - "type": "Album", - "id": "http://hello.album", - "name": "Purple album", - "musicbrainzId": str(uuid.uuid4()), - "published": published.isoformat(), - "released": released.isoformat(), - "cover": { - "type": "Link", - "href": "https://cover.image/test.png", - "mediaType": "image/png", - }, - "artists": [ - { - "type": "Artist", - "id": "http://hello.artist", - "name": "John Smith", - "musicbrainzId": str(uuid.uuid4()), - "published": published.isoformat(), - } - ], - }, - "artists": [ - { - "type": "Artist", - "id": "http://hello.trackartist", - "name": "Bob Smith", - "musicbrainzId": str(uuid.uuid4()), - "published": published.isoformat(), - } - ], - }, - } - r_mock.get(data["track"]["album"]["cover"]["href"], body=io.BytesIO(b"coucou")) - - serializer = serializers.UploadSerializer(data=data, context={"activity": activity}) - assert serializer.is_valid(raise_exception=True) - track_create = mocker.spy(serializers.TrackSerializer, "create") - upload = serializer.save() - - assert upload.track.from_activity == activity - assert upload.from_activity == activity - assert track_create.call_count == 1 - assert upload.fid == data["id"] - assert upload.track.fid == data["track"]["id"] - assert upload.duration == data["duration"] - assert upload.size == data["size"] - assert upload.bitrate == data["bitrate"] - assert upload.source == data["url"]["href"] - assert upload.mimetype == data["url"]["mediaType"] - assert upload.creation_date == published - assert upload.import_status == "finished" - assert upload.modification_date == updated - - -def test_activity_pub_upload_serializer_validtes_library_actor(factories, mocker): - library = factories["music.Library"]() - usurpator = factories["federation.Actor"]() - - serializer = serializers.UploadSerializer(data={}, context={"actor": usurpator}) - - with pytest.raises(serializers.serializers.ValidationError): - serializer.validate_library(library.fid) - - -def test_activity_pub_audio_serializer_to_ap(factories): - upload = factories["music.Upload"]( - mimetype="audio/mp3", bitrate=42, duration=43, size=44 - ) - expected = { - "@context": serializers.AP_CONTEXT, - "type": "Audio", - "id": upload.fid, - "name": upload.track.full_name, - "published": upload.creation_date.isoformat(), - "updated": upload.modification_date.isoformat(), - "duration": upload.duration, - "bitrate": upload.bitrate, - "size": upload.size, - "url": { - "href": utils.full_url(upload.listen_url), - "type": "Link", - "mediaType": "audio/mp3", - }, - "library": upload.library.fid, - "track": serializers.TrackSerializer( - upload.track, context={"include_ap_context": False} - ).data, - } - - serializer = serializers.UploadSerializer(upload) - - assert serializer.data == expected - - -def test_local_actor_serializer_to_ap(factories): - expected = { - "@context": [ - "https://www.w3.org/ns/activitystreams", - "https://w3id.org/security/v1", - {}, - ], - "id": "https://test.federation/user", - "type": "Person", - "following": "https://test.federation/user/following", - "followers": "https://test.federation/user/followers", - "inbox": "https://test.federation/user/inbox", - "outbox": "https://test.federation/user/outbox", - "preferredUsername": "user", - "name": "Real User", - "summary": "Hello world", - "manuallyApprovesFollowers": False, - "publicKey": { - "id": "https://test.federation/user#main-key", - "owner": "https://test.federation/user", - "publicKeyPem": "yolo", - }, - "endpoints": {"sharedInbox": "https://test.federation/inbox"}, - } - ac = models.Actor.objects.create( - fid=expected["id"], - inbox_url=expected["inbox"], - outbox_url=expected["outbox"], - shared_inbox_url=expected["endpoints"]["sharedInbox"], - followers_url=expected["followers"], - following_url=expected["following"], - public_key=expected["publicKey"]["publicKeyPem"], - preferred_username=expected["preferredUsername"], - name=expected["name"], - domain=models.Domain.objects.create(pk="test.federation"), - summary=expected["summary"], - type="Person", - manually_approves_followers=False, - ) - user = factories["users.User"]() - user.actor = ac - user.save() - ac.refresh_from_db() - expected["icon"] = { - "type": "Image", - "mediaType": "image/jpeg", - "url": utils.full_url(user.avatar.crop["400x400"].url), - } - serializer = serializers.ActorSerializer(ac) - - assert serializer.data == expected - - -def test_activity_serializer_validate_recipients_empty(db): - s = serializers.BaseActivitySerializer() - - with pytest.raises(serializers.serializers.ValidationError): - s.validate_recipients({}) - - with pytest.raises(serializers.serializers.ValidationError): - s.validate_recipients({"to": []}) - - with pytest.raises(serializers.serializers.ValidationError): - s.validate_recipients({"cc": []}) + assert actor.fid == actor_url + assert actor.manually_approves_followers is True + assert actor.private_key is None + assert actor.public_key == payload["publicKey"]["publicKeyPem"] + assert actor.domain_id == "test.federation" diff --git a/api/tests/federation/test_views.py b/api/tests/federation/test_views.py index a7d64366b..93ce05b8e 100644 --- a/api/tests/federation/test_views.py +++ b/api/tests/federation/test_views.py @@ -93,6 +93,35 @@ def test_local_actor_inbox_post(factories, api_client, mocker, authenticated_act ) +def test_local_actor_inbox_post_receive( + factories, api_client, mocker, authenticated_actor +): + payload = { + "to": [ + "https://test.server/federation/music/libraries/956af6c9-1eb9-4117-8d17-b15e7b34afeb/followers" + ], + "type": "Create", + "actor": authenticated_actor.fid, + "object": { + "id": "https://test.server/federation/music/uploads/fe564a47-b1d4-4596-bf96-008ccf407672", + "type": "Audio", + }, + "@context": [ + "https://www.w3.org/ns/activitystreams", + "https://w3id.org/security/v1", + {}, + ], + } + user = factories["users.User"](with_actor=True) + url = reverse( + "federation:actors-inbox", + kwargs={"preferred_username": user.actor.preferred_username}, + ) + response = api_client.post(url, payload, format="json") + + assert response.status_code == 200 + + def test_shared_inbox_post(factories, api_client, mocker, authenticated_actor): patched_receive = mocker.patch("funkwhale_api.federation.activity.receive") url = reverse("federation:shared-inbox") diff --git a/api/tests/music/test_tasks.py b/api/tests/music/test_tasks.py index b7b04674f..af1a9557d 100644 --- a/api/tests/music/test_tasks.py +++ b/api/tests/music/test_tasks.py @@ -8,6 +8,7 @@ from django.core.paginator import Paginator from django.utils import timezone from funkwhale_api.federation import serializers as federation_serializers +from funkwhale_api.federation import jsonld from funkwhale_api.music import licenses, metadata, signals, tasks DATA_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -400,6 +401,7 @@ def test_federation_audio_track_to_metadata(now): published = now released = now.date() payload = { + "@context": jsonld.get_default_context(), "type": "Track", "id": "http://hello.track", "musicbrainzId": str(uuid.uuid4()), @@ -425,6 +427,11 @@ def test_federation_audio_track_to_metadata(now): "musicbrainzId": str(uuid.uuid4()), } ], + "cover": { + "type": "Link", + "href": "http://cover.test", + "mediaType": "image/png", + }, }, "artists": [ { @@ -464,6 +471,10 @@ def test_federation_audio_track_to_metadata(now): "published" ], "album_fdate": serializer.validated_data["album"]["published"], + "cover_data": { + "mimetype": serializer.validated_data["album"]["cover"]["mediaType"], + "url": serializer.validated_data["album"]["cover"]["href"], + }, } result = tasks.federation_audio_track_to_metadata(serializer.validated_data)