Merge branch 'ld-signatures' into 'master'

LD signatures and context management

See merge request jaywink/federation!175

CHANGELOG.md

@@ -1,5 +1,44 @@
 # Changelog
 
+## [0.25] - Unreleased
+
+### Added
+
+* LD signature. Relayable AP payloads signatures are checked (inbound) and signed (outbound). A missing
+  or invalid signature on inbound payloads will trigger a fetch if the sender differs from the author
+  (i.e., a relay).
+
+* The `signable` attribute has been added. It defaults to `False` and will enforce the fetching of relayed
+  payloads with a bad signature when set to `True`on a given class.
+
+* The `url` property is now set to the `id` propety as some platforms make use of it.
+
+### Changed
+
+* Re-implement dynamically generated LD contexts for outbound payloads. AP extensions are defined on a
+  per class/property basis. For classes, a `ctx` attribute is set if required. For properties, the calamus
+  field `metadata` property is used.
+
+* For inbound payload, a cached dict of all the defined AP extensions is merged with each incoming LD context.
+
+* Better handle conflicting property defaults by having `get_base_attributes` return only attributes that
+  are not empty (or bool). This helps distinguishing between `marshmallow.missing` and empty values.
+
+* JsonLD document caching now set in `activitypub/__init__.py`.
+
+* Patch outbound payloads for platform that don't handle arrays compacted to a single value and
+  `as:Public`.
+
+### Fixed
+
+* Inbound AP share retractions (undo announce) were deserialized as a `base.Retraction` class, which would
+  throw an error when accessing the missing `signable` attribute. To fix this, a `Retraction` class was added.
+
+* Because of the additions and changes above, a number of tests needed to be fixed.
+
+* HTTP signature verification now returns the signature author fid which is used as the actual
+  sender by `message_to_object`.
+
 ## [0.24.1] - 2023-03-18
 
 ### Fixed

dev-requirements.txt

@@ -23,7 +23,7 @@ arrow
 freezegun
 
 # Django support
-django>=1.8,<2.3
+django>=3.2,<4
 pytest-django
 
 # Releasing

federation/entities/activitypub/__init__.py

@@ -0,0 +1,37 @@
+import json
+from datetime import timedelta
+from pyld import jsonld
+
+try:
+    from federation.utils.django import get_redis
+    cache = get_redis() or {}
+    EXPIRATION = int(timedelta(weeks=4).total_seconds())
+except:
+    cache = {}
+
+
+# This is required to workaround a bug in pyld that has the Accept header
+# accept other content types. From what I understand, precedence handling
+# is broken
+# from https://github.com/digitalbazaar/pyld/issues/133
+# cacheing loosely inspired by https://github.com/digitalbazaar/pyld/issues/70
+def get_loader(*args, **kwargs):
+    requests_loader = jsonld.requests_document_loader(*args, **kwargs)
+
+    def loader(url, options={}):
+        key = f'ld_cache:{url}'
+        try:
+            return json.loads(cache[key])
+        except KeyError:
+            options['headers']['Accept'] = 'application/ld+json'
+            doc = requests_loader(url, options)
+            if isinstance(cache, dict):
+                cache[key] = json.dumps(doc)
+            else:
+                cache.set(key, json.dumps(doc), ex=EXPIRATION)
+            return doc
+
+    return loader
+
+
+jsonld.set_document_loader(get_loader())

federation/entities/activitypub/constants.py

@@ -1,22 +1,4 @@
 CONTEXT_ACTIVITYSTREAMS = "https://www.w3.org/ns/activitystreams"
-CONTEXT_DIASPORA = {"diaspora": "https://diasporafoundation.org/ns/"}
-CONTEXT_HASHTAG = {"Hashtag": "as:Hashtag"}
-CONTEXT_LD_SIGNATURES = "https://w3id.org/security/v1"
-CONTEXT_MANUALLY_APPROVES_FOLLOWERS = {"manuallyApprovesFollowers": "as:manuallyApprovesFollowers"}
-CONTEXT_PYTHON_FEDERATION = {"pyfed": "https://docs.jasonrobinson.me/ns/python-federation#"}
-CONTEXT_SENSITIVE = {"sensitive": "as:sensitive"}
-
-CONTEXTS_DEFAULT = [
-    CONTEXT_ACTIVITYSTREAMS,
-    CONTEXT_PYTHON_FEDERATION,
-]
-
-CONTEXT = [CONTEXT_ACTIVITYSTREAMS, CONTEXT_LD_SIGNATURES]
-CONTEXT_DICT = {}
-for ctx in [CONTEXT_DIASPORA, CONTEXT_HASHTAG, CONTEXT_MANUALLY_APPROVES_FOLLOWERS, CONTEXT_SENSITIVE, CONTEXT_PYTHON_FEDERATION]:
-    CONTEXT_DICT.update(ctx)
-CONTEXT_SETS = {prop: {'@id': f'as:{prop}', '@container': '@set'} for prop in ['to', 'cc', 'tag', 'attachment']}
-CONTEXT_DICT.update(CONTEXT_SETS)
-CONTEXT.append(CONTEXT_DICT)
+CONTEXT_SECURITY = "https://w3id.org/security/v1"
 
 NAMESPACE_PUBLIC = "https://www.w3.org/ns/activitystreams#Public"

federation/entities/activitypub/ldcontext.py

@@ -0,0 +1,155 @@
+import copy
+import json
+
+from marshmallow import missing
+from pyld import jsonld
+
+from federation.entities.activitypub.constants import CONTEXT_ACTIVITYSTREAMS, CONTEXT_SECURITY, NAMESPACE_PUBLIC
+
+
+# Extract context information from the metadata parameter defined for fields
+# that are not part of the official AP spec. Use the same extended context for
+# inbound payload. For outbound payload, build a context with only the required
+# extensions
+class LdContextManager:
+    _named = [CONTEXT_ACTIVITYSTREAMS, CONTEXT_SECURITY]
+    _extensions = {}
+    _merged = []
+    _models = []
+
+    def __init__(self, models):
+        self._models = models
+        for klass in models:
+            self._extensions[klass] = {}
+            ctx = getattr(klass, 'ctx', [])
+            if ctx:
+                self._extensions[klass].update({klass.__name__: ctx})
+            for name, value in klass.schema().declared_fields.items():
+                ctx = value.metadata.get('ctx') or []
+                if ctx:
+                    self._extensions[klass].update({name: ctx})
+        merged = {}
+        for field in self._extensions.values():
+            for ctx in field.values():
+                self._add_extensions(ctx, self._named, merged)
+        self._merged = copy.copy(self._named)
+        self._merged.append(merged)
+
+    def _add_extensions(self, field, named, extensions):
+        for item in field:
+            if isinstance(item, str) and item not in named:
+                named.append(item)
+            elif isinstance(item, dict):
+                extensions.update(item)
+
+    def _get_fields(self, obj):
+        for klass in self._extensions.keys():
+            if issubclass(type(obj), klass):
+                return self._extensions[klass]
+        return {}
+
+    def compact(self, obj):
+        payload = jsonld.compact(obj.dump(), self.build_context(obj))
+        patched = copy.copy(payload)
+
+        # This is for platforms that don't handle the single element array
+        # compaction to a single value and https://www.w3.org/ns/activitystreams#Public
+        # being compacted to as:Public
+        def patch_payload(payload, patched):
+            for field in ('attachment', 'cc', 'tag', 'to'):
+                value = payload.get(field)
+                if not value:
+                    continue
+                if not isinstance(value, list):
+                    value = [value]
+                patched[field] = value
+                if field in ('cc', 'to'):
+                    try:
+                        idx = value.index('as:Public')
+                        patched[field][idx] = value[idx].replace('as:Public', NAMESPACE_PUBLIC)
+                    except ValueError:
+                        pass
+            if isinstance(payload.get('object'), dict):
+                patch_payload(payload['object'], patched['object'])
+
+        patch_payload(payload, patched)
+        return patched
+
+    def build_context(self, obj):
+        from federation.entities.activitypub.models import Object, Link
+
+        final = [CONTEXT_ACTIVITYSTREAMS]
+        extensions = {}
+
+        def walk_object(obj):
+            if type(obj) in self._extensions.keys():
+                self._add_extensions(self._extensions[type(obj)].get(type(obj).__name__, []), final, extensions)
+            to_add = self._get_fields(obj)
+            for field in type(obj).schema().declared_fields.keys():
+                field_value = getattr(obj, field)
+                if field in to_add.keys():
+                    if field_value is not missing or obj.signable and field == 'signature':
+                        self._add_extensions(to_add[field], final, extensions)
+                if not isinstance(field_value, list):
+                    field_value = [field_value]
+                for value in field_value:
+                    if issubclass(type(value), (Object, Link)):
+                        walk_object(value)
+
+        walk_object(obj)
+        if extensions:
+            final.append(extensions)
+        # compact the array if len == 1 to minimize test changes
+        return final if len(final) > 1 else final[0]
+
+    def merge_context(self, ctx):
+        # One platform sends a single string context
+        if isinstance(ctx, str):
+            ctx = [ctx]
+
+        # add a # at the end of the python-federation string
+        # for legacy socialhome payloads
+        s = json.dumps(ctx)
+        if 'python-federation"' in s:
+            ctx = json.loads(s.replace('python-federation', 'python-federation#', 1))
+
+        # some platforms have http://joinmastodon.com/ns in @context. This
+        # is not a json-ld document.
+        try:
+            ctx.pop(ctx.index('http://joinmastodon.org/ns'))
+        except ValueError:
+            pass
+
+        # remove @language in context since this directive is not
+        # processed by calamus. Pleroma adds a useless @language: 'und'
+        # which is discouraged in best practices and in some cases makes
+        # calamus return dict where str is expected.
+        # see https://www.rfc-editor.org/rfc/rfc5646, page 56
+        idx = []
+        for i, v in enumerate(ctx):
+            if isinstance(v, dict):
+                v.pop('@language', None)
+                if len(v) == 0:
+                    idx.insert(0, i)
+        for i in idx:
+            ctx.pop(i)
+
+        # Merge all defined AP extensions to the inbound context
+        uris = []
+        defs = {}
+        # Merge original context dicts in one dict
+        for item in ctx:
+            if isinstance(item, str):
+                uris.append(item)
+            else:
+                defs.update(item)
+
+        for item in self._merged:
+            if isinstance(item, str) and item not in uris:
+                uris.append(item)
+            elif isinstance(item, dict):
+                defs.update(item)
+
+        final = copy.copy(uris)
+        final.append(defs)
+        return final

federation/entities/activitypub/ldsigning.py

@@ -0,0 +1,102 @@
+import datetime
+import logging
+import math
+import re
+from base64 import b64encode, b64decode
+from copy import copy
+from funcy import omit
+from pyld import jsonld
+
+from Crypto.Hash import SHA256
+from Crypto.PublicKey.RSA import import_key
+from Crypto.Signature import pkcs1_15
+
+from federation.utils.activitypub import retrieve_and_parse_document
+
+
+logger = logging.getLogger("federation")
+
+
+def create_ld_signature(obj, author):
+    # Use models.Signature? Maybe overkill...
+    sig = {
+        'created': datetime.datetime.now(tz=datetime.timezone.utc).isoformat(timespec='seconds'),
+        'creator': f'{author.id}#main-key',
+        '@context': 'https://w3id.org/security/v1'
+    }
+
+    try:
+        private_key = import_key(author.private_key)
+    except (ValueError, TypeError) as exc:
+        logger.warning('ld_signature - %s', exc)
+        return None
+    signer = pkcs1_15.new(private_key)
+
+    sig_digest = hash(sig)
+    obj_digest = hash(obj)
+    digest = (sig_digest + obj_digest).encode('utf-8')
+
+    signature = signer.sign(SHA256.new(digest))
+    sig.update({'type': 'RsaSignature2017', 'signatureValue': b64encode(signature).decode()})
+    sig.pop('@context')
+
+    obj.update({'signature': sig})
+
+
+def verify_ld_signature(payload):
+    """
+    Verify inbound payload LD signature
+    """
+    signature = copy(payload.get('signature', None))
+    if not signature:
+        logger.warning('ld_signature - No signature in %s', payload.get("id", "the payload"))
+        return None
+
+    # retrieve the author's public key
+    profile = retrieve_and_parse_document(signature.get('creator'))
+    if not profile:
+
+        logger.warning('ld_signature - Failed to retrieve profile for %s', signature.get("creator"))
+        return None
+    try:
+        pkey = import_key(profile.public_key)
+    except ValueError as exc:
+        logger.warning('ld_signature - %s', exc)
+        return None
+    verifier = pkcs1_15.new(pkey)
+
+    # Compute digests and verify signature
+    sig = omit(signature, ('type', 'signatureValue'))
+    sig.update({'@context': 'https://w3id.org/security/v1'})
+    sig_digest = hash(sig)
+    obj = omit(payload, 'signature')
+    obj_digest = hash(obj)
+    digest = (sig_digest + obj_digest).encode('utf-8')
+
+    sig_value = b64decode(signature.get('signatureValue'))
+    try:
+        verifier.verify(SHA256.new(digest), sig_value)
+        logger.debug('ld_signature - %s has a valid signature', payload.get("id"))
+        return profile.id
+    except ValueError:
+        logger.warning('ld_signature - Invalid signature for %s', payload.get("id"))
+        return None
+
+
+def hash(obj):
+    nquads = NormalizedDoubles().normalize(obj, options={'format': 'application/nquads', 'algorithm': 'URDNA2015'})
+    return SHA256.new(nquads.encode('utf-8')).hexdigest()
+
+
+# We need this to ensure the digests are identical.
+class NormalizedDoubles(jsonld.JsonLdProcessor):
+    def _object_to_rdf(self, item, issuer, triples, rdfDirection):
+        value = item['@value'] if jsonld._is_value(item) else None
+        # The ruby rdf_normalize library turns floats with a zero fraction into integers.
+        if isinstance(value, float) and value == math.floor(value):
+            item['@value'] = math.floor(value)
+        obj = super()._object_to_rdf(item, issuer, triples, rdfDirection)
+        # This is to address https://github.com/digitalbazaar/pyld/issues/175
+        if obj.get('datatype') == jsonld.XSD_DOUBLE:
+            obj['value'] = re.sub(r'(\d)0*E\+?(-)?0*(\d)', r'\1E\2\3', obj['value'])
+        return obj

federation/entities/activitypub/mappers.py

@@ -84,6 +84,6 @@ def message_to_objects(
     Takes in a message extracted by a protocol and maps it to entities.
     """
     # We only really expect one element here for ActivityPub.
-    return element_to_objects(message)
+    return element_to_objects(message, sender)
 
 

federation/entities/activitypub/models.py

@@ -1,7 +1,7 @@
+import copy
 import json
 import logging
 import uuid
-from copy import copy
 from datetime import timedelta
 from typing import List, Dict, Union
 from urllib.parse import urlparse
@@ -10,52 +10,26 @@ import bleach
 from calamus import fields
 from calamus.schema import JsonLDAnnotation, JsonLDSchema, JsonLDSchemaOpts
 from calamus.utils import normalize_value
+from cryptography.exceptions import InvalidSignature
 from marshmallow import exceptions, pre_load, post_load, post_dump
 from marshmallow.fields import Integer
 from marshmallow.utils import EXCLUDE, missing
 from pyld import jsonld
 
 import federation.entities.base as base
-from federation.entities.activitypub.constants import CONTEXT, CONTEXT_SETS, NAMESPACE_PUBLIC
+from federation.entities.activitypub.constants import CONTEXT_ACTIVITYSTREAMS, CONTEXT_SECURITY, NAMESPACE_PUBLIC
+from federation.entities.activitypub.ldcontext import LdContextManager
+from federation.entities.activitypub.ldsigning import create_ld_signature, verify_ld_signature
 from federation.entities.mixins import BaseEntity, RawContentMixin
 from federation.entities.utils import get_base_attributes, get_profile
 from federation.outbound import handle_send
 from federation.types import UserType, ReceiverVariant
 from federation.utils.activitypub import retrieve_and_parse_document, retrieve_and_parse_profile, \
     get_profile_id_from_webfinger
-from federation.utils.django import get_configuration, get_redis
 from federation.utils.text import with_slash, validate_handle
 
 logger = logging.getLogger("federation")
 
-cache = get_redis() or {}
-EXPIRATION = int(timedelta(weeks=2).total_seconds())
-    
-# This is required to workaround a bug in pyld that has the Accept header
-# accept other content types. From what I understand, precedence handling
-# is broken
-# from https://github.com/digitalbazaar/pyld/issues/133
-# cacheing loosely inspired by https://github.com/digitalbazaar/pyld/issues/70
-def get_loader(*args, **kwargs):
-    requests_loader = jsonld.requests_document_loader(*args, **kwargs)
-    
-    def loader(url, options={}):
-        key = f'ld_cache:{url}'
-        try:
-            return json.loads(cache[key])
-        except KeyError:
-            options['headers']['Accept'] = 'application/ld+json'
-            doc = requests_loader(url, options)
-            if isinstance(cache, dict):
-                cache[url] = json.dumps(doc)
-            else:
-                cache.set(f'ld_cache:{url}', json.dumps(doc), ex=EXPIRATION)
-            return doc
-    
-    return loader
-
-jsonld.set_document_loader(get_loader())
-
 
 def get_profile_or_entity(fid):
     obj = get_profile(fid=fid)
@@ -135,7 +109,7 @@ class NormalizedList(fields.List):
 
 # Don't want expanded IRIs to be exposed as dict keys
 class CompactedDict(fields.Dict):
-    ctx = ["https://www.w3.org/ns/activitystreams", "https://w3id.org/security/v1"]
+    ctx = [CONTEXT_ACTIVITYSTREAMS, CONTEXT_SECURITY]
 
     # may or may not be needed
     def _serialize(self, value, attr, obj, **kwargs):
@@ -160,12 +134,12 @@ class CompactedDict(fields.Dict):
 # calamus sets a XMLSchema#integer type, but different definitions
 # maybe used, hence the flavor property
 # TODO: handle non negative types
-class Integer(fields._JsonLDField, Integer):
+class MixedInteger(fields._JsonLDField, Integer):
     flavor = None  # add fields.IRIReference type hint 
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
-        self.flavor = kwargs.get('flavor')
+        self.flavor = self.metadata.get('flavor')
 
     def _serialize(self, value, attr, obj, **kwargs):
         value = super()._serialize(value, attr, obj, **kwargs)
@@ -254,7 +228,6 @@ OBJECTS = [
         'VideoSchema'
 ]
 
-
 def set_public(entity):
     for attr in [entity.to, entity.cc]:
         if isinstance(attr, list):
@@ -262,38 +235,31 @@ def set_public(entity):
         elif attr == NAMESPACE_PUBLIC: entity.public = True
 
 
-def add_props_to_attrs(obj, props):
-    return obj.__dict__
-    attrs = copy(obj.__dict__)
-    for prop in props:
-        attrs.update({prop: getattr(obj, prop, None)})
-        attrs.pop('_'+prop, None)
-    attrs.update({'schema': True})
-    return attrs
-
-
 class Object(BaseEntity, metaclass=JsonLDAnnotation):
     atom_url = fields.String(ostatus.atomUri)
-    also_known_as = IRI(as2.alsoKnownAs)
+    also_known_as = IRI(as2.alsoKnownAs,
+                        metadata={'ctx':[{ 'alsoKnownAs':{'@id':'as:alsoKnownAs','@type':'@id'}}]})
     icon = MixedField(as2.icon, nested='ImageSchema')
-    image = MixedField(as2.image, nested='ImageSchema', default='')
+    image = MixedField(as2.image, nested='ImageSchema')
     tag_objects = MixedField(as2.tag, nested=['HashtagSchema','MentionSchema','PropertyValueSchema','EmojiSchema'], many=True)
-    attachment = fields.Nested(as2.attachment, nested=['ImageSchema', 'AudioSchema', 'DocumentSchema','PropertyValueSchema','IdentityProofSchema'], many=True)
+    attachment = fields.Nested(as2.attachment, nested=['ImageSchema', 'AudioSchema', 'DocumentSchema','PropertyValueSchema','IdentityProofSchema'],
+                               many=True, default=[])
     content_map = LanguageMap(as2.content)  # language maps are not implemented in calamus
-    context = IRI(as2.context)
-    guid = fields.String(diaspora.guid, default='')
-    handle = fields.String(diaspora.handle, default='')
+    context = fields.RawJsonLD(as2.context)
     name = fields.String(as2.name, default='')
     generator = MixedField(as2.generator, nested=['ApplicationSchema','ServiceSchema'])
     created_at = fields.DateTime(as2.published, add_value_types=True)
     replies = MixedField(as2.replies, nested=['CollectionSchema','OrderedCollectionSchema'])
-    signature = MixedField(sec.signature, nested = 'SignatureSchema')
+    signature = MixedField(sec.signature, nested = 'SignatureSchema',
+                           metadata={'ctx': [CONTEXT_SECURITY,
+                                             {'RsaSignature2017':'sec:RsaSignature2017'}]})
     start_time = fields.DateTime(as2.startTime, add_value_types=True)
     updated = fields.DateTime(as2.updated, add_value_types=True)
-    to = fields.List(as2.to, cls_or_instance=fields.String(as2.to))
-    cc = fields.List(as2.cc, cls_or_instance=fields.String(as2.cc))
+    to = fields.List(as2.to, cls_or_instance=IRI(as2.to), default=[])
+    cc = fields.List(as2.cc, cls_or_instance=IRI(as2.cc), default=[])
     media_type = fields.String(as2.mediaType)
     source = CompactedDict(as2.source)
+    signable = False
 
     # The following properties are defined by some platforms, but are not implemented yet
     #audience
@@ -306,24 +272,53 @@ class Object(BaseEntity, metaclass=JsonLDAnnotation):
 
     def to_as2(self):
         obj = self.activity if isinstance(self.activity, Activity) else self
-        return jsonld.compact(obj.dump(), CONTEXT)
+        return context_manager.compact(obj)
+
+    def sign_as2(self, sender=None):
+        obj = self.to_as2()
+        if self.signable and sender: create_ld_signature(obj, sender)
+        return obj
 
     @classmethod
     def from_base(cls, entity):
         # noinspection PyArgumentList
         return cls(**get_base_attributes(entity))
 
+    # This is to ensure the original payload is relayed in order
+    # to preserve the validity of the LD signature.
+    # Note: the function name comes from the Diaspora logic and does
+    # not reflect what is actually happening here. For AP, the parent
+    # user's key is used for the http signature.
+    def sign_with_parent(self, private_key):
+        self.outbound_doc = getattr(self, '_source_object', None)
+
     # Before validation, assign None to fields that are set to marshmallow.missing
     # Setting missing fields to marshmallow.missing starts with calamus 0.4.1
     # TODO: rework validation
     def validate(self, direction='inbound'):
         if direction == 'inbound':
+            # ensure marshmallow.missing is not sent to the client app
             for attr in type(self).schema().load_fields.keys():
                 if getattr(self, attr) is missing:
                     setattr(self, attr, None)
 
         super().validate(direction)
 
+    def _validate_signatures(self):
+        # Objects extracted from collections don't have a source object.
+        # To avoid infinite recursion, only verify a profile signature
+        # if it was sent, not retrieved.
+        if not self._source_object or (not self._sender and isinstance(self, Person)):
+            return
+        # Always verify inbound LD signature, for monitoring purposes
+        actor = verify_ld_signature(self._source_object)
+        if not self._sender:
+            return
+        if self.signable and self._sender not in (self.id, getattr(self, 'actor_id', None)):
+            # Relayed payload
+            if not actor:
+                raise InvalidSignature('no or invalid signature for %s, a relayed payload', self.id)
+
     def to_string(self):
         # noinspection PyUnresolvedReferences
         return str(self.to_as2())
@@ -334,86 +329,8 @@ class Object(BaseEntity, metaclass=JsonLDAnnotation):
         @pre_load
         def patch_context(self, data, **kwargs):
             if not data.get('@context'): return data
-            ctx = copy(data['@context'])
-
-            # One platform send a single string context
-            if isinstance(ctx, str): ctx = [ctx]
-
-            # add a # at the end of the python-federation string
-            # for socialhome payloads
-            s = json.dumps(ctx)
-            if 'python-federation"' in s:
-                ctx = json.loads(s.replace('python-federation', 'python-federation#', 1))
-
-            # some paltforms have http://joinmastodon.com/ns in @context. This
-            # is not a json-ld document.
-            try:
-                ctx.pop(ctx.index('http://joinmastodon.org/ns'))
-            except:
-                pass
-
-            # remove @language in context since this directive is not
-            # processed by calamus. Pleroma adds a useless @language: 'und'
-            # which is discouraged in best practices and in some cases makes 
-            # calamus return dict where str is expected.
-            # see https://www.rfc-editor.org/rfc/rfc5646, page 56
-            idx = []
-            for i,v in enumerate(ctx):
-                if isinstance(v, dict): 
-                    v.pop('@language',None)
-                    if len(v) == 0: idx.insert(0, i)
-            for i in idx: ctx.pop(i)
-
-            # AP activities may be signed, but most platforms don't
-            # define RsaSignature2017. add it to the context
-            # hubzilla doesn't define the discoverable property in its context
-            # include all Mastodon extensions for platforms that only define http://joinmastodon.org/ns in their context
-            may_add = {'signature': ['https://w3id.org/security/v1', {'sec':'https://w3id.org/security#','RsaSignature2017':'sec:RsaSignature2017'}],
-                    'publicKey': ['https://w3id.org/security/v1'],
-                    'discoverable': [{'toot':'http://joinmastodon.org/ns#','discoverable': 'toot:discoverable'}], #for hubzilla
-                    'suspended': [{'toot':'http://joinmastodon.org/ns#','suspended': 'toot:suspended'}],
-                    'copiedTo': [{'toot':'http://joinmastodon.org/ns#','copiedTo': 'toot:copiedTo'}], #for hubzilla
-                    'featured': [{'toot':'http://joinmastodon.org/ns#','featured': 'toot:featured'}], #for litepub and pleroma
-                    'featuredTags': [{'toot':'http://joinmastodon.org/ns#','featuredTags': 'toot:featuredTags'}],
-                    'focalPoint': [{'toot':'http://joinmastodon.org/ns#',
-                                    'focalPoint': {'@id':'toot:focalPoint','@container':'@list'},
-                                    }],
-                    'tag': [{'Hashtag': 'as:Hashtag', #for epicyon
-                             'toot':'http://joinmastodon.org/ns#',
-                             'Emoji':'toot:Emoji'}],
-                    'attachment': [{'schema': 'http://schema.org#', 'PropertyValue': 'schema:PropertyValue', # for owncast
-                                    'toot':'http://joinmastodon.org/ns#','blurHash': 'toot:blurHash',
-                                    'IdentityProof': 'toot:IdentityProof'}]
-                    }
-
-            to_add = [val for key,val in may_add.items() if data.get(key)]
-            if to_add:
-                idx = [i for i,v in enumerate(ctx) if isinstance(v, dict)]
-                if idx:
-                    upd = ctx[idx[0]]
-                    # merge context dicts
-                    if len(idx) > 1:
-                        idx.reverse()
-                        for i in idx[:-1]:
-                            upd.update(ctx[i])
-                            ctx.pop(i)
-                else:
-                    upd = {}
-
-                for add in to_add:
-                    for val in add:
-                        if isinstance(val, str) and val not in ctx:
-                            try:
-                                ctx.append(val)
-                            except AttributeError:
-                                ctx = [ctx, val]
-                        if isinstance(val, dict):
-                            upd.update(val)
-                if not idx and upd: ctx.append(upd)
-            
-            # for to and cc fields to be processed as strings
-            ctx.append(CONTEXT_SETS)
-            data['@context'] = ctx
+            ctx = copy.copy(data['@context'])
+            data['@context'] = context_manager.merge_context(ctx)
             return data
 
         # A node without an id isn't true json-ld, but many payloads have
@@ -443,7 +360,7 @@ class Collection(Object, base.Collection):
     first = MixedField(as2.first, nested=['CollectionPageSchema', 'OrderedCollectionPageSchema'])
     current = IRI(as2.current)
     last = IRI(as2.last)
-    total_items = Integer(as2.totalItems, flavor=xsd.nonNegativeInteger, add_value_types=True)
+    total_items = MixedInteger(as2.totalItems, metafdata={'flavor':xsd.nonNegativeInteger}, add_value_types=True)
 
     class Meta:
         rdf_type = as2.Collection
@@ -466,7 +383,7 @@ class CollectionPage(Collection):
 
 
 class OrderedCollectionPage(OrderedCollection, CollectionPage):
-    start_index = Integer(as2.startIndex, flavor=xsd.nonNegativeInteger, add_value_types=True)
+    start_index = MixedInteger(as2.startIndex, metadata={'flavor':xsd.nonNegativeInteger}, add_value_types=True)
 
     class Meta:
         rdf_type = as2.OrderedCollectionPage
@@ -476,10 +393,12 @@ class OrderedCollectionPage(OrderedCollection, CollectionPage):
 # AP defines [Ii]mage and [Aa]udio objects/properties, but only a Video object
 # seen with Peertube payloads only so far
 class Document(Object):
-    inline = fields.Boolean(pyfed.inlineImage, default=False)
-    height = Integer(as2.height, default=0, flavor=xsd.nonNegativeInteger, add_value_types=True)
-    width = Integer(as2.width, default=0, flavor=xsd.nonNegativeInteger, add_value_types=True)
-    blurhash = fields.String(toot.blurhash)
+    inline = fields.Boolean(pyfed.inlineImage, default=False,
+                            metadata={'ctx':[{'pyfed':str(pyfed)}]})
+    height = MixedInteger(as2.height, default=0, metadata={'flavor':xsd.nonNegativeInteger}, add_value_types=True)
+    width = MixedInteger(as2.width, default=0, metadata={'flavor':xsd.nonNegativeInteger}, add_value_types=True)
+    blurhash = fields.String(toot.blurHash,
+                             metadata={'ctx':[{'toot':str(toot),'blurHash':'toot:blurHash'}]})
     url = MixedField(as2.url, nested='LinkSchema')
 
     def to_base(self):
@@ -526,10 +445,10 @@ class Link(metaclass=JsonLDAnnotation):
     media_type = fields.String(as2.mediaType)
     name = fields.String(as2.name)
     href_lang = fields.String(as2.hrefLang)
-    height = Integer(as2.height, flavor=xsd.nonNegativeInteger, add_value_types=True)
-    width = Integer(as2.width, flavor=xsd.nonNegativeInteger, add_value_types=True)
-    fps = Integer(pt.fps, flavor=schema.Number, add_value_types=True)
-    size = Integer(pt.size, flavor=schema.Number, add_value_types=True)
+    height = MixedInteger(as2.height, metadata={'flavor':xsd.nonNegativeInteger}, add_value_types=True)
+    width = MixedInteger(as2.width, metadata={'flavor':xsd.nonNegativeInteger}, add_value_types=True)
+    fps = MixedInteger(pt.fps, metadata={'flavor':schema.Number}, add_value_types=True)
+    size = MixedInteger(pt.size, metadata={'flavor':schema.Number}, add_value_types=True)
     tag = MixedField(as2.tag, nested=['InfohashSchema', 'LinkSchema'], many=True)
     # Not implemented yet
     #preview : variable type?
@@ -553,6 +472,8 @@ class Link(metaclass=JsonLDAnnotation):
 
 
 class Hashtag(Link):
+    ctx = [{'Hashtag': 'as:Hashtag'}]
+    id = fields.Id() # Hubzilla uses id instead of href
 
     class Meta:
         rdf_type = as2.Hashtag
@@ -565,7 +486,9 @@ class Mention(Link):
 
 
 class PropertyValue(Object):
-    value = fields.String(schema.value)
+    value = fields.String(schema.value,
+                          metadata={'ctx':[{'schema':str(schema),'value':'schema:value'}]})
+    ctx = [{'schema':str(schema),'PropertyValue':'schema:PropertyValue'}]
 
     class Meta:
         rdf_type = schema.PropertyValue
@@ -574,12 +497,14 @@ class PropertyValue(Object):
 class IdentityProof(Object):
     signature_value = fields.String(sec.signatureValue)
     signing_algorithm = fields.String(sec.signingAlgorithm)
+    ctx = [CONTEXT_SECURITY]
 
     class Meta:
         rdf_type = toot.IdentityProof
 
 
 class Emoji(Object):
+    ctx = [{'toot':'http://joinmastodon.org/ns#','Emoji':'toot:Emoji'}]
 
     class Meta:
         rdf_type = toot.Emoji
@@ -591,26 +516,40 @@ class Person(Object, base.Profile):
     outbox = IRI(as2.outbox)
     following = IRI(as2.following)
     followers = IRI(as2.followers)
+    guid = fields.String(diaspora.guid, metadata={'ctx':[{'diaspora':str(diaspora)}]})
+    handle = fields.String(diaspora.handle, metadata={'ctx':[{'diaspora':str(diaspora)}]})
     username = fields.String(as2.preferredUsername)
     endpoints = CompactedDict(as2.endpoints)
     shared_inbox = IRI(as2.sharedInbox) # misskey adds this
     url = MixedField(as2.url, nested='LinkSchema')
     playlists = IRI(pt.playlists)
-    featured = IRI(toot.featured)
-    featuredTags = IRI(toot.featuredTags)
-    manuallyApprovesFollowers = fields.Boolean(as2.manuallyApprovesFollowers, default=False)
-    discoverable = fields.Boolean(toot.discoverable)
+    featured = IRI(toot.featured,
+                   metadata={'ctx':[{'toot':str(toot),
+                                     'featured': {'@id':'toot:featured','@type':'@id'}}]})
+    featured_tags = IRI(toot.featuredTags,
+                        metadata={'ctx':[{'toot':str(toot),
+                                          'featuredTags': {'@id':'toot:featuredTags','@type':'@id'}}]})
+    manually_approves_followers = fields.Boolean(as2.manuallyApprovesFollowers, default=False,
+                                                 metadata={'ctx':[{'manuallyApprovesFollowers':'as:manuallyApprovesFollowers'}]})
+    discoverable = fields.Boolean(toot.discoverable,
+                                  metadata={'ctx':[{'toot':str(toot),
+                                                    'discoverable': 'toot:discoverable'}]})
     devices = IRI(toot.devices)
-    public_key_dict = CompactedDict(sec.publicKey)
-    raw_content = fields.String(as2.summary, default="")
+    public_key_dict = CompactedDict(sec.publicKey,
+                                    metadata={'ctx':[CONTEXT_SECURITY]})
+    raw_content = fields.String(as2.summary, default='')
     has_address = MixedField(vcard.hasAddress, nested='HomeSchema')
     has_instant_message = fields.List(vcard.hasInstantMessage, cls_or_instance=fields.String)
     address = fields.String(vcard.Address)
     is_cat = fields.Boolean(misskey.isCat)
-    moved_to = IRI(as2.movedTo)
-    copied_to = IRI(toot.copiedTo)
+    moved_to = IRI(as2.movedTo,
+                   metadata={'ctx':[{'movedTo':{'@id':'as:movedTo','@type':'@id'}}]})
+    copied_to = IRI(as2.copiedTo,
+                    metadata={'ctx':[{'copiedTo':{'@id':'as:copiedTo','@type':'@id'}}]})
     capabilities = CompactedDict(litepub.capabilities)
-    suspended = fields.Boolean(toot.suspended)
+    suspended = fields.Boolean(toot.suspended,
+                               metadata={'ctx':[{'toot':str(toot),
+                                                 'suspended': 'toot:suspended'}]})
     public = True
     _cached_inboxes = None
     _cached_public_key = None
@@ -641,7 +580,7 @@ class Person(Object, base.Profile):
             if get_profile_id_from_webfinger(finger) == self.id:
                 self.finger = finger
         # multi-protocol platform
-        if self.finger and self.guid and not self.handle:
+        if self.finger and self.guid is not missing and self.handle is missing:
             self.handle = self.finger
 
     def to_as2(self):
@@ -697,10 +636,11 @@ class Person(Object, base.Profile):
     def image_urls(self):
         if getattr(self, 'icon', None):
             icon = self.icon if not isinstance(self.icon, list) else self.icon[0]
+            url = icon if isinstance(icon, str) else icon.url
             self._cached_image_urls = {
-                'small': icon.url,
-                'medium': icon.url,
-                'large': icon.url
+                'small': url,
+                'medium': url,
+                'large': url
                 }
         return self._cached_image_urls
 
@@ -756,14 +696,22 @@ class Note(Object, RawContentMixin):
     id = fields.Id()
     actor_id = IRI(as2.attributedTo)
     target_id = IRI(as2.inReplyTo, default=None)
-    conversation = fields.RawJsonLD(ostatus.conversation)
+    conversation = fields.RawJsonLD(ostatus.conversation,
+                                    metadata={'ctx':[{'ostatus':str(ostatus),
+                                                      'conversation':'ostatus:conversation'}]})
     entity_type = 'Post'
-    in_reply_to_atom_uri = IRI(ostatus.inReplyToAtomUri)
-    sensitive = fields.Boolean(as2.sensitive, default=False)
+    guid = fields.String(diaspora.guid, metadata={'ctx':[{'diaspora':str(diaspora)}]})
+    in_reply_to_atom_uri = IRI(ostatus.inReplyToAtomUri,
+                               metadata={'ctx':[{'ostatus':str(ostatus),
+                               'inReplyToAtomUri':'ostatus:inReplyToAtomUri'}]})
+    sensitive = fields.Boolean(as2.sensitive, default=False,
+                               metadata={'ctx':[{'sensitive':'as:sensitive'}]})
     summary = fields.String(as2.summary)
     url = IRI(as2.url)
+
     _cached_raw_content = ''
     _cached_children = []
+    signable = True
 
     def __init__(self, *args, **kwargs):
         self.tag_objects = [] # mutable objects...
@@ -772,6 +720,7 @@ class Note(Object, RawContentMixin):
 
     def to_as2(self):
         self.sensitive = 'nsfw' in self.tags
+        self.url = self.id
 
         edited = False
         if hasattr(self, 'times'):
@@ -798,7 +747,8 @@ class Note(Object, RawContentMixin):
 
     def to_base(self):
         kwargs = get_base_attributes(self, keep=(
-            '_mentions', '_media_type', '_rendered_content', '_cached_children', '_cached_raw_content'))
+            '_mentions', '_media_type', '_rendered_content', '_source_object',
+            '_cached_children', '_cached_raw_content'))
         entity = Comment(**kwargs) if getattr(self, 'target_id') else Post(**kwargs)
         # Plume (and maybe other platforms) send the attrbutedTo field as an array
         if isinstance(entity.actor_id, list): entity.actor_id = entity.actor_id[0]
@@ -835,7 +785,13 @@ class Note(Object, RawContentMixin):
             # Skip when markdown
             return
 
-        hrefs = [tag.href.lower() for tag in self.tag_objects if isinstance(tag, Hashtag)]
+        hrefs = []
+        for tag in self.tag_objects:
+            if isinstance(tag, Hashtag):
+                if tag.href is not missing:
+                    hrefs.append(tag.href.lower())
+                elif tag.id is not missing:
+                    hrefs.append(tag.id.lower())
         # noinspection PyUnusedLocal
         def remove_tag_links(attrs, new=False):
             # Hashtag object hrefs
@@ -874,6 +830,7 @@ class Note(Object, RawContentMixin):
         Populate tags to the object.tag list.
         """
         try:
+            from federation.utils.django import get_configuration
             config = get_configuration()
         except ImportError:
             tags_path = None
@@ -977,13 +934,11 @@ class Note(Object, RawContentMixin):
 
     class Meta:
         rdf_type = as2.Note
-        exclude = ('handle',)
 
 
 class Post(Note, base.Post):
     class Meta:
         rdf_type = as2.Note
-        exclude = ('handle',)
 
 
 class Comment(Note, base.Comment):
@@ -997,7 +952,6 @@ class Comment(Note, base.Comment):
 
     class Meta:
         rdf_type = as2.Note
-        exclude = ('handle',)
 
 
 class Article(Note):
@@ -1015,6 +969,7 @@ class Video(Document, base.Video):
     id = fields.Id()
     actor_id = MixedField(as2.attributedTo, nested=['PersonSchema', 'GroupSchema'], many=True)
     url = MixedField(as2.url, nested='LinkSchema')
+    signable = True
 
     class Meta:
         unknown = EXCLUDE # required until all the pt fields are defined
@@ -1050,7 +1005,8 @@ class Video(Document, base.Video):
                 self.actor_id = new_act[0]
             
             entity = Post(**get_base_attributes(self,
-                keep=('_mentions', '_media_type', '_rendered_content', '_cached_children', '_cached_raw_content')))
+                keep=('_mentions', '_media_type', '_rendered_content',
+                      '_cached_children', '_cached_raw_content', '_source_object')))
             set_public(entity)
             return entity
         #Some Video object
@@ -1160,12 +1116,13 @@ class Follow(Activity, base.Follow):
 
     class Meta:
         rdf_type = as2.Follow
-        exclude = ('created_at', 'handle')
+        exclude = ('created_at',)
 
 
 class Announce(Activity, base.Share):
     id = fields.Id()
     target_id = IRI(as2.object)
+    signable = True
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
@@ -1195,14 +1152,20 @@ class Announce(Activity, base.Share):
             self.target_id = self.id
             self.entity_type = 'Object'
             self.__dict__.update({'schema': True})
-            entity = base.Retraction(**get_base_attributes(self))
+            entity = Retraction(**get_base_attributes(self, keep=('_source_object',)))
 
         set_public(entity)
         return entity
 
     class Meta:
         rdf_type = as2.Announce
-    
+
+
+# Only used for inbound share retraction (undo announce)
+class Retraction(Announce, base.Retraction):
+    class Meta:
+        rdf_type = as2.Announce
+
 
 class Tombstone(Object, base.Retraction):
     target_id = fields.Id()
@@ -1259,6 +1222,8 @@ class Accept(Create, base.Accept):
 
 
 class Delete(Create, base.Retraction):
+    signable = True
+
     def to_base(self):
         if hasattr(self, 'object_') and not isinstance(self.object_, Tombstone):
             self.target_id = self.object_
@@ -1358,7 +1323,7 @@ def extract_replies(replies):
                     obj = obj.to_base()
                     extract_and_validate(obj)
                 except ValueError as ex:
-                    logger.error("extract_replies - Failed to validate entity %s: %s", entity, ex)
+                    logger.error("extract_replies - Failed to validate entity %s: %s", obj, ex)
                     continue
             elif not isinstance(obj, str): continue
             objs.append(obj)
@@ -1373,7 +1338,7 @@ def extract_replies(replies):
     return objs
 
 
-def element_to_objects(element: Union[Dict, Object]) -> List:
+def element_to_objects(element: Union[Dict, Object], sender: str = "") -> List:
     """
     Transform an Element to a list of entities.
     """
@@ -1383,6 +1348,7 @@ def element_to_objects(element: Union[Dict, Object]) -> List:
     entity = model_to_objects(element) if not isinstance(element, Object) else element
     if entity and hasattr(entity, 'to_base'):
         entity = entity.to_base()
+        entity._sender = sender
     if isinstance(entity, (
         base.Post, base.Comment, base.Profile, base.Share, base.Follow,
         base.Retraction, base.Accept,)
@@ -1391,7 +1357,12 @@ def element_to_objects(element: Union[Dict, Object]) -> List:
             extract_and_validate(entity)
         except ValueError as ex:
             logger.error("Failed to validate entity %s: %s", entity, ex)
-            return None
+            return []
+        except InvalidSignature as exc:
+            logger.info('%s, fetching from remote', exc)
+            entity = retrieve_and_parse_document(entity.id)
+            if not entity:
+                return []
         logger.info('Entity type "%s" was handled through the json-ld processor', entity.__class__.__name__)
         return [entity]
     elif entity:
@@ -1404,17 +1375,31 @@ def element_to_objects(element: Union[Dict, Object]) -> List:
 
 
 def model_to_objects(payload):
+    original_payload = copy.copy(payload)
     model = globals().get(payload.get('type'))
     if model and issubclass(model, Object):
         try:
             entity = model.schema().load(payload)
         except (KeyError, jsonld.JsonLdError, exceptions.ValidationError) as exc :  # Just give up for now. This must be made robust
-            logger.error(f"Error parsing jsonld payload ({exc})")
+            logger.error("Error parsing jsonld payload (%s)", exc)
             return None
 
         if isinstance(getattr(entity, 'object_', None), Object):
             entity.object_.activity = entity
             entity = entity.object_
-    
+
+        entity._source_object = original_payload
         return entity
     return None
+
+
+CLASSES_WITH_CONTEXT_EXTENSIONS = (
+    Document,
+    Emoji,
+    Hashtag,
+    IdentityProof,
+    Note,
+    Person,
+    PropertyValue
+)
+context_manager = LdContextManager(CLASSES_WITH_CONTEXT_EXTENSIONS)

federation/entities/mixins.py

@@ -5,6 +5,7 @@ import warnings
 from typing import List, Set, Union, Dict, Tuple
 
 from commonmark import commonmark
+from marshmallow import missing
 
 from federation.entities.activitypub.enums import ActivityType
 from federation.entities.utils import get_name_for_profile, get_profile
@@ -19,6 +20,7 @@ class BaseEntity:
     _source_protocol: str = ""
     # Contains the original object from payload as a string
     _source_object: Union[str, Dict] = None
+    _sender: str = ""
     _sender_key: str = ""
     # ActivityType
     activity: ActivityType = None
@@ -100,7 +102,7 @@ class BaseEntity:
     def _validate_required(self, attributes):
         """Ensure required attributes are present."""
         required_fulfilled = set(self._required).issubset(set(attributes))
-        if not required_fulfilled:
+        if not required_fulfilled or required_fulfilled is missing:
             raise ValueError(
                 "Not all required attributes fulfilled. Required: {required}".format(required=set(self._required))
             )
@@ -115,7 +117,7 @@ class BaseEntity:
         attrs_to_check = set(self._required) & set(attributes)
         for attr in attrs_to_check:
             value = getattr(self, attr)  # We should always have a value here
-            if value is None or value == "":
+            if value is None or value == "" or value is missing:
                 raise ValueError(
                     "Attribute %s cannot be None or an empty string since it is required." % attr
                 )
@@ -230,8 +232,8 @@ class RawContentMixin(BaseEntity):
     @property
     def rendered_content(self) -> str:
         """Returns the rendered version of raw_content, or just raw_content."""
-        from federation.utils.django import get_configuration
         try:
+            from federation.utils.django import get_configuration
             config = get_configuration()
             if config["tags_path"]:
                 def linkifier(tag: str) -> str:

federation/entities/utils.py

@@ -15,7 +15,9 @@ def get_base_attributes(entity, keep=()):
     cls = entity.__class__
     for attr, _ in inspect.getmembers(cls, lambda o: not isinstance(o, property) and not inspect.isroutine(o)):
         if not attr.startswith("_") or attr in keep:
-            attributes[attr] = getattr(entity, attr)
+            value = getattr(entity, attr)
+            if value or isinstance(value, bool):
+                attributes[attr] = value
     return attributes
 
 

federation/hostmeta/django/generators.py

@@ -60,7 +60,7 @@ def rfc7033_webfinger_view(request, *args, **kwargs):
     if not resource.startswith("acct:"):
         return HttpResponseBadRequest("Invalid resource")
     handle = resource.replace("acct:", "").lower()
-    logger.debug(f"{handle} requested with {request}")
+    logger.debug("%s requested with %s", handle, request)
     profile_func = get_function_from_config("get_profile_function")
 
     try:

federation/protocols/activitypub/protocol.py

@@ -43,6 +43,7 @@ class Protocol:
     get_contact_key = None
     payload = None
     request = None
+    sender = None
     user = None
 
     def build_send(self, entity: BaseEntity, from_user: UserType, to_user_key: RsaKey = None) -> Union[str, Dict]:
@@ -58,7 +59,7 @@ class Protocol:
             # Use pregenerated outbound document
             rendered = entity.outbound_doc
         else:
-            rendered = entity.to_as2()
+            rendered = entity.sign_as2(sender=from_user)
         return rendered
 
     def extract_actor(self):
@@ -86,12 +87,9 @@ class Protocol:
         # Verify the message is from who it claims to be
         if not skip_author_verification:
             try:
-                self.verify_signature()
+                # Verify the HTTP signature
+                self.sender = verify_request_signature(self.request)
             except (ValueError, KeyError, InvalidSignature) as exc:
-                logger.warning(f'Signature verification failed: {exc}')
+                logger.warning('HTTP signature verification failed: %s', exc)
                 return self.actor, {}
-        return self.actor, self.payload
-
-    def verify_signature(self):
-        # Verify the HTTP signature
-        verify_request_signature(self.request)
+        return self.sender, self.payload

federation/protocols/activitypub/signing.py

@@ -5,7 +5,6 @@ https://funkwhale.audio/
 """
 import datetime
 import logging
-from typing import Union
 from urllib.parse import urlsplit
 
 import pytz
@@ -14,7 +13,6 @@ from httpsig.sign_algorithms import PSS
 from httpsig.requests_auth import HTTPSignatureAuth
 from httpsig.verify import HeaderVerifier
 
-
 from federation.types import RequestType
 from federation.utils.network import parse_http_date
 from federation.utils.text import encode_if_text

federation/tests/entities/activitypub/test_entities.py

@@ -5,7 +5,7 @@ from pprint import pprint
 # noinspection PyPackageRequirements
 from Crypto.PublicKey.RSA import RsaKey
 
-from federation.entities.activitypub.constants import CONTEXT
+from federation.entities.activitypub.models import context_manager
 from federation.entities.activitypub.models import Accept
 from federation.tests.fixtures.keys import PUBKEY
 from federation.types import UserType
@@ -15,7 +15,7 @@ class TestEntitiesConvertToAS2:
     def test_accept_to_as2(self, activitypubaccept):
         result = activitypubaccept.to_as2()
         assert result == {
-            "@context": CONTEXT,
+            "@context": context_manager.build_context(activitypubaccept),
             "id": "https://localhost/accept",
             "type": "Accept",
             "actor": "https://localhost/profile",
@@ -30,7 +30,7 @@ class TestEntitiesConvertToAS2:
     def test_announce_to_as2(self, activitypubannounce):
         result = activitypubannounce.to_as2()
         assert result == {
-            "@context": CONTEXT,
+            "@context": context_manager.build_context(activitypubannounce),
             "id": "http://127.0.0.1:8000/post/123456/#create",
             "type": "Announce",
             "actor": "http://127.0.0.1:8000/profile/123456/",
@@ -42,13 +42,14 @@ class TestEntitiesConvertToAS2:
         activitypubcomment.pre_send()
         result = activitypubcomment.to_as2()
         assert result == {
-            '@context': CONTEXT,
+            '@context': context_manager.build_context(activitypubcomment),
             'type': 'Create',
             'id': 'http://127.0.0.1:8000/post/123456/#create',
             'actor': 'http://127.0.0.1:8000/profile/123456/',
             'object': {
                 'id': 'http://127.0.0.1:8000/post/123456/',
                 'type': 'Note',
+                'url': 'http://127.0.0.1:8000/post/123456/',
                 'attributedTo': 'http://127.0.0.1:8000/profile/123456/',
                 'content': '<p>raw_content</p>',
                 'published': '2019-04-27T00:00:00',
@@ -67,13 +68,14 @@ class TestEntitiesConvertToAS2:
         activitypubcomment.pre_send()
         result = activitypubcomment.to_as2()
         assert result == {
-            '@context': CONTEXT,
+            '@context': context_manager.build_context(activitypubcomment),
             'type': 'Create',
             'id': 'http://127.0.0.1:8000/post/123456/#create',
             'actor': 'http://127.0.0.1:8000/profile/123456/',
             'object': {
                 'id': 'http://127.0.0.1:8000/post/123456/',
                 'type': 'Note',
+                'url': 'http://127.0.0.1:8000/post/123456/',
                 'attributedTo': 'http://127.0.0.1:8000/profile/123456/',
                 'content': '<p>raw_content <a href="http://example.com" rel="nofollow" target="_blank">'
                            'http://example.com</a></p>',
@@ -91,7 +93,7 @@ class TestEntitiesConvertToAS2:
     def test_follow_to_as2(self, activitypubfollow):
         result = activitypubfollow.to_as2()
         assert result == {
-            "@context": CONTEXT,
+            "@context": context_manager.build_context(activitypubfollow),
             "id": "https://localhost/follow",
             "type": "Follow",
             "actor": "https://localhost/profile",
@@ -103,7 +105,7 @@ class TestEntitiesConvertToAS2:
         result["id"] = "https://localhost/undo"  # Real object will have a random UUID postfix here
         result["object"]["id"] = "https://localhost/follow"  # Real object will have a random UUID postfix here
         assert result == {
-            "@context": CONTEXT,
+            "@context": context_manager.build_context(activitypubundofollow),
             "id": "https://localhost/undo",
             "type": "Undo",
             "actor": "https://localhost/profile",
@@ -119,7 +121,7 @@ class TestEntitiesConvertToAS2:
         activitypubpost.pre_send()
         result = activitypubpost.to_as2()
         assert result == {
-            '@context': CONTEXT,
+            '@context': context_manager.build_context(activitypubpost),
             'type': 'Create',
             'id': 'http://127.0.0.1:8000/post/123456/#create',
             'actor': 'http://127.0.0.1:8000/profile/123456/',
@@ -130,6 +132,7 @@ class TestEntitiesConvertToAS2:
                 'cc': ['https://http://127.0.0.1:8000/profile/123456/followers/'],
                 'to': ['https://www.w3.org/ns/activitystreams#Public'],
                 'type': 'Note',
+                'url': 'http://127.0.0.1:8000/post/123456/',
                 'attributedTo': 'http://127.0.0.1:8000/profile/123456/',
                 'content': '<h1>raw_content</h1>',
                 'published': '2019-04-27T00:00:00',
@@ -148,13 +151,14 @@ class TestEntitiesConvertToAS2:
         activitypubpost_mentions.pre_send()
         result = activitypubpost_mentions.to_as2()
         assert result == {
-            '@context': CONTEXT,
+            '@context': context_manager.build_context(activitypubpost_mentions),
             'type': 'Create',
             'id': 'http://127.0.0.1:8000/post/123456/#create',
             'actor': 'http://127.0.0.1:8000/profile/123456/',
             'object': {
                 'id': 'http://127.0.0.1:8000/post/123456/',
                 'type': 'Note',
+                'url': 'http://127.0.0.1:8000/post/123456/',
                 'attributedTo': 'http://127.0.0.1:8000/profile/123456/',
                 'content': '<h1>raw_content</h1>\n<p>@{someone@localhost.local} @<a class="mention" '
                            'href="http://localhost.local/someone" rel="nofollow" target="_blank">'
@@ -190,13 +194,14 @@ class TestEntitiesConvertToAS2:
         activitypubpost_tags.pre_send()
         result = activitypubpost_tags.to_as2()
         assert result == {
-            '@context': CONTEXT,
+            '@context': context_manager.build_context(activitypubpost_tags),
             'type': 'Create',
             'id': 'http://127.0.0.1:8000/post/123456/#create',
             'actor': 'http://127.0.0.1:8000/profile/123456/',
             'object': {
                 'id': 'http://127.0.0.1:8000/post/123456/',
                 'type': 'Note',
+                'url': 'http://127.0.0.1:8000/post/123456/',
                 'attributedTo': 'http://127.0.0.1:8000/profile/123456/',
                 'content': '<h1>raw_content</h1>\n'
                            '<p><a class="mention hashtag" '
@@ -233,13 +238,14 @@ class TestEntitiesConvertToAS2:
         activitypubpost_images.pre_send()
         result = activitypubpost_images.to_as2()
         assert result == {
-            '@context': CONTEXT,
+            '@context': context_manager.build_context(activitypubpost_images),
             'type': 'Create',
             'id': 'http://127.0.0.1:8000/post/123456/#create',
             'actor': 'http://127.0.0.1:8000/profile/123456/',
             'object': {
                 'id': 'http://127.0.0.1:8000/post/123456/',
                 'type': 'Note',
+                'url': 'http://127.0.0.1:8000/post/123456/',
                 'attributedTo': 'http://127.0.0.1:8000/profile/123456/',
                 'content': '<p>raw_content</p>',
                 'published': '2019-04-27T00:00:00',
@@ -271,7 +277,7 @@ class TestEntitiesConvertToAS2:
         activitypubpost_diaspora_guid.pre_send()
         result = activitypubpost_diaspora_guid.to_as2()
         assert result == {
-            '@context': CONTEXT,
+            '@context': context_manager.build_context(activitypubpost_diaspora_guid),
             'type': 'Create',
             'id': 'http://127.0.0.1:8000/post/123456/#create',
             'actor': 'http://127.0.0.1:8000/profile/123456/',
@@ -279,6 +285,7 @@ class TestEntitiesConvertToAS2:
                 'id': 'http://127.0.0.1:8000/post/123456/',
                 'diaspora:guid': 'totallyrandomguid',
                 'type': 'Note',
+                'url': 'http://127.0.0.1:8000/post/123456/',
                 'attributedTo': 'http://127.0.0.1:8000/profile/123456/',
                 'content': '<p>raw_content</p>',
                 'published': '2019-04-27T00:00:00',
@@ -295,7 +302,7 @@ class TestEntitiesConvertToAS2:
     def test_profile_to_as2(self, activitypubprofile):
         result = activitypubprofile.to_as2()
         assert result == {
-            "@context": CONTEXT,
+            "@context": context_manager.build_context(activitypubprofile),
             "endpoints": {
                 "sharedInbox": "https://example.com/public",
             },
@@ -327,7 +334,7 @@ class TestEntitiesConvertToAS2:
     def test_profile_to_as2__with_diaspora_guid(self, activitypubprofile_diaspora_guid):
         result = activitypubprofile_diaspora_guid.to_as2()
         assert result == {
-            "@context": CONTEXT,
+            "@context": context_manager.build_context(activitypubprofile_diaspora_guid),
             "endpoints": {
                 "sharedInbox": "https://example.com/public",
             },
@@ -360,7 +367,7 @@ class TestEntitiesConvertToAS2:
     def test_retraction_to_as2(self, activitypubretraction):
         result = activitypubretraction.to_as2()
         assert result == {
-            '@context': CONTEXT,
+            '@context': context_manager.build_context(activitypubretraction),
             'type': 'Delete',
             'id': 'http://127.0.0.1:8000/post/123456/#delete',
             'actor': 'http://127.0.0.1:8000/profile/123456/',
@@ -374,7 +381,7 @@ class TestEntitiesConvertToAS2:
     def test_retraction_to_as2__announce(self, activitypubretraction_announce):
         result = activitypubretraction_announce.to_as2()
         assert result == {
-            '@context': CONTEXT,
+            '@context': context_manager.build_context(activitypubretraction_announce),
             'type': 'Undo',
             'id': 'http://127.0.0.1:8000/post/123456/#delete',
             'actor': 'http://127.0.0.1:8000/profile/123456/',

federation/tests/entities/activitypub/test_mappers.py

@@ -187,7 +187,7 @@ class TestActivitypubEntityMappersReceive:
             "private": "https://diaspodon.fr/users/jaywink/inbox",
             "public": "https://diaspodon.fr/inbox",
         }
-        assert profile.handle == ""
+        assert profile.handle == None
         assert profile.name == "Jason Robinson"
         assert profile.image_urls == {
             "large": "https://diaspodon.fr/system/accounts/avatars/000/033/155/original/pnc__picked_media_be51984c-4"

federation/tests/entities/diaspora/test_utils.py

@@ -13,22 +13,17 @@ from federation.entities.utils import get_base_attributes
 
 
 class TestGetBaseAttributes:
-    def test_get_base_attributes_returns_only_intended_attributes(self):
-        entity = Post()
+    def test_get_base_attributes_returns_only_intended_attributes(self, diasporapost, diasporaprofile):
+        entity = diasporapost
         attrs = get_base_attributes(entity).keys()
         assert set(attrs) == {
-            "created_at", "location", "provider_display_name", "public", "raw_content",
-            "signature", "base_url", "actor_id", "id", "handle", "guid", "activity", "activity_id",
-            "url", "mxid", "times", "to", "cc", "finger",
-        }
-        entity = Profile()
+            'activity', 'actor_id', 'created_at', 'guid', 'handle', 'id',
+            'provider_display_name', 'public', 'raw_content'}
+        entity = diasporaprofile
         attrs = get_base_attributes(entity).keys()
         assert set(attrs) == {
-            "created_at", "name", "email", "gender", "raw_content", "location", "public",
-            "nsfw", "public_key", "image_urls", "tag_list", "signature", "url", "atom_url",
-            "base_url", "id", "actor_id", "handle", "handle", "guid", "activity", "activity_id", "username",
-            "inboxes", "mxid", "times", "to", "cc", "finger",
-        }
+            'created_at', 'guid', 'handle', 'id', 'image_urls', 'inboxes',
+            'name', 'nsfw', 'public', 'raw_content', 'tag_list'}
 
 
 class TestGetFullXMLRepresentation:

federation/tests/utils/test_activitypub.py

@@ -79,11 +79,13 @@ class TestRetrieveAndParseDocument:
         assert entity._children[0].url == "https://files.mastodon.social/media_attachments/files/017/792/237/original" \
                                           "/foobar.jpg"
 
+    @patch("federation.entities.activitypub.models.verify_ld_signature", return_value=None)
     @patch("federation.entities.activitypub.models.extract_receivers", return_value=[])
     @patch("federation.utils.activitypub.fetch_document", autospec=True, return_value=(
         json.dumps(ACTIVITYPUB_POST), None, None),
     )
-    def test_returns_entity_for_valid_document__post__wrapped_in_activity(self, mock_fetch, mock_recv):
+    def test_returns_entity_for_valid_document__post__wrapped_in_activity(
+            self, mock_fetch, mock_recv, mock_sign):
         entity = retrieve_and_parse_document("https://example.com/foobar")
         assert isinstance(entity, Note)
 

federation/utils/activitypub.py

@@ -3,14 +3,17 @@ import logging
 from typing import Optional, Any
 
 from federation.protocols.activitypub.signing import get_http_authentication
-from federation.utils.django import get_federation_user
 from federation.utils.network import fetch_document, try_retrieve_webfinger_document
 from federation.utils.text import decode_if_bytes, validate_handle
 
 logger = logging.getLogger('federation')
 
-federation_user = get_federation_user()
-if not federation_user: logger.warning("django is required for get requests signing")
+try:
+    from federation.utils.django import get_federation_user
+    federation_user = get_federation_user()
+except ImportError:
+    federation_user = None
+    logger.warning("django is required for get requests signing")
 
 
 def get_profile_id_from_webfinger(handle: str) -> Optional[str]:

setup.py

@@ -32,6 +32,7 @@ setup(
         "cryptography",
         "cssselect>=0.9.2",
         "dirty-validators>=0.3.0",
+        "funcy",
         "lxml>=3.4.0",
         "iteration_utilities",
         "jsonschema>=2.0.0",