Merge branch 'various-fixes' into 'master'

Various low hanging fruit fixes

See merge request jaywink/federation!174

CHANGELOG.md

@@ -1,5 +1,26 @@
 # Changelog
 
+## [0.24] - unreleased
+
+### Added
+
+* Add a validation function for the Activitypub `attributedTo` property. Ensure it starts with `http`.
+
+### Changed
+
+* Optimize handle_send by ensuring a payload is only sent once per recipient unique endpoint.
+
+* Match the Activitypub Hashtag object `href` property value against the raw content in order to make
+  this process platform agnostic.
+
+### Fixed
+
+* The Activitypub `url` property can now handle nested Link objects for all defined object types.
+
+* Catch cases where an Activitypub CollectionPage `next` property points back to a Collection object.
+
+* Make the Activitypub Follow class handle both the Undo and the Accept activities.
+
 ## [0.23.1] - 2023-02-08
 
 ### Changed

federation/entities/activitypub/models.py

@@ -367,13 +367,23 @@ class Object(BaseEntity, metaclass=JsonLDAnnotation):
             # AP activities may be signed, but most platforms don't
             # define RsaSignature2017. add it to the context
             # hubzilla doesn't define the discoverable property in its context
+            # include all Mastodon extensions for platforms that only define http://joinmastodon.org/ns in their context
             may_add = {'signature': ['https://w3id.org/security/v1', {'sec':'https://w3id.org/security#','RsaSignature2017':'sec:RsaSignature2017'}],
                     'publicKey': ['https://w3id.org/security/v1'],
                     'discoverable': [{'toot':'http://joinmastodon.org/ns#','discoverable': 'toot:discoverable'}], #for hubzilla
+                    'suspended': [{'toot':'http://joinmastodon.org/ns#','suspended': 'toot:suspended'}],
                     'copiedTo': [{'toot':'http://joinmastodon.org/ns#','copiedTo': 'toot:copiedTo'}], #for hubzilla
                     'featured': [{'toot':'http://joinmastodon.org/ns#','featured': 'toot:featured'}], #for litepub and pleroma
-                    'tag': [{'Hashtag': 'as:Hashtag'}], #for epicyon
-                    'attachment': [{'schema': 'http://schema.org#', 'PropertyValue': 'schema:PropertyValue'}] # for owncast
+                    'featuredTags': [{'toot':'http://joinmastodon.org/ns#','featuredTags': 'toot:featuredTags'}],
+                    'focalPoint': [{'toot':'http://joinmastodon.org/ns#',
+                                    'focalPoint': {'@id':'toot:focalPoint','@container':'@list'},
+                                    }],
+                    'tag': [{'Hashtag': 'as:Hashtag', #for epicyon
+                             'toot':'http://joinmastodon.org/ns#',
+                             'Emoji':'toot:Emoji'}],
+                    'attachment': [{'schema': 'http://schema.org#', 'PropertyValue': 'schema:PropertyValue', # for owncast
+                                    'toot':'http://joinmastodon.org/ns#','blurHash': 'toot:blurHash',
+                                    'IdentityProof': 'toot:IdentityProof'}]
                     }
 
             to_add = [val for key,val in may_add.items() if data.get(key)]
@@ -584,7 +594,7 @@ class Person(Object, base.Profile):
     username = fields.String(as2.preferredUsername)
     endpoints = CompactedDict(as2.endpoints)
     shared_inbox = IRI(as2.sharedInbox) # misskey adds this
-    url = IRI(as2.url)
+    url = MixedField(as2.url, nested='LinkSchema')
     playlists = IRI(pt.playlists)
     featured = IRI(toot.featured)
     featuredTags = IRI(toot.featuredTags)
@@ -821,10 +831,23 @@ class Note(Object, RawContentMixin):
         """
         super().post_receive()
 
-        if getattr(self, 'target_id'): self.entity_type = 'Comment'
+        if not self.raw_content or self._media_type == "text/markdown":
+            # Skip when markdown
+            return
 
+        hrefs = [tag.href.lower() for tag in self.tag_objects if isinstance(tag, Hashtag)]
         # noinspection PyUnusedLocal
         def remove_tag_links(attrs, new=False):
+            # Hashtag object hrefs
+            href = (None, "href")
+            url = attrs.get(href, "").lower()
+            if url in hrefs:
+                return
+            # one more time without the query (for pixelfed)
+            parsed = urlparse(url)
+            url = f'{parsed.scheme}://{parsed.netloc}{parsed.path}'
+            if url in hrefs:
+                return
 
             # Mastodon
             rel = (None, "rel")
@@ -832,16 +855,11 @@ class Note(Object, RawContentMixin):
                 return
             
             # Friendica
-            href = (None, "href")
-            if attrs.get(href).endswith(f'tag={attrs.get("_text")}'):
+            if attrs.get(href, "").endswith(f'tag={attrs.get("_text")}'):
                 return
 
             return attrs
 
-        if not self.raw_content or self._media_type == "text/markdown":
-            # Skip when markdown
-            return
-
         self.raw_content = bleach.linkify(
             self.raw_content,
             callbacks=[remove_tag_links],
@@ -849,6 +867,8 @@ class Note(Object, RawContentMixin):
             skip_tags=["code", "pre"],
         )
 
+        if getattr(self, 'target_id'): self.entity_type = 'Comment'
+
     def add_tag_objects(self) -> None:
         """
         Populate tags to the object.tag list.
@@ -951,6 +971,9 @@ class Note(Object, RawContentMixin):
         self._cached_children = value
         self.attachment = [Image.from_base(i) for i in value]
 
+    def validate_actor_id(self):
+        if not self.actor_id.startswith('http'):
+            raise ValueError(f'Invalid actor_id for activitypub ({self.actor_id})')
 
     class Meta:
         rdf_type = as2.Note
@@ -1076,10 +1099,13 @@ class Follow(Activity, base.Follow):
         return super().to_as2()
 
     def to_base(self):
-        # This is assuming Follow can only be the object of an Undo activity. Lazy.
-        if self.activity != self: 
+        if isinstance(self.activity, Undo):
             self.following = False
 
+        # Ensure the Accept activity is returned to the client app.
+        if isinstance(self.activity, Accept):
+            return self.activity
+
         return self
 
     def post_receive(self) -> None:
@@ -1224,6 +1250,9 @@ class Like(Activity, base.Reaction):
 
 # inbound Accept is a noop...
 class Accept(Create, base.Accept):
+    def validate(self):
+        pass
+
     class Meta:
         rdf_type = as2.Accept
         exclude = ('created_at',)
@@ -1333,7 +1362,7 @@ def extract_replies(replies):
                     continue
             elif not isinstance(obj, str): continue
             objs.append(obj)
-        if replies.next_ is not missing:
+        if getattr(replies, 'next_', None) not in (missing, None):
             if (replies.id != replies.next_) and (replies.next_ not in visited):
                 resp = retrieve_and_parse_document(replies.next_, cache=False)
                 if resp:

federation/outbound.py

@@ -167,7 +167,7 @@ def handle_send(
     logger.debug('handle_send - length of recipients: %s', len(recipients))
     # Flatten to unique recipients
     # TODO supply a callable that empties "fid" in the case that public=True
-    unique_recipients = list(unique_everseen(recipients))
+    unique_recipients = list(unique_everseen(recipients, key=lambda val: val['endpoint']))
     logger.debug('handle_send - length of unique_recipients: %s', len(unique_recipients))
     logger.debug('handle_send / unique_recipients - %s', unique_recipients)