Add receivers in mappers to inbound entities

Entities processed by inbound mappers will now have a list of
receivers in `_receivers`. This replaces the
`_receiving_actor_id` which was previously set for Diaspora entities.

UserType now has a `receiver_variant` which is one of `ReceiverVariant`
enum. `ACTOR` means this receiver is a single actor ID.
`FOLLOWERS` means this is the followers of the ID in the receiver.

Contains terrible hack to figure out if ActivityPub to/cc contains
a reference to the followers collection of the sender 🙈 . Will replace
"later" with proper fetch+cache solution, once we have a cache.

Refs: https://git.feneas.org/socialhome/socialhome/issues/522

CHANGELOG.md

@@ -51,6 +51,16 @@
 
 * `Retraction` entity can now also have an `entity_type` of `Object`. Receivers will need to find the correct object using `target_id` only. This is currently only relevant for ActivityPub where retraction messages do not refer to object type.
 
+* **Backwards incompatible.** Inbound entities now have a list of receivers.
+
+  Entities processed by inbound mappers will now have a list of
+  receivers in `_receivers`. This replaces the
+  `_receiving_actor_id` which was previously set for Diaspora entities.
+
+* UserType now has a `receiver_variant` which is one of `ReceiverVariant`
+  enum. `ACTOR` means this receiver is a single actor ID.
+  `FOLLOWERS` means this is the followers of the ID in the receiver.
+
 ### Removed
 
 * **Backwards incompatible.** Support for Legacy Diaspora payloads have been removed to reduce the amount of code needed to maintain while refactoring for ActivityPub.

federation/__init__.py

@@ -1,5 +1,5 @@
 import importlib
-from typing import Union, TYPE_CHECKING
+from typing import Union, TYPE_CHECKING, Any
 
 from federation.exceptions import NoSuitableProtocolFoundError
 
@@ -32,5 +32,5 @@ def identify_protocol_by_id(id: str):
 
 
 def identify_protocol_by_request(request):
-    # type: (RequestType) -> str
+    # type: (RequestType) -> Any
     return identify_protocol('request', request)

federation/entities/activitypub/mappers.py

@@ -1,5 +1,5 @@
 import logging
-from typing import List, Callable, Dict, Union
+from typing import List, Callable, Dict, Union, Optional
 
 from federation.entities.activitypub.constants import NAMESPACE_PUBLIC
 from federation.entities.activitypub.entities import (
@@ -7,7 +7,7 @@ from federation.entities.activitypub.entities import (
     ActivitypubRetraction)
 from federation.entities.base import Follow, Profile, Accept, Post, Comment, Retraction
 from federation.entities.mixins import BaseEntity
-from federation.types import UserType
+from federation.types import UserType, ReceiverVariant
 
 logger = logging.getLogger("federation")
 
@@ -26,7 +26,7 @@ MAPPINGS = {
 
 def element_to_objects(payload: Dict) -> List:
     """
-    Transform an Element to a list of entities recursively.
+    Transform an Element to a list of entities.
     """
     entities = []
     if payload.get('type') == "Delete":
@@ -47,6 +47,8 @@ def element_to_objects(payload: Dict) -> List:
     entity._source_protocol = "activitypub"
     # Save element object to entity for possible later use
     entity._source_object = payload
+    # Extract receivers
+    entity._receivers = extract_receivers(payload)
 
     if hasattr(entity, "post_receive"):
         entity.post_receive()
@@ -66,6 +68,49 @@ def element_to_objects(payload: Dict) -> List:
     return entities
 
 
+def extract_receiver(payload: Dict, receiver: str) -> Optional[UserType]:
+    """
+    Transform a single receiver ID to a UserType.
+    """
+    if receiver == NAMESPACE_PUBLIC:
+        # Ignore since we already store "public" as a boolean on the entity
+        return
+    # Check for this being a list reference to followers of an actor?
+    # TODO: terrible hack! the way some platforms deliver to sharedInbox using just
+    #   the followers collection as a target is annoying to us since we would have to
+    #   store the followers collection references on application side, which we don't
+    #   want to do since it would make application development another step more complex.
+    #   So for now we're going to do a terrible assumption that
+    #     1) if "followers" in ID and
+    #     2) if ID starts with actor ID
+    #     then; assume this is the followers collection of said actor ID.
+    #   When we have a caching system, just fetch each receiver and check what it is.
+    #   Without caching this would be too expensive to do.
+    elif receiver.find("followers") > -1 and receiver.startswith(payload.get('actor')):
+        return UserType(id=payload.get("actor"), receiver_variant=ReceiverVariant.FOLLOWERS)
+    # Assume actor ID
+    return UserType(id=receiver, receiver_variant=ReceiverVariant.ACTOR)
+
+
+def extract_receivers(payload: Dict) -> List[UserType]:
+    """
+    Exctract receivers from a payload.
+    """
+    receivers = []
+    for key in ("to", "cc"):
+        receiver = payload.get(key)
+        if isinstance(receiver, list):
+            for item in receiver:
+                extracted = extract_receiver(payload, item)
+                if extracted:
+                    receivers.append(extracted)
+        elif isinstance(receiver, str):
+            extracted = extract_receiver(payload, receiver)
+            if extracted:
+                receivers.append(extracted)
+    return receivers
+
+
 def get_outbound_entity(entity: BaseEntity, private_key):
     """Get the correct outbound entity for this protocol.
 

federation/entities/diaspora/mappers.py

@@ -13,7 +13,7 @@ from federation.entities.diaspora.entities import (
 from federation.entities.diaspora.mixins import DiasporaRelayableMixin
 from federation.entities.mixins import BaseEntity
 from federation.protocols.diaspora.signatures import get_element_child_info
-from federation.types import UserType
+from federation.types import UserType, ReceiverVariant
 from federation.utils.diaspora import retrieve_and_parse_profile
 
 logger = logging.getLogger("federation")
@@ -74,18 +74,14 @@ def check_sender_and_entity_handle_match(sender_handle, entity_handle):
 
 
 def element_to_objects(
-        element: etree.ElementTree, sender: str, sender_key_fetcher:Callable[[str], str]=None, user: UserType =None,
+        element: etree.ElementTree, sender: str, sender_key_fetcher: Callable[[str], str] = None, user: UserType = None,
 ) -> List:
     """Transform an Element to a list of entities recursively.
 
     Possible child entities are added to each entity ``_children`` list.
 
-    :param tree: Element
-    :param sender: Payload sender id
-    :param sender_key_fetcher: Function to fetch sender public key. If not given, key will always be fetched
-        over network. The function should take sender handle as the only parameter.
-    :param user: Optional receiving user object. If given, should have an ``id``.
-    :returns: list of entities
+    Optional parameter ``sender_key_fetcher`` can be a function to fetch sender public key.
+    If not given, key will always be fetched over the network. The function should take sender as the only parameter.
     """
     entities = []
     cls = MAPPINGS.get(element.tag)
@@ -101,9 +97,15 @@ def element_to_objects(
     entity._source_protocol = "diaspora"
     # Save element object to entity for possible later use
     entity._source_object = etree.tostring(element)
-    # Save receiving id to object
+
+    # Save receivers on the entity
     if user:
-        entity._receiving_actor_id = user.id
+        # Single receiver
+        entity._receivers = [UserType(id=user.id, receiver_variant=ReceiverVariant.ACTOR)]
+    else:
+        # Followers
+        entity._receivers = [UserType(id=sender, receiver_variant=ReceiverVariant.FOLLOWERS)]
+
     if issubclass(cls, DiasporaRelayableMixin):
         # If relayable, fetch sender key for validation
         entity._xml_tags = get_element_child_info(element, "tag")

federation/entities/mixins.py

@@ -1,6 +1,7 @@
 import datetime
 import importlib
 import warnings
+from typing import List, Set
 
 from federation.entities.activitypub.enums import ActivityType
 
@@ -8,8 +9,9 @@ from federation.entities.activitypub.enums import ActivityType
 # TODO someday, rewrite entities as dataclasses or attr's
 class BaseEntity:
     _allowed_children: tuple = ()
-    # If we have a receiver for a private payload, store receiving actor id here
-    _receiving_actor_id: str = ""
+    _children: List = None
+    _mentions: Set = None
+    _receivers: List = None
     _source_protocol: str = ""
     # Contains the original object from payload as a string
     _source_object: str = None
@@ -29,6 +31,7 @@ class BaseEntity:
         self._required = ["id", "actor_id"]
         self._children = []
         self._mentions = set()
+        self._receivers = []
         for key, value in kwargs.items():
             if hasattr(self, key):
                 setattr(self, key, value)

federation/tests/entities/activitypub/test_mappers.py

@@ -10,6 +10,7 @@ from federation.entities.base import Accept, Follow, Profile, Post, Comment
 from federation.tests.fixtures.payloads import (
     ACTIVITYPUB_FOLLOW, ACTIVITYPUB_PROFILE, ACTIVITYPUB_PROFILE_INVALID, ACTIVITYPUB_UNDO_FOLLOW, ACTIVITYPUB_POST,
     ACTIVITYPUB_COMMENT, ACTIVITYPUB_RETRACTION)
+from federation.types import UserType, ReceiverVariant
 
 
 class TestActivitypubEntityMappersReceive:
@@ -140,16 +141,23 @@ class TestActivitypubEntityMappersReceive:
         assert profile.nsfw is False
         assert profile.tag_list == []
 
-    @pytest.mark.skip
-    def test_message_to_objects_receiving_actor_id_is_saved(self):
+    def test_message_to_objects_receivers_are_saved(self):
         # noinspection PyTypeChecker
         entities = message_to_objects(
-            DIASPORA_POST_SIMPLE,
-            "alice@alice.diaspora.example.org",
-            user=Mock(id="bob@example.com")
+            ACTIVITYPUB_POST,
+            "https://diaspodon.fr/users/jaywink",
         )
         entity = entities[0]
-        assert entity._receiving_actor_id == "bob@example.com"
+        
+        assert set(entity._receivers) == {
+            UserType(
+                id='https://diaspodon.fr/users/jaywink', receiver_variant=ReceiverVariant.FOLLOWERS,
+            ),
+            UserType(
+                id='https://dev.jasonrobinson.me/p/d4574854-a5d7-42be-bfac-f70c16fcaa97/',
+                receiver_variant=ReceiverVariant.ACTOR,
+            )
+        }
 
     def test_message_to_objects_retraction(self):
         entities = message_to_objects(ACTIVITYPUB_RETRACTION, "https://friendica.feneas.org/profile/jaywink")

federation/tests/entities/diaspora/test_mappers.py

@@ -19,6 +19,7 @@ from federation.tests.fixtures.payloads import (
     DIASPORA_PROFILE_EMPTY_TAGS, DIASPORA_RESHARE,
     DIASPORA_RESHARE_WITH_EXTRA_PROPERTIES, DIASPORA_POST_SIMPLE_WITH_MENTION,
     DIASPORA_PROFILE_FIRST_NAME_ONLY, DIASPORA_POST_COMMENT_NESTED)
+from federation.types import UserType, ReceiverVariant
 
 
 class TestDiasporaEntityMappersReceive:
@@ -159,7 +160,18 @@ class TestDiasporaEntityMappersReceive:
         entities = message_to_objects(DIASPORA_PROFILE_EMPTY_TAGS, "bob@example.com")
         assert len(entities) == 1
 
-    def test_message_to_objects_receiving_actor_id_is_saved(self):
+    def test_message_to_objects_receivers_are_saved__followers_receiver(self):
+        # noinspection PyTypeChecker
+        entities = message_to_objects(
+            DIASPORA_POST_SIMPLE,
+            "alice@alice.diaspora.example.org",
+        )
+        entity = entities[0]
+        assert entity._receivers == [UserType(
+            id="alice@alice.diaspora.example.org", receiver_variant=ReceiverVariant.FOLLOWERS,
+        )]
+
+    def test_message_to_objects_receivers_are_saved__single_receiver(self):
         # noinspection PyTypeChecker
         entities = message_to_objects(
             DIASPORA_POST_SIMPLE,
@@ -167,7 +179,7 @@ class TestDiasporaEntityMappersReceive:
             user=Mock(id="bob@example.com")
         )
         entity = entities[0]
-        assert entity._receiving_actor_id == "bob@example.com"
+        assert entity._receivers == [UserType(id="bob@example.com", receiver_variant=ReceiverVariant.ACTOR)]
 
     def test_message_to_objects_retraction(self):
         entities = message_to_objects(DIASPORA_RETRACTION, "bob@example.com")

federation/types.py

@@ -1,3 +1,4 @@
+from enum import Enum
 from typing import Optional, Dict, Union
 
 import attr
@@ -17,10 +18,18 @@ class RequestType:
     url: str = attr.ib(default=None)
 
 
-@attr.s
+class ReceiverVariant(Enum):
+    # Indicates this receiver is a single actor
+    ACTOR = "actor"
+    # Indicates this receiver is the followers of this actor
+    FOLLOWERS = "followers"
+
+
+@attr.s(frozen=True)
 class UserType:
     id: str = attr.ib()
     private_key: Optional[RsaKey] = attr.ib(default=None)
+    receiver_variant: Optional[ReceiverVariant] = attr.ib(default=None)
 
     # Required only if sending to Diaspora protocol platforms
     handle: Optional[str] = attr.ib(default=None)