Protocol.targets: find and add originals for targets that are copies

both Object and User originals

docs/index.rst

@@ -9,8 +9,9 @@ Bridgy Fed connects your web site to
 `microformats2 <https://microformats.org/wiki/microformats2>`__. Your
 site gets its own fediverse profile, posts and avatar and header and
 all. Bridgy Fed translates likes, reposts, mentions, follows, and more
-back and forth. `See the user docs <https://fed.brid.gy/docs>`__ for
+back and forth. `See the user docs <https://fed.brid.gy/docs>`__ and
-more details.
+`developer docs <https://bridgy-fed.readthedocs.io/>`__ for more
+details.
 
 https://fed.brid.gy/
 
@@ -23,7 +24,9 @@ License: This project is placed in the public domain.
 Development
 -----------
 
-Pull requests are welcome! Feel free to `ping me in
+Development reference docs are at
+`bridgy-fed.readthedocs.io <https://bridgy-fed.readthedocs.io/>`__. Pull
+requests are welcome! Feel free to `ping me in
 #indieweb-dev <https://indieweb.org/discuss>`__ with any questions.
 
 First, fork and clone this repo. Then, install the `Google Cloud
@@ -80,6 +83,36 @@ added you as an owner - run:
 
    gcloud -q beta app deploy --no-cache --project bridgy-federated *.yaml
 
+How to add a new protocol
+-------------------------
+
+1. Determine `how you’ll map the new protocol to other existing Bridgy
+   Fed protocols <https://fed.brid.gy/docs#translate>`__, specifically
+   identity, protocol inference, events, and operations. `Add those to
+   the existing tables in the
+   docs <https://github.com/snarfed/bridgy-fed/blob/main/templates/docs.html>`__
+   in a PR. This is an important step before you start writing code.
+2. If the new protocol uses a new data format - which is likely - add
+   that format to `granary <https://github.com/snarfed/granary>`__ in a
+   new file with functions that convert to/from `ActivityStreams
+   1 <https://activitystrea.ms/specs/json/1.0/>`__ and tests. See
+   `nostr.py <https://github.com/snarfed/granary/blob/main/granary/nostr.py#L542>`__
+   and
+   `test_nostr.py <https://github.com/snarfed/granary/blob/main/granary/tests/test_nostr.py#>`__
+   for examples.
+3. Implement the protocol in a new ``.py`` file as a subclass of both
+   `Protocol <https://github.com/snarfed/bridgy-fed/blob/main/protocol.py>`__
+   and
+   `User <https://github.com/snarfed/bridgy-fed/blob/main/models.py>`__.
+   Implement the ``send``, ``fetch``, ``serve``, and ``target_for``
+   methods from ``Protocol`` and ``readable_id``, ``web_url``,
+   ``ap_address``, and ``ap_actor`` from ``User`` .
+4. TODO: add a new usage section to the docs for the new protocol.
+5. TODO: does the new protocol need any new UI or signup functionality?
+   Unusual, but not impossible. Add that if necessary.
+6. Add the new protocol’s logo to ``static/``, use it in
+   `templates/user.html <https://github.com/snarfed/bridgy-fed/blob/main/templates/user.html>`__.
+
 Stats
 -----
 
@@ -94,7 +127,7 @@ Bridgy <https://bridgy.readthedocs.io/#stats>`__. Here’s how.
 
    ::
 
-      gcloud datastore export --async gs://bridgy-federated.appspot.com/stats/ --kinds Follower,Response
+      gcloud datastore export --async gs://bridgy-federated.appspot.com/stats/ --kinds Follower,Object
 
    Note that ``--kinds`` is required. `From the export
    docs <https://cloud.google.com/datastore/docs/export-import-entities#limitations>`__:
@@ -109,7 +142,7 @@ Bridgy <https://bridgy.readthedocs.io/#stats>`__. Here’s how.
 
    ::
 
-      for kind in Follower Response; do
+      for kind in Follower Object; do
         bq load --replace --nosync --source_format=DATASTORE_BACKUP datastore.$kind gs://bridgy-federated.appspot.com/stats/all_namespaces/kind_$kind/all_namespaces_kind_$kind.export_metadata
       done
 

protocol.py

@@ -418,7 +418,7 @@ class Protocol:
             obj.changed = orig.changed
 
         # if this is a post, ie not an activity, wrap it in a create or update
-        obj = from_cls._handle_bare_object(obj)
+        obj = from_cls.handle_bare_object(obj)
 
         if obj.type not in SUPPORTED_TYPES:
             error(f'Sorry, {obj.type} activities are not supported yet.', status=501)
@@ -454,7 +454,7 @@ class Protocol:
             return 'OK'  # noop
 
         elif obj.type == 'stop-following':
-            # TODO: unify with _handle_follow?
+            # TODO: unify with handle_follow?
             # TODO: handle multiple followees
             if not actor_id or not inner_obj_id:
                 error(f'Undo of Follow requires actor id and object id. Got: {actor_id} {inner_obj_id} {obj.as1}')
@@ -528,13 +528,13 @@ class Protocol:
                 }
 
         if obj.type == 'follow':
-            from_cls._handle_follow(obj)
+            from_cls.handle_follow(obj)
 
         # deliver to targets
-        return from_cls._deliver(obj)
+        return from_cls.deliver(obj)
 
     @classmethod
-    def _handle_follow(from_cls, obj):
+    def handle_follow(from_cls, obj):
         """Handles an incoming follow activity.
 
         Args:
@@ -626,7 +626,7 @@ class Protocol:
                 accept.put()
 
     @classmethod
-    def _handle_bare_object(cls, obj):
+    def handle_bare_object(cls, obj):
         """If obj is a bare object, wraps it in a create or update activity.
 
         Checks if we've seen it before.
@@ -691,7 +691,7 @@ class Protocol:
         error(f'{obj.key.id()} is unchanged, nothing to do', status=204)
 
     @classmethod
-    def _deliver(from_cls, obj):
+    def deliver(from_cls, obj):
         """Delivers an activity to its external recipients.
 
         Args:
@@ -699,7 +699,7 @@ class Protocol:
         """
         # find delivery targets
         # sort targets so order is deterministic for tests, debugging, etc
-        targets = from_cls._targets(obj)  # maps Target to Object or None
+        targets = from_cls.targets(obj)  # maps Target to Object or None
 
         if not targets:
             obj.status = 'ignored'
@@ -759,24 +759,37 @@ class Protocol:
         return ret
 
     @classmethod
-    def _targets(cls, obj):
+    def targets(cls, obj):
         """Collects the targets to send an :class:`models.Object` to.
 
         Targets are both objects - original posts, events, etc - and actors.
 
         Args:
-          obj: :class:`models.Object`
+          obj (:class:`models.Object`)
 
-        Returns: dict: {
+        Returns:
-          :class:`Target`: original (in response to) :class:`Object`, if any,
+          dict: {
-          otherwise None
+            :class:`Target`: original (in response to) :class:`models.Object`,
-        }
+            if any, otherwise None
+          }
         """
         logger.info('Finding recipients and their targets')
 
+        target_uris = set(as1.targets(obj.as1))
+        logger.info(f'Raw targets: {target_uris}')
+
+        if target_uris:
+            origs = {u.key.id() for u in User.get_for_copies(target_uris)} | \
+                {o.key.id() for o in Object.query(Object.copies.uri.IN(target_uris))}
+            if origs:
+                target_uris |= origs
+                logger.info(f'Added originals: {origs}')
+
+
         orig_obj = None
-        targets = {}
+        targets = {}  # maps Target to Object or None
-        for id in sorted(as1.targets(obj.as1)):
+
+        for id in sorted(target_uris):
             protocol = Protocol.for_id(id)
             if not protocol:
                 logger.info(f"Can't determine protocol for {id}")

tests/test_protocol.py

@@ -303,7 +303,49 @@ class ProtocolTest(TestCase):
         self.assertCountEqual([
             Target(protocol='fake', uri='fake:post:target'),
             Target(protocol='atproto', uri='http://localhost/'),
-        ], Protocol._targets(obj).keys())
+        ], Protocol.targets(obj).keys())
+
+    @patch('requests.get', return_value=requests_response({}))
+    def test_targets_converts_copies_to_originals(self, mock_get):
+        """targets should convert User/Object.copies to their originals."""
+        alice = self.make_user('fake:alice', cls=Fake,
+                               copies=[Target(uri='did:plc:alice', protocol='atproto')])
+        bob = self.make_user(
+            'fake:bob', cls=OtherFake,
+            copies=[Target(uri='other:bob', protocol='other')])
+        obj = self.store_object(
+            id='fake:post', our_as1={'foo': 9},
+            copies=[Target(uri='at://did:plc:eve/post/789', protocol='fake')])
+
+        Fake.fetchable = {
+            'fake:alice': {'foo': 1},
+            'fake:bob': {'foo': 2},
+        }
+        OtherFake.fetchable = {
+            'other:bob': {'foo': 3},
+        }
+
+        obj = Object(our_as1={
+            'id': 'other:reply',
+            'objectType': 'note',
+            'inReplyTo': [
+                'at://did:web:unknown/post/123',
+                'at://did:plc:eve/post/789',
+            ],
+            'tags': [{
+                'objectType': 'mention',
+                'url': 'did:plc:alice',
+            }, {
+                'objectType': 'mention',
+                'url': 'other:bob',
+            }],
+        })
+        self.assertCountEqual([
+            Target(uri='fake:post:target', protocol='fake'),
+            Target(uri='fake:alice:target', protocol='fake'),
+            Target(uri='fake:bob:target', protocol='fake'),
+            Target(uri='other:bob:target', protocol='otherfake'),
+        ], Protocol.targets(obj).keys())
 
 
 class ProtocolReceiveTest(TestCase):
@@ -1161,9 +1203,11 @@ class ProtocolReceiveTest(TestCase):
 
         self.assertEqual('OK', OtherFake.receive_as1(follow_as1))
 
-        self.assertEqual(2, len(Fake.sent))
+        self.assertEqual(1, len(OtherFake.sent))
-        self.assertEqual('accept', Fake.sent[0][0].type)
+        self.assertEqual('accept', OtherFake.sent[0][0].type)
-        self.assertEqual('follow', Fake.sent[1][0].type)
+
+        self.assertEqual(1, len(Fake.sent))
+        self.assertEqual('follow', Fake.sent[0][0].type)
 
         followers = Follower.query().fetch()
         self.assertEqual(1, len(followers))

tests/testutil.py

@@ -140,6 +140,10 @@ class OtherFake(Fake):
     """
     ABBREV = 'other'
 
+    fetchable = {}
+    sent = []
+    fetched = []
+
     @classmethod
     def owns_id(cls, id):
         return id.startswith('other:')