From d72be97d7839e4b0fd32bcb0e6238499fc47f5ef Mon Sep 17 00:00:00 2001 From: Ryan Barrett Date: Sun, 29 Jan 2023 09:45:03 -0800 Subject: [PATCH] Activity => Object: populate ap_* fields for inbox delivery results #286 --- tests/test_webmention.py | 82 ++++++++++++++++++++++------------------ webmention.py | 61 +++++++++++++++++++----------- 2 files changed, 84 insertions(+), 59 deletions(-) diff --git a/tests/test_webmention.py b/tests/test_webmention.py index 529bbb0..772b71a 100644 --- a/tests/test_webmention.py +++ b/tests/test_webmention.py @@ -44,6 +44,12 @@ ACTOR_MF2 = { 'name': ['Ms. ☕ Baz'], }, } +ACTOR_AS1_UNWRAPPED = { + 'objectType': 'person', + 'displayName': 'Ms. ☕ Baz', + 'url': 'https://orig', + 'urls': [{'value': 'https://orig', 'displayName': 'Ms. ☕ Baz'}], +} ACTOR_AS2 = { 'type': 'Person', 'id': 'http://localhost/orig', @@ -335,10 +341,30 @@ class WebmentionTest(testutil.TestCase): content_type=CONTENT_TYPE_HTML) def assert_object(self, id, **props): - self.assert_entities_equal(Object(id=id, **props), - Object.get_by_id(id), + got = Object.get_by_id(id) + assert got, id + + # sort keys in JSON properties + for prop in 'as1', 'as2', 'bsky', 'mf2': + if prop in props: + props[prop] = json_dumps(json_loads(props[prop]), sort_keys=True) + got_val = getattr(got, prop, None) + if got_val: + setattr(got, prop, json_dumps(json_loads(got_val), sort_keys=True)) + + self.assert_entities_equal(Object(id=id, **props), got, ignore=['created', 'updated']) + def assert_deliveries(self, mock_post, inboxes, data): + self.assertEqual(len(inboxes), len(mock_post.call_args_list)) + calls = {call[0][0]: call for call in mock_post.call_args_list} + + for inbox in inboxes: + with self.subTest(inbox=inbox): + got = json_loads(calls[inbox][1]['data']) + got.get('object', {}).pop('publicKey', None) + self.assertEqual(data, got) + def test_bad_source_url(self, mock_get, mock_post): got = self.client.post('/webmention', data=b'') self.assertEqual(400, got.status_code) @@ -607,6 +633,7 @@ class WebmentionTest(testutil.TestCase): status='complete', mf2=json_dumps(self.repost_mf2), as1=json_dumps(self.repost_as1), + ap_delivered=['https://foo.com/inbox'], ) def test_link_rel_alternate_as2(self, mock_get, mock_post): @@ -771,16 +798,9 @@ class WebmentionTest(testutil.TestCase): inboxes = ('https://inbox', 'https://public/inbox', 'https://shared/inbox', 'https://updated/inbox') - self.assertEqual(len(inboxes), len(mock_post.call_args_list), - mock_post.call_args_list) - for call, inbox in zip(mock_post.call_args_list, inboxes): - with self.subTest(call=call, inbox=inbox): - self.assertEqual((inbox,), call[0]) - self.assertEqual( + self.assert_deliveries(mock_post, inboxes, self.create_as2) # TODO # self.update_as2 if inbox == 'https://updated/inbox' else - self.create_as2, - json_loads(call[1]['data'])) self.assert_object(f'https://orig/post', domains=['orig'], @@ -789,6 +809,7 @@ class WebmentionTest(testutil.TestCase): #(different_create_mf2 if inbox == 'https://updated/inbox' else mf2=json_dumps(self.create_mf2), as1=json_dumps(self.create_as1), + ap_delivered=inboxes, ) #(different_create_as1 if inbox == 'https://updated/inbox' else def test_create_with_image(self, mock_get, mock_post): @@ -849,6 +870,7 @@ class WebmentionTest(testutil.TestCase): status='complete', mf2=json_dumps(self.follow_mf2), as1=json_dumps(self.follow_as1), + ap_delivered=['https://foo.com/inbox'], ) followers = Follower.query().fetch() @@ -912,6 +934,7 @@ class WebmentionTest(testutil.TestCase): status='complete', mf2=json_dumps(self.follow_fragment_mf2), as1=json_dumps(self.follow_fragment_as1), + ap_delivered=['https://foo.com/inbox'], ) followers = Follower.query().fetch() @@ -965,7 +988,8 @@ class WebmentionTest(testutil.TestCase): status='failed', mf2=json_dumps(self.follow_mf2), as1=json_dumps(self.follow_as1), - ) + ap_failed=['https://foo.com/inbox'], + ) def test_repost_blocklisted_error(self, mock_get, mock_post): """Reposts of non-fediverse (ie blocklisted) sites aren't yet supported.""" @@ -980,7 +1004,7 @@ class WebmentionTest(testutil.TestCase): self.assertEqual(400, got.status_code) @mock.patch('oauth_dropins.webutil.appengine_config.tasks_client.create_task') - def test_create_post_make_task(self, mock_create_task, mock_get, _): + def test_update_profile_make_task(self, mock_create_task, mock_get, _): mock_get.side_effect = [self.author] got = self.client.post('/webmention', data={ @@ -1023,10 +1047,7 @@ class WebmentionTest(testutil.TestCase): self.req('https://orig/'), )) - inboxes = ('https://inbox', 'https://shared/inbox',) - self.assertEqual(len(inboxes), len(mock_post.call_args_list)) - - expected_update = { + self.assert_deliveries(mock_post, ('https://shared/inbox', 'https://inbox'), { '@context': 'https://www.w3.org/ns/activitystreams', 'type': 'Update', 'id': 'http://localhost/r/https://orig/#update-2022-01-02T03:04:05+00:00', @@ -1036,30 +1057,19 @@ class WebmentionTest(testutil.TestCase): 'updated': util.now().isoformat(), }, 'to': ['https://www.w3.org/ns/activitystreams#Public'], + }) + + expected_as1 = { + 'id': 'https://orig/#update-2022-01-02T03:04:05+00:00', + 'objectType': 'activity', + 'verb': 'update', + 'object': ACTOR_AS1_UNWRAPPED, } - - for call, inbox in zip(mock_post.call_args_list, inboxes): - with self.subTest(call=call, inbox=inbox): - self.assertEqual((inbox,), call[0]) - got_update = json_loads(call[1]['data']) - del got_update['object']['publicKey'] - self.assertEqual(expected_update, got_update) - self.assert_object(f'https://orig/', domains=['orig'], source_protocol='activitypub', status='complete', mf2=json_dumps(ACTOR_MF2), + as1=json_dumps(expected_as1), + ap_delivered=['https://inbox', 'https://shared/inbox'], ) - - self.assert_equals({ - 'id': 'https://orig/#update-2022-01-02T03:04:05+00:00', - 'objectType': 'activity', - 'verb': 'update', - 'object': { - 'objectType': 'person', - 'displayName': 'Ms. ☕ Baz', - 'url': 'https://orig', - 'urls': [{'value': 'https://orig', 'displayName': 'Ms. ☕ Baz'}], - }, - }, json_loads(obj.as1)) diff --git a/webmention.py b/webmention.py index 88b7218..9bec657 100644 --- a/webmention.py +++ b/webmention.py @@ -126,29 +126,46 @@ class Webmention(View): Returns Flask response (string body or tuple) if we succeeded or failed, None if ActivityPub was not available. """ - targets = self._activitypub_targets() - if not targets: + inboxes_to_targets = self._activitypub_targets() + if not inboxes_to_targets: return None - inboxes = [i for _, i in targets] error = None last_success = None log_data = True - obj = Object.get_or_insert(self.source_url, domains=[self.source_domain], - source_protocol='activitypub', - mf2=json_dumps(self.source_mf2), - as1=json_dumps(self.source_as1), - ap_undelivered=inboxes, - ap_delivered=[], - ap_failed=[]) + obj = Object.get_by_id(self.source_url) + if obj: + logging.info(f'Resuming existing Object {obj}') + seen = obj.ap_delivered + obj.ap_undelivered + obj.ap_failed + new_inboxes = [i for i in inboxes_to_targets.keys() if i not in seen] + if new_inboxes: + logging.info(f'Adding new inboxes: {new_inboxes}') + obj.ap_undelivered += new_inboxes + else: + obj = Object(id=self.source_url, domains=[self.source_domain], + source_protocol='activitypub', + mf2=json_dumps(self.source_mf2), + as1=json_dumps(self.source_as1), + ap_undelivered=list(inboxes_to_targets.keys()), + ap_delivered=[], + ap_failed=[]) + if (obj.status == 'complete' and not as1.activity_changed(json_loads(obj.as1), self.source_as1)): logger.info(f'Skipping; new content is same as content published before at {obj.updated}') return 'OK' # TODO: collect by inbox, add 'to' fields, de-dupe inboxes and recipients - for target_as2, inbox in targets: + # + # make copy of ap_undelivered because we modify it below + for inbox in list(obj.ap_undelivered): + if inbox in inboxes_to_targets: + target_as2 = inboxes_to_targets[inbox] + else: + logging.warning(f'Missing target_as2 for inbox {inbox}!') + target_as2 = None + if not self.source_as2: self.source_as2 = common.postprocess_as2( as2.from_as1(self.source_as1), target=target_as2, user=self.user) @@ -175,19 +192,19 @@ class Webmention(View): last_follow=json_dumps(self.source_as2)) try: - obj.ap_undelivered.remove(inbox) last = common.signed_post(inbox, data=self.source_as2, log_data=log_data, user=self.user) obj.ap_delivered.append(inbox) - obj.put() last_success = last except BaseException as e: obj.ap_failed.append(inbox) - obj.put() error = e finally: log_data = False + obj.ap_undelivered.remove(inbox) + obj.put() + obj.status = 'complete' if obj.ap_delivered else 'failed' obj.put() @@ -203,7 +220,7 @@ class Webmention(View): def _activitypub_targets(self): """ - Returns: list of (Object, string inbox URL) + Returns: dict of {str inbox URL: dict target AS2 object} """ # if there's in-reply-to, like-of, or repost-of, they're the targets. # otherwise, it's all followers' inboxes. @@ -248,16 +265,14 @@ class Webmention(View): inboxes.add(actor.get('endpoints', {}).get('sharedInbox') or actor.get('publicInbox') or actor.get('inbox')) - # TODO: collect inboxes into ap_* properties - inboxes = [(None, inbox) for inbox in sorted(inboxes) if inbox] - logger.info(f"Delivering to followers' inboxes: {inboxes}") - return inboxes + logger.info(f"Delivering to followers' inboxes: {sorted(inboxes)}") + return {inbox: None for inbox in inboxes} targets = common.remove_blocklisted(targets) if not targets: error(f"Silo responses are not yet supported.") - targets_and_inbox_urls = [] + inboxes_to_targets = {} for target in targets: # fetch target page as AS2 object try: @@ -298,10 +313,10 @@ class Webmention(View): continue inbox_url = urllib.parse.urljoin(target_url, inbox_url) - targets_and_inbox_urls.append((target_obj, inbox_url)) + inboxes_to_targets[inbox_url] = target_obj - logger.info(f"Delivering to targets' inboxes: {[i for _, i in targets_and_inbox_urls]}") - return targets_and_inbox_urls + logger.info(f"Delivering to targets' inboxes: {inboxes_to_targets.keys()}") + return inboxes_to_targets class WebmentionTask(Webmention):