diff --git a/activitypub.py b/activitypub.py index a609fd4..2dce8aa 100644 --- a/activitypub.py +++ b/activitypub.py @@ -136,7 +136,7 @@ def inbox(domain=None): logging.info('Dropping non-public activity') return '' - # normal post, deliver to BF followers + # normal post, deliver to BF followers locally source = util.get_first(activity, 'url') or activity.get('id') domains = [] if actor: @@ -150,7 +150,7 @@ def inbox(domain=None): key = Object(id=source, source_protocol='activitypub', domains=domains, status='complete', as2=activity_as2_str, as1=activity_as1_str, type=as1_type, object_ids=as1.get_ids(activity_as1, 'object'), - labels=['feed']).put() + labels=['feed', 'activity']).put() logging.info(f'Wrote Object {key} with {len(domains)} follower domains') return '' diff --git a/common.py b/common.py index e8c9a35..baec62b 100644 --- a/common.py +++ b/common.py @@ -294,16 +294,20 @@ def send_webmentions(activity_wrapped, proxy=None, **object_props): # send webmentions and store Objects errors = [] # stores (code, body) tuples + domains = [] + + obj = Object(id=source, labels=['notification'], undelivered=targets, **object_props) + if activity.get('objectType') == 'activity': + obj.labels.append('activity') + obj.put() + for target in targets: domain = util.domain_from_link(target, minimize=False) if (domain == util.domain_from_link(source, minimize=False)): logger.info(f'Skipping same-domain webmention from {source} to {target}') continue - # TODO: unify across targets - obj = Object(id=source, domains=[domain], labels=['notification'], - **object_props) - obj.put() + domains.append(domain) wm_source = (obj.proxy_url() if verb in ('follow', 'like', 'share') or proxy else source) @@ -313,15 +317,24 @@ def send_webmentions(activity_wrapped, proxy=None, **object_props): endpoint = webmention.discover(target).endpoint if endpoint: webmention.send(endpoint, wm_source, target) - obj.status = 'complete' logger.info('Success!') + obj.delivered.append(target) else: - obj.status = 'ignored' - logger.info('Ignoring.') + logger.info('No webmention endpoint') except BaseException as e: - errors.append(util.interpret_http_exception(e)) + code, body = util.interpret_http_exception(e) + if not code and not body: + raise + errors.append((code, body)) + obj.failed.append(target) + + obj.undelivered.remove(target) obj.put() + obj.status = 'complete' if obj.delivered else 'failed' if obj.failed else 'ignored' + obj.domains = domains + obj.put() + if errors: msg = 'Errors: ' + ', '.join(f'{code} {body}' for code, body in errors) error(msg, status=int(errors[0][0] or 502)) diff --git a/follow.py b/follow.py index 5af9ac5..6d4f81e 100644 --- a/follow.py +++ b/follow.py @@ -167,7 +167,7 @@ class FollowCallback(indieauth.Callback): follow_json = json_dumps(follow_as2, sort_keys=True) Follower.get_or_create(dest=id, src=domain, status='active', last_follow=follow_json) - Object(id=follow_id, domains=[domain], labels=['user'], + Object(id=follow_id, domains=[domain], labels=['user', 'activity'], source_protocol='ui', status='complete', as2=follow_json, as1=json_dumps(as2.to_as1(follow_as2), sort_keys=True), ).put() @@ -224,7 +224,7 @@ class UnfollowCallback(indieauth.Callback): follower.status = 'inactive' follower.put() - Object(id=unfollow_id, domains=[domain], labels=['user'], + Object(id=unfollow_id, domains=[domain], labels=['user', 'activity'], source_protocol='ui', status='complete', as2=json_dumps(unfollow_as2, sort_keys=True), as1=json_dumps(as2.to_as1(unfollow_as2), sort_keys=True), diff --git a/models.py b/models.py index ec3167c..e6d9075 100644 --- a/models.py +++ b/models.py @@ -234,7 +234,7 @@ class Object(StringIdModel): """ STATUSES = ('new', 'in progress', 'complete', 'failed', 'ignored') PROTOCOLS = ('activitypub', 'bluesky', 'ostatus', 'webmention', 'ui') - LABELS = ('feed', 'notification', 'user') + LABELS = ('activity', 'feed', 'notification', 'user') # domains of the Bridgy Fed users this activity is to or from domains = ndb.StringProperty(repeated=True) @@ -254,10 +254,10 @@ class Object(StringIdModel): deleted = ndb.BooleanProperty() object_ids = ndb.StringProperty(repeated=True) # id(s) of inner objects - # ActivityPub inbox delivery - ap_delivered = ndb.StringProperty(repeated=True) - ap_undelivered = ndb.StringProperty(repeated=True) - ap_failed = ndb.StringProperty(repeated=True) + # URLs we deliver(ed) this to. ActivityPub inboxes, webmention targets, etc. + delivered = ndb.StringProperty(repeated=True) + undelivered = ndb.StringProperty(repeated=True) + failed = ndb.StringProperty(repeated=True) created = ndb.DateTimeProperty(auto_now_add=True) updated = ndb.DateTimeProperty(auto_now=True) diff --git a/tests/test_activitypub.py b/tests/test_activitypub.py index f59775c..138fe78 100644 --- a/tests/test_activitypub.py +++ b/tests/test_activitypub.py @@ -291,20 +291,25 @@ class ActivityPubTest(testutil.TestCase): def test_inbox_reply_object(self, *mocks): self._test_inbox_reply(REPLY_OBJECT, - {'as2': REPLY_OBJECT, 'type': 'comment'}, + {'as2': REPLY_OBJECT, + 'type': 'comment', + 'labels': ['notification']}, *mocks) def test_inbox_reply_object_wrapped(self, *mocks): self._test_inbox_reply(REPLY_OBJECT_WRAPPED, {'as2': REPLY_OBJECT, - 'type': 'comment'}, + 'type': 'comment', + 'labels': ['notification']}, *mocks) def test_inbox_reply_create_activity(self, *mocks): self._test_inbox_reply(REPLY, {'as2': REPLY, 'type': 'post', - 'object_ids': [REPLY_OBJECT['id']]}, + 'object_ids': [REPLY_OBJECT['id']], + 'labels': ['notification', 'activity'], + }, *mocks) def _test_inbox_reply(self, reply, expected_props, mock_head, mock_get, mock_post): @@ -332,7 +337,7 @@ class ActivityPubTest(testutil.TestCase): source_protocol='activitypub', status='complete', as1=as2.to_as1(expected_props['as2']), - labels=['notification'], + delivered=['http://or.ig/post'], **expected_props) def test_inbox_reply_to_self_domain(self, mock_head, mock_get, mock_post): @@ -354,7 +359,6 @@ class ActivityPubTest(testutil.TestCase): mock_get.assert_not_called() mock_post.assert_not_called() - self.assertEqual(0, Object.query().count()) def test_inbox_create_obj(self, mock_head, mock_get, mock_post): Follower.get_or_create(ACTOR['id'], 'foo.com') @@ -380,7 +384,7 @@ class ActivityPubTest(testutil.TestCase): as1=as2.to_as1(expected_as2), domains=['foo.com', 'baz.com'], type='post', - labels=['feed'], + labels=['feed', 'activity'], object_ids=[NOTE_OBJECT['id']]) def test_inbox_not_public(self, mock_head, mock_get, mock_post): @@ -400,7 +404,10 @@ class ActivityPubTest(testutil.TestCase): def test_inbox_mention_object(self, *mocks): self._test_inbox_mention( MENTION_OBJECT, - {'type': 'note'}, # not mention (?) + { + 'type': 'note', # not mention (?) + 'labels': ['notification'], + }, *mocks, ) @@ -410,6 +417,7 @@ class ActivityPubTest(testutil.TestCase): { 'type': 'post', # not mention (?) 'object_ids': [MENTION_OBJECT['id']], + 'labels': ['notification', 'activity'], }, *mocks, ) @@ -442,7 +450,7 @@ class ActivityPubTest(testutil.TestCase): status='complete', as2=expected_as2, as1=as2.to_as1(expected_as2), - labels=['notification'], + delivered=['http://tar.get/'], **expected_props) def test_inbox_like(self, mock_head, mock_get, mock_post): @@ -477,8 +485,9 @@ class ActivityPubTest(testutil.TestCase): status='complete', as2=LIKE_WITH_ACTOR, as1=as2.to_as1(LIKE_WITH_ACTOR), + delivered=['http://or.ig/post'], type='like', - labels=['notification'], + labels=['notification', 'activity'], object_ids=[LIKE['object']]) def test_inbox_follow_accept_with_id(self, mock_head, mock_get, mock_post): @@ -494,8 +503,9 @@ class ActivityPubTest(testutil.TestCase): status='complete', as2=follow, as1=as2.to_as1(follow), + delivered=['https://www.realize.be/'], type='follow', - labels=['notification'], + labels=['notification', 'activity'], object_ids=[FOLLOW['object']]) follower = Follower.query().get() @@ -537,8 +547,9 @@ class ActivityPubTest(testutil.TestCase): status='complete', as2=follow, as1=as2.to_as1(follow), + delivered=['https://www.realize.be/'], type='follow', - labels=['notification'], + labels=['notification', 'activity'], object_ids=[FOLLOW['object']]) def _test_inbox_follow_accept(self, follow_as2, accept_as2, @@ -750,7 +761,7 @@ class ActivityPubTest(testutil.TestCase): as2=LIKE_WITH_ACTOR, as1=as2.to_as1(LIKE_WITH_ACTOR), type='like', - labels=['notification'], + labels=['notification', 'activity'], object_ids=[LIKE['object']]) def test_followers_collection_unknown_user(self, *args): diff --git a/tests/test_follow.py b/tests/test_follow.py index d636d62..404a099 100644 --- a/tests/test_follow.py +++ b/tests/test_follow.py @@ -180,23 +180,18 @@ class FollowTest(testutil.TestCase): self.assertEqual(('http://bar/inbox',), inbox_args) self.assert_equals(expected_follow, json_loads(inbox_kwargs['data'])) - follow_as2 = json_dumps(expected_follow, sort_keys=True) - follow_as1 = json_dumps(as2.to_as1(expected_follow), sort_keys=True) followers = Follower.query().fetch() self.assert_entities_equal( - Follower(id='https://bar/id snarfed.org', last_follow=follow_as2, + Follower(id='https://bar/id snarfed.org', + last_follow=json_dumps(expected_follow, sort_keys=True), src='snarfed.org', dest='https://bar/id', status='active'), followers, ignore=['created', 'updated']) id = f'http://localhost/user/snarfed.org/following#2022-01-02T03:04:05-{input}' - objects = Object.query().fetch() - self.assert_entities_equal( - [Object(id=id, domains=['snarfed.org'], status='complete', - labels=['user'], source_protocol='ui', - as1=follow_as1, as2=follow_as2)], - objects, - ignore=['created', 'updated']) + self.assert_object(id, domains=['snarfed.org'], status='complete', + labels=['user', 'activity'], source_protocol='ui', + as2=expected_follow, as1=as2.to_as1(expected_follow)) def test_callback_missing_user(self, mock_get, mock_post): mock_post.return_value = requests_response('me=https://snarfed.org') @@ -284,16 +279,11 @@ class UnfollowTest(testutil.TestCase): follower = Follower.get_by_id('https://bar/id snarfed.org') self.assertEqual('inactive', follower.status) - objects = Object.query().fetch() - self.assert_entities_equal( - [Object(id='http://localhost/user/snarfed.org/following#undo-2022-01-02T03:04:05-https://bar/id', - domains=['snarfed.org'], status='complete', - source_protocol='ui', labels=['user'], - as2=json_dumps(UNDO_FOLLOW, sort_keys=True), - as1=json_dumps(as2.to_as1(UNDO_FOLLOW), sort_keys=True), - )], - objects, - ignore=['created', 'updated']) + self.assert_object( + 'http://localhost/user/snarfed.org/following#undo-2022-01-02T03:04:05-https://bar/id', + domains=['snarfed.org'], status='complete', + source_protocol='ui', labels=['user', 'activity'], + as2=UNDO_FOLLOW, as1=as2.to_as1(UNDO_FOLLOW)) def test_callback_missing_user(self, mock_get, mock_post): mock_post.return_value = requests_response('me=https://snarfed.org') diff --git a/tests/test_webmention.py b/tests/test_webmention.py index fa26770..d3f3cb4 100644 --- a/tests/test_webmention.py +++ b/tests/test_webmention.py @@ -493,7 +493,7 @@ class WebmentionTest(testutil.TestCase): domains=['a'], source_protocol='webmention', status='complete', - ap_delivered=['https://foo.com/inbox'], + delivered=['https://foo.com/inbox'], mf2=self.reply_mf2, as1=self.reply_as1, type='comment', @@ -501,8 +501,6 @@ class WebmentionTest(testutil.TestCase): ) def test_update_reply(self, mock_get, mock_post): - # different = copy.deepcopy(self.reply_as1) - # different['content'] += ' other' Object(id='http://a/reply', status='complete', as1='{"content": "other"}').put() mock_get.side_effect = self.activitypub_gets @@ -539,7 +537,7 @@ class WebmentionTest(testutil.TestCase): """https://github.com/snarfed/bridgy-fed/issues/78""" Object(id='http://a/reply', status='complete', as1=json_dumps(self.reply_as1), - ap_delivered=['https://foo.com/inbox']).put() + delivered=['https://foo.com/inbox']).put() mock_get.side_effect = self.activitypub_gets got = self.client.post('/webmention', data={ @@ -621,10 +619,10 @@ class WebmentionTest(testutil.TestCase): status='complete', mf2=self.repost_mf2, as1=self.repost_as1, - ap_delivered=['https://foo.com/inbox'], + delivered=['https://foo.com/inbox'], type='share', object_ids=['https://orig/post'], - labels=['user'], + labels=['user', 'activity'], ) def test_link_rel_alternate_as2(self, mock_get, mock_post): @@ -774,7 +772,7 @@ class WebmentionTest(testutil.TestCase): status='complete', mf2=self.create_mf2, as1=self.create_as1, - ap_delivered=inboxes, + delivered=inboxes, type='note', labels=['user'], ) @@ -785,9 +783,9 @@ class WebmentionTest(testutil.TestCase): Object(id='https://orig/post', domains=['orig'], status='in progress', as1=json_dumps(self.create_as1), - ap_delivered=['https://skipped/inbox'], - ap_undelivered=['https://shared/inbox'], - ap_failed=['https://public/inbox'], + delivered=['https://skipped/inbox'], + undelivered=['https://shared/inbox'], + failed=['https://public/inbox'], ).put() self.make_followers() @@ -816,7 +814,7 @@ class WebmentionTest(testutil.TestCase): status='complete', mf2=self.create_mf2, as1=self.create_as1, - ap_delivered=inboxes + ['https://skipped/inbox'], + delivered=inboxes + ['https://skipped/inbox'], type='note', labels=['user'], ) @@ -830,9 +828,9 @@ class WebmentionTest(testutil.TestCase): Object(id='https://orig/post', domains=['orig'], status='in progress', as1=json_dumps(different_create_as1), - ap_delivered=['https://delivered/inbox'], - ap_undelivered=['https://shared/inbox'], - ap_failed=['https://public/inbox'], + delivered=['https://delivered/inbox'], + undelivered=['https://shared/inbox'], + failed=['https://public/inbox'], ).put() self.make_followers() @@ -858,7 +856,7 @@ class WebmentionTest(testutil.TestCase): status='complete', mf2=self.create_mf2, as1=self.create_as1, - ap_delivered=inboxes, + delivered=inboxes, type='note', labels=['user'], ) @@ -921,10 +919,10 @@ class WebmentionTest(testutil.TestCase): status='complete', mf2=self.follow_mf2, as1=self.follow_as1, - ap_delivered=['https://foo.com/inbox'], + delivered=['https://foo.com/inbox'], type='follow', object_ids=['http://followee'], - labels=['user'], + labels=['user', 'activity'], ) followers = Follower.query().fetch() @@ -988,10 +986,10 @@ class WebmentionTest(testutil.TestCase): status='complete', mf2=self.follow_fragment_mf2, as1=self.follow_fragment_as1, - ap_delivered=['https://foo.com/inbox'], + delivered=['https://foo.com/inbox'], type='follow', object_ids=['http://followee'], - labels=['user'], + labels=['user', 'activity'], ) followers = Follower.query().fetch() @@ -1045,10 +1043,10 @@ class WebmentionTest(testutil.TestCase): status='failed', mf2=self.follow_mf2, as1=self.follow_as1, - ap_failed=['https://foo.com/inbox'], + failed=['https://foo.com/inbox'], type='follow', object_ids=['http://followee'], - labels=['user'], + labels=['user', 'activity'], ) def test_repost_blocklisted_error(self, mock_get, mock_post): @@ -1131,8 +1129,8 @@ class WebmentionTest(testutil.TestCase): status='complete', mf2=ACTOR_MF2, as1=expected_as1, - ap_delivered=['https://inbox', 'https://shared/inbox'], + delivered=['https://inbox', 'https://shared/inbox'], type='update', object_ids=['https://orig'], - labels=['user'], + labels=['user', 'activity'], ) diff --git a/webmention.py b/webmention.py index d02568e..ffd3860 100644 --- a/webmention.py +++ b/webmention.py @@ -140,23 +140,22 @@ class Webmention(View): if obj: logging.info(f'Resuming existing {obj}') - obj.ap_failed = [] - seen = obj.ap_delivered + obj.ap_undelivered + obj.ap_failed + obj.failed = [] + seen = obj.delivered + obj.undelivered + obj.failed new_inboxes = [i for i in inboxes_to_targets.keys() if i not in seen] if new_inboxes: logging.info(f'Adding new inboxes: {new_inboxes}') - obj.ap_undelivered += new_inboxes + obj.undelivered += new_inboxes if type in ('note', 'article', 'comment'): changed = as1.activity_changed(json_loads(obj.as1), self.source_as1) if changed: - obj.ap_undelivered += obj.ap_delivered - obj.ap_delivered = [] - logger.info(f'Content has changed from last time at {obj.updated}! Redelivering to all inboxes: {obj.ap_undelivered}') + obj.undelivered += obj.delivered + obj.delivered = [] + logger.info(f'Content has changed from last time at {obj.updated}! Redelivering to all inboxes: {obj.undelivered}') else: - obj = Object(id=self.source_url, - ap_undelivered=list(inboxes_to_targets.keys()), - ap_delivered=[], ap_failed=[]) + obj = Object(id=self.source_url, undelivered=list(inboxes_to_targets.keys()), + delivered=[], failed=[]) logging.info(f'Storing new {obj}') obj.domains = [self.source_domain] @@ -166,13 +165,15 @@ class Webmention(View): obj.type = type obj.object_ids = as1.get_ids(self.source_as1, 'object') obj.labels = ['user'] + if self.source_as1.get('objectType') == 'activity': + obj.labels.append('activity') obj.put() # TODO: collect by inbox, add 'to' fields, de-dupe inboxes and recipients # - # make copy of ap_undelivered because we modify it below - logger.info(f'Delivering to inboxes: {sorted(obj.ap_undelivered)}') - for inbox in list(obj.ap_undelivered): + # make copy of undelivered because we modify it below + logger.info(f'Delivering to inboxes: {sorted(obj.undelivered)}') + for inbox in list(obj.undelivered): if inbox in inboxes_to_targets: target_as2 = inboxes_to_targets[inbox] else: @@ -207,18 +208,18 @@ class Webmention(View): try: last = common.signed_post(inbox, data=self.source_as2, log_data=log_data, user=self.user) - obj.ap_delivered.append(inbox) + obj.delivered.append(inbox) last_success = last except BaseException as e: - obj.ap_failed.append(inbox) + obj.failed.append(inbox) error = e finally: log_data = False - obj.ap_undelivered.remove(inbox) + obj.undelivered.remove(inbox) obj.put() - obj.status = 'complete' if obj.ap_delivered else 'failed' + obj.status = 'complete' if obj.delivered else 'failed' obj.put() # Pass the AP response status code and body through as our response