diff --git a/activitypub.py b/activitypub.py index 13beea1..0f09254 100644 --- a/activitypub.py +++ b/activitypub.py @@ -600,8 +600,8 @@ def inbox(protocol=None, domain=None): # load user # TODO(#512) parameterize on protocol, move to Protocol if protocol and domain: - g.user = PROTOCOLS[protocol].get_by_id(domain) # receiving user - if (not g.user or not g.user.direct) and actor_id: + g.user = PROTOCOLS[protocol].get_or_create(domain, direct=False) # receiving user + if not g.user.direct and actor_id: # this is a deliberate interaction with an indirect receiving user; # create a local AP User for the sending user actor_obj = ActivityPub.load(actor_id) diff --git a/follow.py b/follow.py index 9cb1fbe..c5c81b2 100644 --- a/follow.py +++ b/follow.py @@ -95,7 +95,6 @@ class FollowCallback(indieauth.Callback): g.user = Web.get_by_id(domain) if not g.user: error(f'No web user for domain {domain}') - domain = g.user.key.id() addr = state if not state: @@ -135,11 +134,12 @@ class FollowCallback(indieauth.Callback): 'actor': g.user.ap_actor(), 'to': [as2.PUBLIC_AUDIENCE], } - follow_obj = Object(id=follow_id, domains=[domain], labels=['user'], - source_protocol='ui', status='complete', as2=follow_as2) + followee_user = ActivityPub.get_or_create(followee_id, actor_as2=followee.as2) + follow_obj = Object(id=follow_id, users=[g.user.key, followee_user.key], + labels=['user'], source_protocol='ui', status='complete', + as2=follow_as2) ActivityPub.send(follow_obj, inbox) - followee_user = ActivityPub.get_or_create(followee_id, actor_as2=followee.as2) Follower.get_or_create(from_=g.user, to=followee_user, status='active', follow=follow_obj.key) follow_obj.put() @@ -187,7 +187,6 @@ class UnfollowCallback(indieauth.Callback): g.user = Web.get_by_id(domain) if not g.user: error(f'No web user for domain {domain}') - domain = g.user.key.id() if util.is_int(state): state = int(state) @@ -219,7 +218,10 @@ class UnfollowCallback(indieauth.Callback): 'object': follower.follow.get().as2 if follower.follow else None, } - obj = Object(id=unfollow_id, domains=[domain], labels=['user'], + # don't include the followee User who's being unfollowed in the users + # property, since we don't want to notify or show them. (standard social + # network etiquette.) + obj = Object(id=unfollow_id, users=[g.user.key], labels=['user'], source_protocol='ui', status='complete', as2=unfollow_as2) ActivityPub.send(obj, inbox) diff --git a/index.yaml b/index.yaml index 1193ed4..79fad6a 100644 --- a/index.yaml +++ b/index.yaml @@ -13,6 +13,13 @@ indexes: - name: updated direction: asc +- kind: Object + properties: + - name: users + - name: labels + - name: updated + direction: asc + - kind: Object properties: - name: domains @@ -20,6 +27,13 @@ indexes: - name: updated direction: desc +- kind: Object + properties: + - name: users + - name: labels + - name: updated + direction: desc + - kind: Object properties: - name: domains @@ -27,6 +41,13 @@ indexes: - name: created direction: desc +- kind: Object + properties: + - name: users + - name: labels + - name: created + direction: desc + - kind: Follower properties: - name: from diff --git a/models.py b/models.py index 8ed8c9e..222f387 100644 --- a/models.py +++ b/models.py @@ -301,8 +301,11 @@ class Object(StringIdModel): STATUSES = ('new', 'in progress', 'complete', 'failed', 'ignored') LABELS = ('activity', 'feed', 'notification', 'user') - # domains of the Bridgy Fed users this activity is to or from + # Users this activity is to or from + users = ndb.KeyProperty(repeated=True) + # DEPRECATED domains = ndb.StringProperty(repeated=True) + status = ndb.StringProperty(choices=STATUSES) # choices is populated in flask_app, after all User subclasses are created, # so that PROTOCOLS is fully populated @@ -383,7 +386,7 @@ class Object(StringIdModel): def _post_put_hook(self, future): """Update :meth:`Protocol.load` cache.""" # TODO: assert that as1 id is same as key id? in pre put hook? - logger.info(f'Wrote Object {self.key.id()} {self.type} {self.status or ""} {self.labels} for {len(self.domains)} users') + logger.info(f'Wrote Object {self.key.id()} {self.type} {self.status or ""} {self.labels} for {len(self.users)} users {len(self.domains)} domains') if '#' not in self.key.id(): import protocol # TODO: actually fix this circular import protocol.objects_cache[self.key.id()] = self @@ -423,7 +426,7 @@ class Object(StringIdModel): attrs = {'class': 'h-card u-author'} if (self.source_protocol in ('web', 'webmention', 'ui') and g.user and - g.user.key.id() in self.domains): + (g.user.key in self.users or g.user.key.id() in self.domains)): # outbound; show a nice link to the user return g.user.user_page_link() diff --git a/pages.py b/pages.py index b037091..dbb8557 100644 --- a/pages.py +++ b/pages.py @@ -8,6 +8,7 @@ import urllib.parse from flask import g, redirect, render_template, request from google.cloud.ndb.model import get_multi +from google.cloud.ndb.query import OR from google.cloud.ndb.stats import KindStat from granary import as1, as2, atom, microformats2, rss import humanize @@ -95,7 +96,8 @@ def user(protocol, id): load_user(protocol, id) query = Object.query( - Object.domains == id, + OR(Object.users == g.user.key, + Object.domains == id), Object.labels.IN(('notification', 'user')), ) objects, before, after = fetch_objects(query) @@ -144,12 +146,12 @@ def feed(protocol, id): load_user(protocol, id) - assert not g.user.use_instead - - objects, _, _ = Object.query( - Object.domains == id, Object.labels == 'feed') \ + objects = Object.query( + OR(Object.users == g.user.key, + Object.domains == id), + Object.labels == 'feed') \ .order(-Object.created) \ - .fetch_page(PAGE_SIZE) + .fetch(PAGE_SIZE) activities = [obj.as1 for obj in objects if not obj.deleted] actor = { @@ -226,12 +228,13 @@ def fetch_objects(query): urls = as1.object_urls(inner_obj) id = common.redirect_unwrap(inner_obj.get('id', '')) url = urls[0] if urls else id - if (type == 'update' and obj.domains and - id.strip('/') == f'https://{obj.domains[0]}'): + if (type == 'update' and + (obj.users and id.strip('/') == obj.users[0].id() + or obj.domains and id.strip('/') == f'https://{obj.domains[0]}')): obj.phrase = 'updated' obj_as1.update({ 'content': 'their profile', - 'url': f'https://{obj.domains[0]}', + 'url': id, }) elif url: content = common.pretty_link(url, text=content) diff --git a/protocol.py b/protocol.py index cc7f17e..85656e2 100644 --- a/protocol.py +++ b/protocol.py @@ -251,9 +251,9 @@ class Protocol: from activitypub import ActivityPub for f in Follower.query(Follower.to == ActivityPub(id=actor_id).key, Follower.status == 'active'): - if f.from_.id() not in obj.domains: - obj.domains.append(f.from_.id()) - if obj.domains and 'feed' not in obj.labels: + if f.from_ not in obj.users: + obj.users.append(f.from_) + if obj.users and 'feed' not in obj.labels: obj.labels.append('feed') obj.put() @@ -345,6 +345,7 @@ class Protocol: # send webmentions and update Object errors = [] # stores (code, body) tuples targets = [Target(uri=uri, protocol='web') for uri in targets] + no_user_domains = set() obj.populate( undelivered=targets, @@ -362,12 +363,25 @@ class Protocol: logger.info(f'Skipping same-domain webmention from {source} to {target.uri}') continue - if domain not in obj.domains: - obj.domains.append(domain) + # only deliver if we have a matching User already. + # TODO: consider delivering or at least storing Users for all + # targets? need to filter out native targets in this protocol + # though, eg mastodon.social targets in AP inbox deliveries. + if domain in no_user_domains: + continue + + # TODO(#512): generalize protocol + from web import Web + recip = Web(id=domain).key + if recip not in obj.users: + if not recip.get(): + logger.info(f'No Web user for {domain}; skipping {target.uri}') + no_user_domains.add(domain) + continue + obj.users.append(recip) try: - # TODO: fix - from web import Web + # TODO(#512): generalize protocol if Web.send(obj, target.uri): obj.delivered.append(target) if 'notification' not in obj.labels: @@ -381,7 +395,7 @@ class Protocol: obj.put() - obj.status = ('complete' if obj.delivered or obj.domains + obj.status = ('complete' if obj.delivered or obj.users else 'failed' if obj.failed else 'ignored') diff --git a/tests/test_activitypub.py b/tests/test_activitypub.py index 0e49007..816db41 100644 --- a/tests/test_activitypub.py +++ b/tests/test_activitypub.py @@ -429,7 +429,7 @@ class ActivityPubTest(TestCase): + [WEBMENTION_DISCOVERY]) mock_post.return_value = requests_response() - got = self.post('/user.com/inbox', json=reply) + got = self.post('/ap/web/user.com/inbox', json=reply) self.assertEqual(200, got.status_code, got.get_data(as_text=True)) self.assert_req(mock_get, 'https://user.com/post') convert_id = reply['id'].replace('://', ':/') @@ -445,7 +445,7 @@ class ActivityPubTest(TestCase): ) self.assert_object(reply['id'], - domains=['user.com'], + users=[self.user.key], source_protocol='activitypub', status='complete', delivered=['https://user.com/post'], @@ -501,7 +501,7 @@ class ActivityPubTest(TestCase): self.assert_object('http://mas.to/note/as2', source_protocol='activitypub', as2=expected_as2, - domains=['user.com', 'baz.com'], + users=[self.user.key, Fake(id='baz.com').key], type='post', labels=['activity', 'feed'], object_ids=[NOTE_OBJECT['id']]) @@ -548,7 +548,7 @@ class ActivityPubTest(TestCase): source_protocol='activitypub', status='complete', as2=repost, - domains=['user.com'], + users=[self.user.key], delivered=['https://user.com/orig'], type='share', labels=['activity', 'feed', 'notification'], @@ -579,7 +579,7 @@ class ActivityPubTest(TestCase): source_protocol='activitypub', status='ignored', as2=REPOST_FULL, - domains=['user.com', 'baz.com'], + users=[self.user.key, Fake(id='baz.com').key], type='share', labels=['activity', 'feed'], object_ids=[REPOST['object']]) @@ -596,9 +596,10 @@ class ActivityPubTest(TestCase): self.assertEqual(200, got.status_code) self.assert_object('http://mas.to/like#ok', - domains=['nope.com'], + # no nope.com Web user key since it didn't exist + users=[], source_protocol='activitypub', - status='complete', + status='ignored', as2={**LIKE_WITH_ACTOR, 'object': 'http://nope.com/post'}, type='like', labels=['activity'], @@ -650,6 +651,8 @@ class ActivityPubTest(TestCase): type='note') def _test_inbox_mention(self, mention, expected_props, mock_head, mock_get, mock_post): + self.make_user('tar.get') + mock_get.side_effect = [ WEBMENTION_DISCOVERY, HTML, @@ -673,7 +676,7 @@ class ActivityPubTest(TestCase): expected_as2 = common.redirect_unwrap(mention) self.assert_object(mention['id'], - domains=['tar.get', 'masto.foo'], + users=[Web(id='tar.get').key], source_protocol='activitypub', status='complete', as2=expected_as2, @@ -705,7 +708,7 @@ class ActivityPubTest(TestCase): }, kwargs['data']) self.assert_object('http://mas.to/like#ok', - domains=['user.com'], + users=[self.user.key], source_protocol='activitypub', status='complete', as2=LIKE_WITH_ACTOR, @@ -733,7 +736,7 @@ class ActivityPubTest(TestCase): 'url': 'https://mas.to/users/swentel#followed-https://user.com/', } self.assert_object('https://mas.to/6d1a', - domains=['user.com'], + users=[self.user.key], source_protocol='activitypub', status='complete', as2=follow, @@ -765,7 +768,7 @@ class ActivityPubTest(TestCase): 'url': 'https://mas.to/users/swentel#followed-https://user.com/', }) self.assert_object('https://mas.to/6d1a', - domains=['user.com'], + users=[self.user.key], source_protocol='activitypub', status='complete', as2=follow, @@ -787,7 +790,7 @@ class ActivityPubTest(TestCase): 'url': 'https://mas.to/users/swentel#followed-https://user.com/', } self.assert_object('https://mas.to/6d1a', - domains=['user.com'], + users=[self.user.key], source_protocol='activitypub', status='complete', as2=follow, @@ -980,6 +983,7 @@ class ActivityPubTest(TestCase): obj = Object.get_by_id(id) self.assertEqual(['activity'], obj.labels) + self.assertEqual([], obj.users) self.assertEqual([], obj.domains) self.assertIsNone(Object.get_by_id(bad_url)) @@ -1159,7 +1163,7 @@ class ActivityPubTest(TestCase): self.assertEqual(200, got.status_code) self.assert_object('http://mas.to/like#ok', - domains=['user.com'], + users=[self.user.key], source_protocol='activitypub', status='complete', as2=LIKE_WITH_ACTOR, diff --git a/tests/test_follow.py b/tests/test_follow.py index d48d878..6e39485 100644 --- a/tests/test_follow.py +++ b/tests/test_follow.py @@ -210,15 +210,17 @@ class FollowTest(TestCase): follow_id = f'http://localhost/web/alice.com/following#2022-01-02T03:04:05-{input}' followers = Follower.query().fetch() + followee = ActivityPub(id='https://bar/id').key self.assert_entities_equal( - Follower(from_=self.user.key, to=ActivityPub(id='https://bar/id').key, + Follower(from_=self.user.key, to=followee, follow=Object(id=follow_id).key, status='active'), followers, ignore=['created', 'updated']) - self.assert_object(follow_id, domains=['alice.com'], status='complete', - labels=['user', 'activity'], source_protocol='ui', - as2=expected_follow, as1=as2.to_as1(expected_follow)) + self.assert_object(follow_id, users=[self.user.key, followee], + status='complete', labels=['user', 'activity'], + source_protocol='ui', as2=expected_follow, + as1=as2.to_as1(expected_follow)) self.assertEqual('https://alice.com', session['indieauthed-me']) @@ -258,15 +260,15 @@ class FollowTest(TestCase): 'object': FOLLOWEE, 'to': [as2.PUBLIC_AUDIENCE], } + followee = ActivityPub(id='https://bar/id').key follow_obj = self.assert_object( - id, domains=['www.alice.com'], status='complete', + id, users=[user.key, followee], status='complete', labels=['user', 'activity'], source_protocol='ui', as2=expected_follow, as1=as2.to_as1(expected_follow)) followers = Follower.query().fetch() self.assert_entities_equal( - Follower(from_=user.key, to=ActivityPub(id='https://bar/id').key, - follow=follow_obj.key, status='active'), + Follower(from_=user.key, to=followee, follow=follow_obj.key, status='active'), followers, ignore=['created', 'updated']) @@ -395,10 +397,8 @@ class UnfollowTest(TestCase): self.assert_object( 'http://localhost/web/alice.com/following#undo-2022-01-02T03:04:05-https://bar/id', - domains=['alice.com'], status='complete', - source_protocol='ui', labels=['user', 'activity'], - as2=expected_undo, - as1=as2.to_as1(expected_undo)) + users=[self.user.key], status='complete', source_protocol='ui', + labels=['user', 'activity'], as2=expected_undo, as1=as2.to_as1(expected_undo)) self.assertEqual('https://alice.com', session['indieauthed-me']) @@ -448,7 +448,7 @@ class UnfollowTest(TestCase): follower = Follower.query().get() self.assertEqual('inactive', follower.status) - self.assert_object(id, domains=['www.alice.com'], status='complete', + self.assert_object(id, users=[user.key], status='complete', source_protocol='ui', labels=['user', 'activity'], as2=expected_undo, as1=as2.to_as1(expected_undo)) diff --git a/tests/test_models.py b/tests/test_models.py index 277604c..b0337c2 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -164,7 +164,7 @@ class ObjectTest(TestCase): def test_actor_link_user(self): g.user = Fake(id='user.com', actor_as2={"name": "Alice"}) - obj = Object(id='x', source_protocol='ui', domains=['user.com']) + obj = Object(id='x', source_protocol='ui', users=[g.user.key]) self.assertIn( 'href="/fake/user.com"> Alice', obj.actor_link()) diff --git a/tests/test_pages.py b/tests/test_pages.py index fc50af8..4c0cb99 100644 --- a/tests/test_pages.py +++ b/tests/test_pages.py @@ -110,7 +110,7 @@ class PagesTest(TestCase): def test_user_object_bare_string_id(self): with self.request_context: - Object(id='a', domains=['user.com'], labels=['notification'], + Object(id='a', users=[self.user.key], labels=['notification'], as2=REPOST_AS2).put() got = self.client.get('/web/user.com') @@ -118,7 +118,7 @@ class PagesTest(TestCase): def test_user_object_url_object(self): with self.request_context: - Object(id='a', domains=['user.com'], labels=['notification'], our_as1={ + Object(id='a', users=[self.user.key], labels=['notification'], our_as1={ **REPOST_AS2, 'object': { 'id': 'https://mas.to/toot/id', diff --git a/tests/test_protocol.py b/tests/test_protocol.py index f2c028c..d6fc819 100644 --- a/tests/test_protocol.py +++ b/tests/test_protocol.py @@ -45,6 +45,7 @@ class ProtocolTest(TestCase): def test_receive_reply_not_feed_not_notification(self, mock_get): Follower.get_or_create(to=Fake.get_or_create(id=ACTOR['id']), from_=Fake.get_or_create(id='foo.com')) + other_user = self.make_user('user.com', cls=Web) # user.com webmention discovery mock_get.return_value = requests_response('') @@ -54,7 +55,7 @@ class ProtocolTest(TestCase): self.assert_object(REPLY['id'], as2=REPLY, type='post', - domains=['user.com'], + users=[other_user.key], # not feed since it's a reply # not notification since it doesn't involve the user labels=['activity'], diff --git a/tests/test_web.py b/tests/test_web.py index 88ad020..49250f6 100644 --- a/tests/test_web.py +++ b/tests/test_web.py @@ -606,14 +606,14 @@ class WebTest(TestCase): self.assert_deliveries(mock_post, ['https://mas.to/inbox'], AS2_CREATE) self.assert_object('https://user.com/reply', - domains=['user.com'], + users=[g.user.key], source_protocol='web', mf2=REPLY_MF2, as1=REPLY_AS1, type='comment', ) self.assert_object('https://user.com/reply#bridgy-fed-create', - domains=['user.com'], + users=[g.user.key], source_protocol='web', status='complete', mf2=REPLY_MF2, @@ -767,7 +767,7 @@ class WebTest(TestCase): mf2 = util.parse_mf2(html)['items'][0] self.assert_object('https://user.com/repost', - domains=['user.com'], + users=[g.user.key], source_protocol='web', status='complete', mf2=mf2, @@ -825,7 +825,7 @@ class WebTest(TestCase): mock_post.assert_not_called() self.assert_object('https://user.com/like', - domains=['user.com'], + users=[g.user.key], source_protocol='web', mf2=LIKE_MF2, as1=microformats2.json_to_object(LIKE_MF2), @@ -930,13 +930,13 @@ class WebTest(TestCase): self.assert_deliveries(mock_post, inboxes, CREATE_AS2) self.assert_object('https://user.com/post', - domains=['user.com'], + users=[g.user.key], mf2=NOTE_MF2, type='note', source_protocol='web', ) self.assert_object('https://user.com/post#bridgy-fed-create', - domains=['user.com'], + users=[g.user.key], source_protocol='web', status='complete', mf2=NOTE_MF2, @@ -953,7 +953,7 @@ class WebTest(TestCase): with self.request_context: mf2 = copy.deepcopy(NOTE_MF2) mf2['properties']['content'] = 'different' - Object(id='https://user.com/post', domains=['user.com'], mf2=mf2).put() + Object(id='https://user.com/post', users=[g.user.key], mf2=mf2).put() self.make_followers() @@ -981,7 +981,7 @@ class WebTest(TestCase): } self.assert_object( f'https://user.com/post#bridgy-fed-update-2022-01-02T03:04:05+00:00', - domains=['user.com'], + users=[g.user.key], source_protocol='web', status='complete', mf2=NOTE_MF2, @@ -1036,7 +1036,7 @@ class WebTest(TestCase): self.assert_deliveries(mock_post, ['https://mas.to/inbox'], FOLLOW_AS2) obj = self.assert_object('https://user.com/follow', - domains=['user.com'], + users=[g.user.key], source_protocol='web', status='complete', mf2=FOLLOW_MF2, @@ -1119,7 +1119,7 @@ class WebTest(TestCase): FOLLOW_FRAGMENT_AS2) obj = self.assert_object('https://user.com/follow#2', - domains=['user.com'], + users=[g.user.key], source_protocol='web', status='complete', mf2=FOLLOW_FRAGMENT_MF2, @@ -1180,7 +1180,7 @@ class WebTest(TestCase): mf2 = util.parse_mf2(html)['items'][0] as1 = microformats2.json_to_object(mf2) obj = self.assert_object('https://user.com/follow', - domains=['user.com'], + users=[g.user.key], source_protocol='web', status='complete', mf2=mf2, @@ -1239,7 +1239,7 @@ class WebTest(TestCase): self.assert_deliveries(mock_post, inboxes, DELETE_AS2) self.assert_object('https://user.com/post#bridgy-fed-delete', - domains=['user.com'], + users=[g.user.key], source_protocol='web', status='complete', our_as1=DELETE_AS1, @@ -1298,7 +1298,7 @@ class WebTest(TestCase): self.assert_deliveries(mock_post, ['https://mas.to/inbox'], FOLLOW_AS2) self.assert_object('https://user.com/follow', - domains=['user.com'], + users=[g.user.key], source_protocol='web', status='failed', mf2=FOLLOW_MF2, @@ -1386,7 +1386,7 @@ class WebTest(TestCase): }, } self.assert_object(id, - domains=['user.com'], + users=[g.user.key], source_protocol='web', status='complete', our_as1=expected_as1, diff --git a/web.py b/web.py index 474e894..044cdcd 100644 --- a/web.py +++ b/web.py @@ -482,7 +482,7 @@ def webmention_task(): inboxes_to_targets = _activitypub_targets(obj) obj.populate( - domains=[g.user.key.id()], + users=[g.user.key], source_protocol='web', ) if not inboxes_to_targets: @@ -519,7 +519,7 @@ def webmention_task(): }, } obj = Object(id=id, mf2=obj.mf2, our_as1=update_as1, labels=['user'], - domains=[g.user.key.id()], source_protocol='web') + users=[g.user.key], source_protocol='web') elif obj.new or 'force' in request.form: logger.info(f'New Object {obj.key.id()}') @@ -534,7 +534,7 @@ def webmention_task(): 'object': obj.as1, } obj = Object(id=id, mf2=obj.mf2, our_as1=create_as1, - domains=[g.user.key.id()], labels=['user'], + users=[g.user.key], labels=['user'], source_protocol='web') else: