switch from Object.users + labels to users + notify + feed lists

needed to distinguish an activity's owners from who it should notify from who should see it in their feeds.

also, unrelated, start sending stop-following activities.

in progress, test_web and test_activitypub still need updating.
pull/590/head
Ryan Barrett 2023-07-16 14:06:03 -07:00
rodzic 764494be16
commit 57350ab81a
Nie znaleziono w bazie danych klucza dla tego podpisu
ID klucza GPG: 6BE31FDF4776E9D4
6 zmienionych plików z 176 dodań i 103 usunięć

Wyświetl plik

@ -353,10 +353,22 @@ class Object(StringIdModel):
Key name is the id. We synthesize ids if necessary.
"""
STATUSES = ('new', 'in progress', 'complete', 'failed', 'ignored')
LABELS = ('activity', 'feed', 'notification', 'user')
LABELS = ('activity',
# DEPRECATED, replaced by users, notify, feed
'feed', 'notification', 'user')
# Users this activity is to or from
# Keys for user(s) who created or otherwise own this activity.
#
# DEPRECATED: this used to include all users related the activity, including
# followers, but we've now moved those to the notify and feed properties.
users = ndb.KeyProperty(repeated=True)
# User keys who should see this activity in their user page, eg in reply to,
# reaction to, share of, etc.
notify = ndb.KeyProperty(repeated=True)
# User keys who should see this activity in their feeds, eg followers of its
# creator
feed = ndb.KeyProperty(repeated=True)
# DEPRECATED but still used read only to maintain backward compatibility
# with old Objects in the datastore that we haven't bothered migrating.
domains = ndb.StringProperty(repeated=True)

Wyświetl plik

@ -4,7 +4,7 @@ import logging
import os
from flask import g, render_template, request
from google.cloud.ndb.query import OR
from google.cloud.ndb.query import AND, OR
from google.cloud.ndb.stats import KindStat
from granary import as1, as2, atom, microformats2, rss
import humanize
@ -89,11 +89,8 @@ def web_user_redirects(**kwargs):
def user(protocol, id):
load_user(protocol, id)
query = Object.query(
OR(Object.users == g.user.key,
Object.domains == id),
Object.labels.IN(('notification', 'user')),
)
query = Object.query(OR(Object.users == g.user.key,
Object.notify == g.user.key))
objects, before, after = fetch_objects(query)
followers = Follower.query(Follower.to == g.user.key,
@ -140,12 +137,12 @@ def feed(protocol, id):
load_user(protocol, id)
objects = Object.query(
OR(Object.users == g.user.key,
Object.domains == id),
Object.labels == 'feed') \
.order(-Object.created) \
.fetch(PAGE_SIZE)
objects = Object.query(OR(Object.feed == g.user.key,
# backward compatibility
AND(Object.users == g.user.key,
Object.labels == 'feed'))) \
.order(-Object.created) \
.fetch(PAGE_SIZE)
activities = [obj.as1 for obj in objects if not obj.deleted]
actor = {

Wyświetl plik

@ -409,7 +409,10 @@ class Protocol:
# if this is a post, ie not an activity, wrap it in a create or update
obj = cls._handle_bare_object(obj)
# add involved users
if obj.type not in SUPPORTED_TYPES:
error(f'Sorry, {obj.type} activities are not supported yet.', status=501)
# add owner(s)
actor_key = cls.actor_key(obj, default_g_user=False)
if actor_key:
add(obj.users, actor_key)
@ -423,9 +426,6 @@ class Protocol:
obj.source_protocol = cls.LABEL
obj.put()
if obj.type not in SUPPORTED_TYPES:
error(f'Sorry, {obj.type} activities are not supported yet.', status=501)
# store inner object
inner_obj_id = inner_obj_as1.get('id')
inner_obj = None
@ -441,6 +441,8 @@ class Protocol:
return 'OK' # noop
elif obj.type == 'stop-following':
# TODO: unify with _handle_follow?
# TODO: handle multiple followees
if not actor_id or not inner_obj_id:
error(f'Undo of Follow requires actor id and object id. Got: {actor_id} {inner_obj_id} {obj.as1}')
@ -448,7 +450,8 @@ class Protocol:
# TODO: avoid import?
from web import Web
from_ = cls.key_for(actor_id)
to = (Protocol.for_id(inner_obj_id) or Web).key_for(inner_obj_id)
to_cls = Protocol.for_id(inner_obj_id) or Web
to = to_cls.key_for(inner_obj_id)
follower = Follower.query(Follower.to == to,
Follower.from_ == from_,
Follower.status == 'active').get()
@ -459,15 +462,14 @@ class Protocol:
else:
logger.warning(f'No Follower found for {from_} => {to}')
# TODO send webmention with 410 of u-follow
obj.status = 'complete'
obj.put()
return 'OK'
# fall through to deliver to followee
# TODO: do we convert stop-following to webmention 410 of original
# follow?
elif obj.type == 'update':
if not inner_obj_id:
error("Couldn't find id of object to update")
# fall through to deliver to followers
elif obj.type == 'delete':
@ -573,8 +575,8 @@ class Protocol:
to_obj.put()
# If followee user is already direct, follower may not know they're
# interacting with a bridge. f followee user is indirect though,
# follower should know, so the're direct.
# interacting with a bridge. if followee user is indirect though,
# follower should know, so they're direct.
to_key = to_cls.key_for(to_id)
to_user = to_cls.get_or_create(id=to_key.id(), obj=to_obj, direct=False)
@ -584,9 +586,7 @@ class Protocol:
direct=not to_user.direct)
follower_obj = Follower.get_or_create(to=to_user, from_=from_user,
follow=obj.key, status='active')
add(obj.users, to_key)
add(obj.labels, 'notification')
add(obj.notify, to_key)
# send accept. note that this is one accept for the whole follow, even
# if it has multiple followees!
@ -676,8 +676,6 @@ class Protocol:
Args:
obj: :class:`Object`, activity to deliver
"""
add(obj.labels, 'user')
# find delivery targets
# sort targets so order is deterministic for tests, debugging, etc
targets = cls._targets(obj) # maps Target to Object or None
@ -782,13 +780,12 @@ class Protocol:
orig_user = protocol.actor_key(orig_obj, default_g_user=False)
if orig_user:
logger.info(f'Recipient is {orig_user}')
add(obj.users, orig_user)
add(obj.labels, 'notification')
add(obj.notify, orig_user)
logger.info(f'Direct targets: {candidates}')
logger.info(f'Direct targets: {targets.keys()}')
# deliver to followers?
user_key = cls.actor_key(obj)
# deliver to followers, if appropriate
user_key = cls.actor_key(obj, default_g_user=False)
if not user_key:
logger.info("Can't tell who this is from! Skipping followers.")
return targets
@ -802,10 +799,19 @@ class Protocol:
).fetch()
users = [u for u in ndb.get_multi(f.from_ for f in followers) if u]
User.load_multi(users)
if obj.type not in ('update', 'delete'):
for u in users:
add(obj.users, u.key)
add(obj.labels, 'feed')
# which object should we add to followers' feeds, if any
feed_obj = None
if obj.type == 'share':
feed_obj = obj
else:
inner = as1.get_object(obj.as1)
# don't add profile updates to feeds
if not (obj.type == 'update'
and inner.get('objectType') in as1.ACTOR_TYPES):
inner_id = inner.get('id')
if inner_id:
feed_obj = cls.load(inner_id)
for user in users:
# TODO: should we pass remote=False through here to Protocol.load?
@ -823,6 +829,13 @@ class Protocol:
targets[Target(protocol=user.LABEL, uri=target)] = \
orig_obj if obj.as1.get('verb') == 'share' else None
if feed_obj:
add(feed_obj.feed, user.key)
if feed_obj:
feed_obj.put()
# de-dupe targets, discard same-domain and blocklisted
candidates = {t.uri: (t, obj) for t, obj in targets.items()}
targets = {}

Wyświetl plik

@ -3,6 +3,7 @@
from arroba.mst import dag_cbor_cid
from Crypto.PublicKey import ECC
from flask import g
from google.cloud import ndb
from granary.tests.test_bluesky import ACTOR_PROFILE_BSKY
from oauth_dropins.webutil.testutil import NOW
@ -168,15 +169,16 @@ class ObjectTest(TestCase):
self.assertEqual(0, Object.query().count())
user = ndb.Key(Web, 'user.com')
obj = Object.get_or_create('foo', our_as1={'content': 'foo'},
source_protocol='ui', labels=['notification'])
source_protocol='ui', notify=[user])
check([obj], Object.query().fetch())
self.assertTrue(obj.new)
self.assertIsNone(obj.changed)
self.assertEqual('foo', obj.key.id())
self.assertEqual({'content': 'foo', 'id': 'foo'}, obj.as1)
self.assertEqual('ui', obj.source_protocol)
self.assertEqual(['notification'], obj.labels)
self.assertEqual([user], obj.notify)
obj2 = Object.get_or_create('foo')
self.assertFalse(obj2.new)
@ -186,11 +188,11 @@ class ObjectTest(TestCase):
# non-null **props should be populated
obj3 = Object.get_or_create('foo', our_as1={'content': 'bar'},
source_protocol=None, labels=[])
source_protocol=None, notify=[])
self.assertEqual('foo', obj3.key.id())
self.assertEqual({'content': 'bar', 'id': 'foo'}, obj3.as1)
self.assertEqual('ui', obj3.source_protocol)
self.assertEqual(['notification'], obj3.labels)
self.assertEqual([user], obj3.notify)
self.assertFalse(obj3.new)
self.assertTrue(obj3.changed)
check([obj3], Object.query().fetch())
@ -206,7 +208,7 @@ class ObjectTest(TestCase):
self.assertTrue(obj5.new)
self.assertIsNone(obj5.changed)
obj6 = Object.get_or_create('baz', labels=['feed'])
obj6 = Object.get_or_create('baz', notify=[ndb.Key(Web, 'other')])
self.assertTrue(obj6.new)
self.assertIsNone(obj6.changed)

Wyświetl plik

@ -311,14 +311,15 @@ class ProtocolReceiveTest(TestCase):
self.assert_object('fake:post',
our_as1=post_as1,
type='note',
feed=[self.alice.key, self.bob.key],
)
obj = self.assert_object('fake:create',
status='complete',
our_as1=create_as1,
delivered=['shared:target'],
type='post',
labels=['user', 'activity', 'feed'],
users=[g.user.key, self.alice.key, self.bob.key],
users=[g.user.key],
notify=[],
)
self.assertEqual([(obj, 'shared:target')], Fake.sent)
@ -336,6 +337,7 @@ class ProtocolReceiveTest(TestCase):
self.assert_object('fake:post',
our_as1=post_as1,
type='note',
feed=[self.alice.key, self.bob.key],
)
obj = self.assert_object('fake:post#bridgy-fed-create',
@ -350,8 +352,8 @@ class ProtocolReceiveTest(TestCase):
},
delivered=['shared:target'],
type='post',
labels=['user', 'activity', 'feed'],
users=[g.user.key, self.alice.key, self.bob.key],
users=[g.user.key],
notify=[],
)
self.assertEqual([(obj, 'shared:target')], Fake.sent)
@ -377,14 +379,15 @@ class ProtocolReceiveTest(TestCase):
self.assert_object('fake:post',
our_as1=post_as1,
type='note',
feed=[self.alice.key, self.bob.key],
)
obj = self.assert_object('fake:update',
status='complete',
our_as1=update_as1,
delivered=['shared:target'],
type='update',
labels=['user', 'activity'],
users=[g.user.key],
notify=[],
)
self.assertEqual([(obj, 'shared:target')], Fake.sent)
@ -392,6 +395,7 @@ class ProtocolReceiveTest(TestCase):
def test_update_post_bare_object(self):
self.make_followers()
# post has no author
post_as1 = {
'id': 'fake:post',
'objectType': 'note',
@ -401,30 +405,33 @@ class ProtocolReceiveTest(TestCase):
existing = Object.get_by_id('fake:post')
post_as1['content'] = 'second'
self.assertEqual('OK', Fake.receive(post_as1))
with self.assertRaises(NoContent):
Fake.receive(post_as1)
post_as1['updated'] = '2022-01-02T03:04:05+00:00'
self.assert_object('fake:post',
our_as1=post_as1,
type='note',
feed=[],
)
update_id = 'fake:post#bridgy-fed-update-2022-01-02T03:04:05+00:00'
obj = self.assert_object(update_id,
status='complete',
status='ignored',
our_as1={
'objectType': 'activity',
'verb': 'update',
'id': update_id,
'object': post_as1,
},
delivered=['shared:target'],
delivered=[],
type='update',
labels=['user', 'activity'],
# post has no author
users=[],
notify=[],
)
self.assertEqual([(obj, 'shared:target')], Fake.sent)
self.assertEqual([], Fake.sent)
def test_create_reply(self):
self.make_followers()
@ -457,8 +464,8 @@ class ProtocolReceiveTest(TestCase):
our_as1=create_as1,
delivered=['fake:post:target'],
type='post',
labels=['user', 'activity', 'notification'],
users=[g.user.key, self.bob.key, self.alice.key],
users=[g.user.key, self.alice.key],
notify=[self.bob.key],
)
self.assertEqual([(obj, 'fake:post:target')], Fake.sent)
@ -497,8 +504,8 @@ class ProtocolReceiveTest(TestCase):
our_as1=create_as1,
delivered=['fake:post:target'],
type='post',
labels=['user', 'activity', 'notification'],
users=[self.alice.key, self.bob.key],
users=[self.alice.key],
notify=[self.bob.key],
)
self.assertEqual([(obj, 'fake:post:target')], Fake.sent)
@ -536,8 +543,8 @@ class ProtocolReceiveTest(TestCase):
our_as1=update_as1,
delivered=['fake:post:target'],
type='update',
labels=['user', 'activity', 'notification'],
users=[g.user.key, self.alice.key, self.bob.key],
users=[g.user.key, self.alice.key],
notify=[self.bob.key],
)
self.assertEqual([(obj, 'fake:post:target')], Fake.sent)
@ -575,7 +582,6 @@ class ProtocolReceiveTest(TestCase):
# users=[Fake(id='fake:eve').key],
# # not feed since it's a reply
# # not notification since it doesn't involve the user
# labels=['notification', 'user'],
# delivered=['fake:post:target'],
# status='complete',
# )
@ -610,8 +616,9 @@ class ProtocolReceiveTest(TestCase):
},
delivered=['fake:post:target', 'shared:target'],
type='share',
labels=['user', 'activity', 'notification', 'feed'],
users=[g.user.key, self.alice.key, self.bob.key],
users=[g.user.key],
notify=[self.bob.key],
feed=[self.alice.key, self.bob.key],
)
self.assertEqual([
(obj, 'fake:post:target'),
@ -641,7 +648,6 @@ class ProtocolReceiveTest(TestCase):
our_as1=repost_as1,
delivered=[],
type='share',
labels=['user', 'activity', 'feed'],
users=[g.user.key],
)
self.assertEqual([], Fake.sent)
@ -649,6 +655,7 @@ class ProtocolReceiveTest(TestCase):
def test_like(self):
Fake.fetchable['fake:post'] = {
'objectType': 'note',
'author': 'fake:bob',
}
like_as1 = {
@ -662,11 +669,11 @@ class ProtocolReceiveTest(TestCase):
like_obj = self.assert_object('fake:like',
users=[g.user.key],
notify=[self.bob.key],
status='complete',
our_as1=like_as1,
delivered=['fake:post:target'],
type='like',
labels=['user', 'activity'],
object_ids=['fake:post'])
self.assertEqual([(like_obj, 'fake:post:target')], Fake.sent)
@ -695,6 +702,7 @@ class ProtocolReceiveTest(TestCase):
our_as1=post_as1,
deleted=True,
source_protocol=None,
feed=[self.alice.key, self.bob.key],
)
obj = self.assert_object('fake:delete',
@ -702,8 +710,8 @@ class ProtocolReceiveTest(TestCase):
our_as1=delete_as1,
delivered=['shared:target'],
type='delete',
labels=['user', 'activity'],
users=[self.user.key],
notify=[],
)
self.assertEqual([(obj, 'shared:target')], Fake.sent)
@ -722,6 +730,7 @@ class ProtocolReceiveTest(TestCase):
self.assert_object('fake:post',
deleted=True,
source_protocol=None,
feed=[],
)
self.assert_object('fake:delete',
@ -729,8 +738,8 @@ class ProtocolReceiveTest(TestCase):
our_as1=delete_as1,
delivered=[],
type='delete',
labels=['user', 'activity'],
users=[self.user.key],
notify=[],
)
self.assertEqual([], Fake.sent)
@ -802,6 +811,7 @@ class ProtocolReceiveTest(TestCase):
self.assert_object('fake:post',
our_as1=post_as1,
type='note',
feed=[self.alice.key, self.bob.key],
)
obj = self.assert_object('fake:create',
status='complete',
@ -809,8 +819,7 @@ class ProtocolReceiveTest(TestCase):
delivered=['target:2'],
failed=['target:1'],
type='post',
labels=['user', 'activity', 'feed'],
users=[g.user.key, self.alice.key, self.bob.key],
users=[g.user.key],
)
self.assertEqual(['fail', 'sent'], sent)
@ -839,6 +848,7 @@ class ProtocolReceiveTest(TestCase):
self.assert_object('fake:user',
our_as1=update_as1['object'],
type='person',
feed=[],
)
# update activity
@ -851,15 +861,14 @@ class ProtocolReceiveTest(TestCase):
delivered=['shared:target'],
type='update',
object_ids=['fake:user'],
labels=['user', 'activity'],
)
self.assertEqual([(update_obj, 'shared:target')], Fake.sent)
def test_mention_object(self, *mocks):
self.alice.obj.our_as1 = {'foo': 'bar'}
self.alice.obj.our_as1 = {'id': 'fake:alice', 'objectType': 'person'}
self.alice.obj.put()
self.bob.obj.our_as1 = {'foo': 'baz'}
self.bob.obj.our_as1 = {'id': 'fake:bob', 'objectType': 'person'}
self.bob.obj.put()
mention_as1 = {
@ -894,8 +903,8 @@ class ProtocolReceiveTest(TestCase):
},
delivered=['fake:alice:target', 'fake:bob:target'],
type='post',
labels=['user', 'activity', 'feed'],
users=[g.user.key],
notify=[self.alice.key, self.bob.key],
)
self.assertEqual([(obj, 'fake:alice:target'), (obj, 'fake:bob:target')],
@ -935,8 +944,9 @@ class ProtocolReceiveTest(TestCase):
follow_obj = self.assert_object('fake:follow',
our_as1=follow_as1,
status='complete',
users=[self.alice.key, user.key],
labels=['activity', 'user', 'notification'],
users=[self.alice.key],
notify=[user.key],
feed=[],
delivered=['fake:user:target'],
)
@ -951,10 +961,11 @@ class ProtocolReceiveTest(TestCase):
accept_obj = self.assert_object(accept_id,
our_as1=accept_as1,
type='accept',
labels=['activity'],
status='complete',
delivered=['fake:alice:target'],
users=[],
notify=[],
feed=[],
source_protocol=None,
)
@ -980,6 +991,7 @@ class ProtocolReceiveTest(TestCase):
})
self.assertEqual([], Follower.query().fetch())
self.assertEqual([], Fake.sent)
def test_follow_no_object(self):
with self.assertRaises(BadRequest):
@ -991,47 +1003,78 @@ class ProtocolReceiveTest(TestCase):
})
self.assertEqual([], Follower.query().fetch())
self.assertEqual([], Fake.sent)
def test_undo_follow(self):
def test_stop_following(self):
follower = Follower.get_or_create(to=g.user, from_=self.alice)
Fake.fetchable['fake:alice'] = {}
self.assertEqual('OK', Fake.receive({
'id': 'fake:undo-follow',
g.user.obj.our_as1 = {'id': 'fake:user'}
g.user.obj.put()
stop_as1 = {
'id': 'fake:stop-following',
'objectType': 'activity',
'verb': 'stop-following',
'actor': 'fake:alice',
'object': 'fake:user',
}))
}
self.assertEqual('OK', Fake.receive(stop_as1))
stop_obj = self.assert_object('fake:stop-following',
our_as1=stop_as1,
type='stop-following',
status='complete',
delivered=['fake:user:target'],
users=[self.alice.key],
notify=[],
feed=[],
)
self.assertEqual('inactive', follower.key.get().status)
self.assertEqual([(stop_obj, 'fake:user:target')], Fake.sent)
def test_stop_following_doesnt_exist(self):
g.user.obj.our_as1 = {'id': 'fake:user'}
g.user.obj.put()
def test_undo_follow_doesnt_exist(self):
self.assertEqual('OK', Fake.receive({
'id': 'fake:undo-follow',
'id': 'fake:stop-following',
'objectType': 'activity',
'verb': 'stop-following',
'actor': 'fake:alice',
'object': 'fake:user',
}))
# it's a noop
self.assertEqual(0, Follower.query().count())
def test_undo_follow_inactive(self):
self.assertEqual(1, len(Fake.sent))
obj, target = Fake.sent[0]
self.assertEqual('fake:stop-following', obj.key.id())
self.assertEqual('fake:user:target', target)
def test_stop_following_inactive(self):
follower = Follower.get_or_create(to=g.user, from_=self.alice,
status='inactive')
Fake.fetchable['fake:alice'] = {}
g.user.obj.our_as1 = {'id': 'fake:user'}
g.user.obj.put()
self.assertEqual('OK', Fake.receive({
'id': 'fake:undo-follow',
'id': 'fake:stop-following',
'objectType': 'activity',
'verb': 'stop-following',
'actor': 'fake:alice',
'object': 'fake:user',
}))
self.assertEqual('inactive', follower.key.get().status)
def test_receive_from_bridgy_fed_fails(self):
self.assertEqual(1, len(Fake.sent))
obj, target = Fake.sent[0]
self.assertEqual('fake:stop-following', obj.key.id())
self.assertEqual('fake:user:target', target)
def test_receive_from_bridgy_fed_domain_fails(self):
with self.assertRaises(BadRequest):
Fake.receive({
'id': 'https://fed.brid.gy/r/foo',
@ -1103,9 +1146,8 @@ class ProtocolReceiveTest(TestCase):
self.assert_object('http://x.com/follow',
our_as1=follow_as1,
status='ignored',
labels=['activity', 'user', 'notification'],
users=[ndb.Key(Fake, 'http://x.com/alice'),
ndb.Key(Fake, 'http://x.com/bob'),
ndb.Key(Fake, 'http://x.com/eve')],
users=[ndb.Key(Fake, 'http://x.com/alice')],
notify=[ndb.Key(Fake, 'http://x.com/bob'),
ndb.Key(Fake, 'http://x.com/eve')],
)
self.assertEqual(2, Follower.query().count())

Wyświetl plik

@ -230,23 +230,31 @@ class TestCase(unittest.TestCase, testutil.Asserts):
return user
def add_objects(self):
user = ndb.Key(Web, 'user.com')
# post
self.store_object(id='a', domains=['user.com'],
labels=['feed', 'notification'],
self.store_object(id='a',
users=[user],
notify=[user],
feed=[user],
as2=as2.from_as1(NOTE))
# different domain
self.store_object(id='b', domains=['nope.org'],
labels=['feed', 'notification'],
nope = ndb.Key(Web, 'nope.org')
self.store_object(id='b',
notify=[nope],
feed=[nope],
as2=as2.from_as1(MENTION))
# reply
self.store_object(id='d', domains=['user.com'],
labels=['feed', 'notification'],
self.store_object(id='d',
notify=[user],
feed=[user],
as2=as2.from_as1(COMMENT))
# not feed/notif
self.store_object(id='e', domains=['user.com'], as2=as2.from_as1(NOTE))
self.store_object(id='e', users=[user], as2=as2.from_as1(NOTE))
# deleted
self.store_object(id='f', domains=['user.com'],
labels=['feed', 'notification', 'user'],
self.store_object(id='f',
notify=[user],
feed=[user],
as2=as2.from_as1(NOTE), deleted=True)
@staticmethod
@ -316,7 +324,6 @@ class TestCase(unittest.TestCase, testutil.Asserts):
got = Object.get_by_id(id)
assert got, id
# right now we only do ActivityPub
for field in 'delivered', 'undelivered', 'failed':
props[field] = [Target(uri=uri, protocol=delivered_protocol)
for uri in props.get(field, [])]
@ -349,7 +356,7 @@ class TestCase(unittest.TestCase, testutil.Asserts):
del target.key
self.assert_entities_equal(Object(id=id, **props), got,
ignore=['as1', 'created', 'expire',
ignore=['as1', 'created', 'expire', 'labels',
'object_ids', 'type', 'updated'
] + ignore)
return got