Activity => Object: add activity label, unify Objects for AP => wm

#286
activity-redesign
Ryan Barrett 2023-02-01 12:22:04 -08:00
rodzic 4d2fcdd76f
commit f9891b6ef7
Nie znaleziono w bazie danych klucza dla tego podpisu
ID klucza GPG: 6BE31FDF4776E9D4
8 zmienionych plików z 101 dodań i 88 usunięć

Wyświetl plik

@ -136,7 +136,7 @@ def inbox(domain=None):
logging.info('Dropping non-public activity')
return ''
# normal post, deliver to BF followers
# normal post, deliver to BF followers locally
source = util.get_first(activity, 'url') or activity.get('id')
domains = []
if actor:
@ -150,7 +150,7 @@ def inbox(domain=None):
key = Object(id=source, source_protocol='activitypub', domains=domains,
status='complete', as2=activity_as2_str, as1=activity_as1_str,
type=as1_type, object_ids=as1.get_ids(activity_as1, 'object'),
labels=['feed']).put()
labels=['feed', 'activity']).put()
logging.info(f'Wrote Object {key} with {len(domains)} follower domains')
return ''

Wyświetl plik

@ -294,16 +294,20 @@ def send_webmentions(activity_wrapped, proxy=None, **object_props):
# send webmentions and store Objects
errors = [] # stores (code, body) tuples
domains = []
obj = Object(id=source, labels=['notification'], undelivered=targets, **object_props)
if activity.get('objectType') == 'activity':
obj.labels.append('activity')
obj.put()
for target in targets:
domain = util.domain_from_link(target, minimize=False)
if (domain == util.domain_from_link(source, minimize=False)):
logger.info(f'Skipping same-domain webmention from {source} to {target}')
continue
# TODO: unify across targets
obj = Object(id=source, domains=[domain], labels=['notification'],
**object_props)
obj.put()
domains.append(domain)
wm_source = (obj.proxy_url()
if verb in ('follow', 'like', 'share') or proxy
else source)
@ -313,15 +317,24 @@ def send_webmentions(activity_wrapped, proxy=None, **object_props):
endpoint = webmention.discover(target).endpoint
if endpoint:
webmention.send(endpoint, wm_source, target)
obj.status = 'complete'
logger.info('Success!')
obj.delivered.append(target)
else:
obj.status = 'ignored'
logger.info('Ignoring.')
logger.info('No webmention endpoint')
except BaseException as e:
errors.append(util.interpret_http_exception(e))
code, body = util.interpret_http_exception(e)
if not code and not body:
raise
errors.append((code, body))
obj.failed.append(target)
obj.undelivered.remove(target)
obj.put()
obj.status = 'complete' if obj.delivered else 'failed' if obj.failed else 'ignored'
obj.domains = domains
obj.put()
if errors:
msg = 'Errors: ' + ', '.join(f'{code} {body}' for code, body in errors)
error(msg, status=int(errors[0][0] or 502))

Wyświetl plik

@ -167,7 +167,7 @@ class FollowCallback(indieauth.Callback):
follow_json = json_dumps(follow_as2, sort_keys=True)
Follower.get_or_create(dest=id, src=domain, status='active',
last_follow=follow_json)
Object(id=follow_id, domains=[domain], labels=['user'],
Object(id=follow_id, domains=[domain], labels=['user', 'activity'],
source_protocol='ui', status='complete', as2=follow_json,
as1=json_dumps(as2.to_as1(follow_as2), sort_keys=True),
).put()
@ -224,7 +224,7 @@ class UnfollowCallback(indieauth.Callback):
follower.status = 'inactive'
follower.put()
Object(id=unfollow_id, domains=[domain], labels=['user'],
Object(id=unfollow_id, domains=[domain], labels=['user', 'activity'],
source_protocol='ui', status='complete',
as2=json_dumps(unfollow_as2, sort_keys=True),
as1=json_dumps(as2.to_as1(unfollow_as2), sort_keys=True),

Wyświetl plik

@ -234,7 +234,7 @@ class Object(StringIdModel):
"""
STATUSES = ('new', 'in progress', 'complete', 'failed', 'ignored')
PROTOCOLS = ('activitypub', 'bluesky', 'ostatus', 'webmention', 'ui')
LABELS = ('feed', 'notification', 'user')
LABELS = ('activity', 'feed', 'notification', 'user')
# domains of the Bridgy Fed users this activity is to or from
domains = ndb.StringProperty(repeated=True)
@ -254,10 +254,10 @@ class Object(StringIdModel):
deleted = ndb.BooleanProperty()
object_ids = ndb.StringProperty(repeated=True) # id(s) of inner objects
# ActivityPub inbox delivery
ap_delivered = ndb.StringProperty(repeated=True)
ap_undelivered = ndb.StringProperty(repeated=True)
ap_failed = ndb.StringProperty(repeated=True)
# URLs we deliver(ed) this to. ActivityPub inboxes, webmention targets, etc.
delivered = ndb.StringProperty(repeated=True)
undelivered = ndb.StringProperty(repeated=True)
failed = ndb.StringProperty(repeated=True)
created = ndb.DateTimeProperty(auto_now_add=True)
updated = ndb.DateTimeProperty(auto_now=True)

Wyświetl plik

@ -291,20 +291,25 @@ class ActivityPubTest(testutil.TestCase):
def test_inbox_reply_object(self, *mocks):
self._test_inbox_reply(REPLY_OBJECT,
{'as2': REPLY_OBJECT, 'type': 'comment'},
{'as2': REPLY_OBJECT,
'type': 'comment',
'labels': ['notification']},
*mocks)
def test_inbox_reply_object_wrapped(self, *mocks):
self._test_inbox_reply(REPLY_OBJECT_WRAPPED,
{'as2': REPLY_OBJECT,
'type': 'comment'},
'type': 'comment',
'labels': ['notification']},
*mocks)
def test_inbox_reply_create_activity(self, *mocks):
self._test_inbox_reply(REPLY,
{'as2': REPLY,
'type': 'post',
'object_ids': [REPLY_OBJECT['id']]},
'object_ids': [REPLY_OBJECT['id']],
'labels': ['notification', 'activity'],
},
*mocks)
def _test_inbox_reply(self, reply, expected_props, mock_head, mock_get, mock_post):
@ -332,7 +337,7 @@ class ActivityPubTest(testutil.TestCase):
source_protocol='activitypub',
status='complete',
as1=as2.to_as1(expected_props['as2']),
labels=['notification'],
delivered=['http://or.ig/post'],
**expected_props)
def test_inbox_reply_to_self_domain(self, mock_head, mock_get, mock_post):
@ -354,7 +359,6 @@ class ActivityPubTest(testutil.TestCase):
mock_get.assert_not_called()
mock_post.assert_not_called()
self.assertEqual(0, Object.query().count())
def test_inbox_create_obj(self, mock_head, mock_get, mock_post):
Follower.get_or_create(ACTOR['id'], 'foo.com')
@ -380,7 +384,7 @@ class ActivityPubTest(testutil.TestCase):
as1=as2.to_as1(expected_as2),
domains=['foo.com', 'baz.com'],
type='post',
labels=['feed'],
labels=['feed', 'activity'],
object_ids=[NOTE_OBJECT['id']])
def test_inbox_not_public(self, mock_head, mock_get, mock_post):
@ -400,7 +404,10 @@ class ActivityPubTest(testutil.TestCase):
def test_inbox_mention_object(self, *mocks):
self._test_inbox_mention(
MENTION_OBJECT,
{'type': 'note'}, # not mention (?)
{
'type': 'note', # not mention (?)
'labels': ['notification'],
},
*mocks,
)
@ -410,6 +417,7 @@ class ActivityPubTest(testutil.TestCase):
{
'type': 'post', # not mention (?)
'object_ids': [MENTION_OBJECT['id']],
'labels': ['notification', 'activity'],
},
*mocks,
)
@ -442,7 +450,7 @@ class ActivityPubTest(testutil.TestCase):
status='complete',
as2=expected_as2,
as1=as2.to_as1(expected_as2),
labels=['notification'],
delivered=['http://tar.get/'],
**expected_props)
def test_inbox_like(self, mock_head, mock_get, mock_post):
@ -477,8 +485,9 @@ class ActivityPubTest(testutil.TestCase):
status='complete',
as2=LIKE_WITH_ACTOR,
as1=as2.to_as1(LIKE_WITH_ACTOR),
delivered=['http://or.ig/post'],
type='like',
labels=['notification'],
labels=['notification', 'activity'],
object_ids=[LIKE['object']])
def test_inbox_follow_accept_with_id(self, mock_head, mock_get, mock_post):
@ -494,8 +503,9 @@ class ActivityPubTest(testutil.TestCase):
status='complete',
as2=follow,
as1=as2.to_as1(follow),
delivered=['https://www.realize.be/'],
type='follow',
labels=['notification'],
labels=['notification', 'activity'],
object_ids=[FOLLOW['object']])
follower = Follower.query().get()
@ -537,8 +547,9 @@ class ActivityPubTest(testutil.TestCase):
status='complete',
as2=follow,
as1=as2.to_as1(follow),
delivered=['https://www.realize.be/'],
type='follow',
labels=['notification'],
labels=['notification', 'activity'],
object_ids=[FOLLOW['object']])
def _test_inbox_follow_accept(self, follow_as2, accept_as2,
@ -750,7 +761,7 @@ class ActivityPubTest(testutil.TestCase):
as2=LIKE_WITH_ACTOR,
as1=as2.to_as1(LIKE_WITH_ACTOR),
type='like',
labels=['notification'],
labels=['notification', 'activity'],
object_ids=[LIKE['object']])
def test_followers_collection_unknown_user(self, *args):

Wyświetl plik

@ -180,23 +180,18 @@ class FollowTest(testutil.TestCase):
self.assertEqual(('http://bar/inbox',), inbox_args)
self.assert_equals(expected_follow, json_loads(inbox_kwargs['data']))
follow_as2 = json_dumps(expected_follow, sort_keys=True)
follow_as1 = json_dumps(as2.to_as1(expected_follow), sort_keys=True)
followers = Follower.query().fetch()
self.assert_entities_equal(
Follower(id='https://bar/id snarfed.org', last_follow=follow_as2,
Follower(id='https://bar/id snarfed.org',
last_follow=json_dumps(expected_follow, sort_keys=True),
src='snarfed.org', dest='https://bar/id', status='active'),
followers,
ignore=['created', 'updated'])
id = f'http://localhost/user/snarfed.org/following#2022-01-02T03:04:05-{input}'
objects = Object.query().fetch()
self.assert_entities_equal(
[Object(id=id, domains=['snarfed.org'], status='complete',
labels=['user'], source_protocol='ui',
as1=follow_as1, as2=follow_as2)],
objects,
ignore=['created', 'updated'])
self.assert_object(id, domains=['snarfed.org'], status='complete',
labels=['user', 'activity'], source_protocol='ui',
as2=expected_follow, as1=as2.to_as1(expected_follow))
def test_callback_missing_user(self, mock_get, mock_post):
mock_post.return_value = requests_response('me=https://snarfed.org')
@ -284,16 +279,11 @@ class UnfollowTest(testutil.TestCase):
follower = Follower.get_by_id('https://bar/id snarfed.org')
self.assertEqual('inactive', follower.status)
objects = Object.query().fetch()
self.assert_entities_equal(
[Object(id='http://localhost/user/snarfed.org/following#undo-2022-01-02T03:04:05-https://bar/id',
domains=['snarfed.org'], status='complete',
source_protocol='ui', labels=['user'],
as2=json_dumps(UNDO_FOLLOW, sort_keys=True),
as1=json_dumps(as2.to_as1(UNDO_FOLLOW), sort_keys=True),
)],
objects,
ignore=['created', 'updated'])
self.assert_object(
'http://localhost/user/snarfed.org/following#undo-2022-01-02T03:04:05-https://bar/id',
domains=['snarfed.org'], status='complete',
source_protocol='ui', labels=['user', 'activity'],
as2=UNDO_FOLLOW, as1=as2.to_as1(UNDO_FOLLOW))
def test_callback_missing_user(self, mock_get, mock_post):
mock_post.return_value = requests_response('me=https://snarfed.org')

Wyświetl plik

@ -493,7 +493,7 @@ class WebmentionTest(testutil.TestCase):
domains=['a'],
source_protocol='webmention',
status='complete',
ap_delivered=['https://foo.com/inbox'],
delivered=['https://foo.com/inbox'],
mf2=self.reply_mf2,
as1=self.reply_as1,
type='comment',
@ -501,8 +501,6 @@ class WebmentionTest(testutil.TestCase):
)
def test_update_reply(self, mock_get, mock_post):
# different = copy.deepcopy(self.reply_as1)
# different['content'] += ' other'
Object(id='http://a/reply', status='complete', as1='{"content": "other"}').put()
mock_get.side_effect = self.activitypub_gets
@ -539,7 +537,7 @@ class WebmentionTest(testutil.TestCase):
"""https://github.com/snarfed/bridgy-fed/issues/78"""
Object(id='http://a/reply', status='complete',
as1=json_dumps(self.reply_as1),
ap_delivered=['https://foo.com/inbox']).put()
delivered=['https://foo.com/inbox']).put()
mock_get.side_effect = self.activitypub_gets
got = self.client.post('/webmention', data={
@ -621,10 +619,10 @@ class WebmentionTest(testutil.TestCase):
status='complete',
mf2=self.repost_mf2,
as1=self.repost_as1,
ap_delivered=['https://foo.com/inbox'],
delivered=['https://foo.com/inbox'],
type='share',
object_ids=['https://orig/post'],
labels=['user'],
labels=['user', 'activity'],
)
def test_link_rel_alternate_as2(self, mock_get, mock_post):
@ -774,7 +772,7 @@ class WebmentionTest(testutil.TestCase):
status='complete',
mf2=self.create_mf2,
as1=self.create_as1,
ap_delivered=inboxes,
delivered=inboxes,
type='note',
labels=['user'],
)
@ -785,9 +783,9 @@ class WebmentionTest(testutil.TestCase):
Object(id='https://orig/post', domains=['orig'], status='in progress',
as1=json_dumps(self.create_as1),
ap_delivered=['https://skipped/inbox'],
ap_undelivered=['https://shared/inbox'],
ap_failed=['https://public/inbox'],
delivered=['https://skipped/inbox'],
undelivered=['https://shared/inbox'],
failed=['https://public/inbox'],
).put()
self.make_followers()
@ -816,7 +814,7 @@ class WebmentionTest(testutil.TestCase):
status='complete',
mf2=self.create_mf2,
as1=self.create_as1,
ap_delivered=inboxes + ['https://skipped/inbox'],
delivered=inboxes + ['https://skipped/inbox'],
type='note',
labels=['user'],
)
@ -830,9 +828,9 @@ class WebmentionTest(testutil.TestCase):
Object(id='https://orig/post', domains=['orig'], status='in progress',
as1=json_dumps(different_create_as1),
ap_delivered=['https://delivered/inbox'],
ap_undelivered=['https://shared/inbox'],
ap_failed=['https://public/inbox'],
delivered=['https://delivered/inbox'],
undelivered=['https://shared/inbox'],
failed=['https://public/inbox'],
).put()
self.make_followers()
@ -858,7 +856,7 @@ class WebmentionTest(testutil.TestCase):
status='complete',
mf2=self.create_mf2,
as1=self.create_as1,
ap_delivered=inboxes,
delivered=inboxes,
type='note',
labels=['user'],
)
@ -921,10 +919,10 @@ class WebmentionTest(testutil.TestCase):
status='complete',
mf2=self.follow_mf2,
as1=self.follow_as1,
ap_delivered=['https://foo.com/inbox'],
delivered=['https://foo.com/inbox'],
type='follow',
object_ids=['http://followee'],
labels=['user'],
labels=['user', 'activity'],
)
followers = Follower.query().fetch()
@ -988,10 +986,10 @@ class WebmentionTest(testutil.TestCase):
status='complete',
mf2=self.follow_fragment_mf2,
as1=self.follow_fragment_as1,
ap_delivered=['https://foo.com/inbox'],
delivered=['https://foo.com/inbox'],
type='follow',
object_ids=['http://followee'],
labels=['user'],
labels=['user', 'activity'],
)
followers = Follower.query().fetch()
@ -1045,10 +1043,10 @@ class WebmentionTest(testutil.TestCase):
status='failed',
mf2=self.follow_mf2,
as1=self.follow_as1,
ap_failed=['https://foo.com/inbox'],
failed=['https://foo.com/inbox'],
type='follow',
object_ids=['http://followee'],
labels=['user'],
labels=['user', 'activity'],
)
def test_repost_blocklisted_error(self, mock_get, mock_post):
@ -1131,8 +1129,8 @@ class WebmentionTest(testutil.TestCase):
status='complete',
mf2=ACTOR_MF2,
as1=expected_as1,
ap_delivered=['https://inbox', 'https://shared/inbox'],
delivered=['https://inbox', 'https://shared/inbox'],
type='update',
object_ids=['https://orig'],
labels=['user'],
labels=['user', 'activity'],
)

Wyświetl plik

@ -140,23 +140,22 @@ class Webmention(View):
if obj:
logging.info(f'Resuming existing {obj}')
obj.ap_failed = []
seen = obj.ap_delivered + obj.ap_undelivered + obj.ap_failed
obj.failed = []
seen = obj.delivered + obj.undelivered + obj.failed
new_inboxes = [i for i in inboxes_to_targets.keys() if i not in seen]
if new_inboxes:
logging.info(f'Adding new inboxes: {new_inboxes}')
obj.ap_undelivered += new_inboxes
obj.undelivered += new_inboxes
if type in ('note', 'article', 'comment'):
changed = as1.activity_changed(json_loads(obj.as1), self.source_as1)
if changed:
obj.ap_undelivered += obj.ap_delivered
obj.ap_delivered = []
logger.info(f'Content has changed from last time at {obj.updated}! Redelivering to all inboxes: {obj.ap_undelivered}')
obj.undelivered += obj.delivered
obj.delivered = []
logger.info(f'Content has changed from last time at {obj.updated}! Redelivering to all inboxes: {obj.undelivered}')
else:
obj = Object(id=self.source_url,
ap_undelivered=list(inboxes_to_targets.keys()),
ap_delivered=[], ap_failed=[])
obj = Object(id=self.source_url, undelivered=list(inboxes_to_targets.keys()),
delivered=[], failed=[])
logging.info(f'Storing new {obj}')
obj.domains = [self.source_domain]
@ -166,13 +165,15 @@ class Webmention(View):
obj.type = type
obj.object_ids = as1.get_ids(self.source_as1, 'object')
obj.labels = ['user']
if self.source_as1.get('objectType') == 'activity':
obj.labels.append('activity')
obj.put()
# TODO: collect by inbox, add 'to' fields, de-dupe inboxes and recipients
#
# make copy of ap_undelivered because we modify it below
logger.info(f'Delivering to inboxes: {sorted(obj.ap_undelivered)}')
for inbox in list(obj.ap_undelivered):
# make copy of undelivered because we modify it below
logger.info(f'Delivering to inboxes: {sorted(obj.undelivered)}')
for inbox in list(obj.undelivered):
if inbox in inboxes_to_targets:
target_as2 = inboxes_to_targets[inbox]
else:
@ -207,18 +208,18 @@ class Webmention(View):
try:
last = common.signed_post(inbox, data=self.source_as2,
log_data=log_data, user=self.user)
obj.ap_delivered.append(inbox)
obj.delivered.append(inbox)
last_success = last
except BaseException as e:
obj.ap_failed.append(inbox)
obj.failed.append(inbox)
error = e
finally:
log_data = False
obj.ap_undelivered.remove(inbox)
obj.undelivered.remove(inbox)
obj.put()
obj.status = 'complete' if obj.ap_delivered else 'failed'
obj.status = 'complete' if obj.delivered else 'failed'
obj.put()
# Pass the AP response status code and body through as our response