AP users: Object schema change, domains => users

also for AP => wm, only try to send webmentions to domains that we already have an existing Web user for. the vast majority of targets are fediverse URLs, and we were trying to send them all wms, ie at least running wm discovery and finding nothing. harmless, but a waste.
pull/542/head
Ryan Barrett 2023-06-09 12:56:45 -07:00
rodzic 62d44bdc63
commit c98ab3f2d5
Nie znaleziono w bazie danych klucza dla tego podpisu
ID klucza GPG: 6BE31FDF4776E9D4
13 zmienionych plików z 122 dodań i 74 usunięć

Wyświetl plik

@ -600,8 +600,8 @@ def inbox(protocol=None, domain=None):
# load user
# TODO(#512) parameterize on protocol, move to Protocol
if protocol and domain:
g.user = PROTOCOLS[protocol].get_by_id(domain) # receiving user
if (not g.user or not g.user.direct) and actor_id:
g.user = PROTOCOLS[protocol].get_or_create(domain, direct=False) # receiving user
if not g.user.direct and actor_id:
# this is a deliberate interaction with an indirect receiving user;
# create a local AP User for the sending user
actor_obj = ActivityPub.load(actor_id)

Wyświetl plik

@ -95,7 +95,6 @@ class FollowCallback(indieauth.Callback):
g.user = Web.get_by_id(domain)
if not g.user:
error(f'No web user for domain {domain}')
domain = g.user.key.id()
addr = state
if not state:
@ -135,11 +134,12 @@ class FollowCallback(indieauth.Callback):
'actor': g.user.ap_actor(),
'to': [as2.PUBLIC_AUDIENCE],
}
follow_obj = Object(id=follow_id, domains=[domain], labels=['user'],
source_protocol='ui', status='complete', as2=follow_as2)
followee_user = ActivityPub.get_or_create(followee_id, actor_as2=followee.as2)
follow_obj = Object(id=follow_id, users=[g.user.key, followee_user.key],
labels=['user'], source_protocol='ui', status='complete',
as2=follow_as2)
ActivityPub.send(follow_obj, inbox)
followee_user = ActivityPub.get_or_create(followee_id, actor_as2=followee.as2)
Follower.get_or_create(from_=g.user, to=followee_user, status='active',
follow=follow_obj.key)
follow_obj.put()
@ -187,7 +187,6 @@ class UnfollowCallback(indieauth.Callback):
g.user = Web.get_by_id(domain)
if not g.user:
error(f'No web user for domain {domain}')
domain = g.user.key.id()
if util.is_int(state):
state = int(state)
@ -219,7 +218,10 @@ class UnfollowCallback(indieauth.Callback):
'object': follower.follow.get().as2 if follower.follow else None,
}
obj = Object(id=unfollow_id, domains=[domain], labels=['user'],
# don't include the followee User who's being unfollowed in the users
# property, since we don't want to notify or show them. (standard social
# network etiquette.)
obj = Object(id=unfollow_id, users=[g.user.key], labels=['user'],
source_protocol='ui', status='complete', as2=unfollow_as2)
ActivityPub.send(obj, inbox)

Wyświetl plik

@ -13,6 +13,13 @@ indexes:
- name: updated
direction: asc
- kind: Object
properties:
- name: users
- name: labels
- name: updated
direction: asc
- kind: Object
properties:
- name: domains
@ -20,6 +27,13 @@ indexes:
- name: updated
direction: desc
- kind: Object
properties:
- name: users
- name: labels
- name: updated
direction: desc
- kind: Object
properties:
- name: domains
@ -27,6 +41,13 @@ indexes:
- name: created
direction: desc
- kind: Object
properties:
- name: users
- name: labels
- name: created
direction: desc
- kind: Follower
properties:
- name: from

Wyświetl plik

@ -301,8 +301,11 @@ class Object(StringIdModel):
STATUSES = ('new', 'in progress', 'complete', 'failed', 'ignored')
LABELS = ('activity', 'feed', 'notification', 'user')
# domains of the Bridgy Fed users this activity is to or from
# Users this activity is to or from
users = ndb.KeyProperty(repeated=True)
# DEPRECATED
domains = ndb.StringProperty(repeated=True)
status = ndb.StringProperty(choices=STATUSES)
# choices is populated in flask_app, after all User subclasses are created,
# so that PROTOCOLS is fully populated
@ -383,7 +386,7 @@ class Object(StringIdModel):
def _post_put_hook(self, future):
"""Update :meth:`Protocol.load` cache."""
# TODO: assert that as1 id is same as key id? in pre put hook?
logger.info(f'Wrote Object {self.key.id()} {self.type} {self.status or ""} {self.labels} for {len(self.domains)} users')
logger.info(f'Wrote Object {self.key.id()} {self.type} {self.status or ""} {self.labels} for {len(self.users)} users {len(self.domains)} domains')
if '#' not in self.key.id():
import protocol # TODO: actually fix this circular import
protocol.objects_cache[self.key.id()] = self
@ -423,7 +426,7 @@ class Object(StringIdModel):
attrs = {'class': 'h-card u-author'}
if (self.source_protocol in ('web', 'webmention', 'ui') and g.user and
g.user.key.id() in self.domains):
(g.user.key in self.users or g.user.key.id() in self.domains)):
# outbound; show a nice link to the user
return g.user.user_page_link()

Wyświetl plik

@ -8,6 +8,7 @@ import urllib.parse
from flask import g, redirect, render_template, request
from google.cloud.ndb.model import get_multi
from google.cloud.ndb.query import OR
from google.cloud.ndb.stats import KindStat
from granary import as1, as2, atom, microformats2, rss
import humanize
@ -95,7 +96,8 @@ def user(protocol, id):
load_user(protocol, id)
query = Object.query(
Object.domains == id,
OR(Object.users == g.user.key,
Object.domains == id),
Object.labels.IN(('notification', 'user')),
)
objects, before, after = fetch_objects(query)
@ -144,12 +146,12 @@ def feed(protocol, id):
load_user(protocol, id)
assert not g.user.use_instead
objects, _, _ = Object.query(
Object.domains == id, Object.labels == 'feed') \
objects = Object.query(
OR(Object.users == g.user.key,
Object.domains == id),
Object.labels == 'feed') \
.order(-Object.created) \
.fetch_page(PAGE_SIZE)
.fetch(PAGE_SIZE)
activities = [obj.as1 for obj in objects if not obj.deleted]
actor = {
@ -226,12 +228,13 @@ def fetch_objects(query):
urls = as1.object_urls(inner_obj)
id = common.redirect_unwrap(inner_obj.get('id', ''))
url = urls[0] if urls else id
if (type == 'update' and obj.domains and
id.strip('/') == f'https://{obj.domains[0]}'):
if (type == 'update' and
(obj.users and id.strip('/') == obj.users[0].id()
or obj.domains and id.strip('/') == f'https://{obj.domains[0]}')):
obj.phrase = 'updated'
obj_as1.update({
'content': 'their profile',
'url': f'https://{obj.domains[0]}',
'url': id,
})
elif url:
content = common.pretty_link(url, text=content)

Wyświetl plik

@ -251,9 +251,9 @@ class Protocol:
from activitypub import ActivityPub
for f in Follower.query(Follower.to == ActivityPub(id=actor_id).key,
Follower.status == 'active'):
if f.from_.id() not in obj.domains:
obj.domains.append(f.from_.id())
if obj.domains and 'feed' not in obj.labels:
if f.from_ not in obj.users:
obj.users.append(f.from_)
if obj.users and 'feed' not in obj.labels:
obj.labels.append('feed')
obj.put()
@ -345,6 +345,7 @@ class Protocol:
# send webmentions and update Object
errors = [] # stores (code, body) tuples
targets = [Target(uri=uri, protocol='web') for uri in targets]
no_user_domains = set()
obj.populate(
undelivered=targets,
@ -362,12 +363,25 @@ class Protocol:
logger.info(f'Skipping same-domain webmention from {source} to {target.uri}')
continue
if domain not in obj.domains:
obj.domains.append(domain)
# only deliver if we have a matching User already.
# TODO: consider delivering or at least storing Users for all
# targets? need to filter out native targets in this protocol
# though, eg mastodon.social targets in AP inbox deliveries.
if domain in no_user_domains:
continue
# TODO(#512): generalize protocol
from web import Web
recip = Web(id=domain).key
if recip not in obj.users:
if not recip.get():
logger.info(f'No Web user for {domain}; skipping {target.uri}')
no_user_domains.add(domain)
continue
obj.users.append(recip)
try:
# TODO: fix
from web import Web
# TODO(#512): generalize protocol
if Web.send(obj, target.uri):
obj.delivered.append(target)
if 'notification' not in obj.labels:
@ -381,7 +395,7 @@ class Protocol:
obj.put()
obj.status = ('complete' if obj.delivered or obj.domains
obj.status = ('complete' if obj.delivered or obj.users
else 'failed' if obj.failed
else 'ignored')

Wyświetl plik

@ -429,7 +429,7 @@ class ActivityPubTest(TestCase):
+ [WEBMENTION_DISCOVERY])
mock_post.return_value = requests_response()
got = self.post('/user.com/inbox', json=reply)
got = self.post('/ap/web/user.com/inbox', json=reply)
self.assertEqual(200, got.status_code, got.get_data(as_text=True))
self.assert_req(mock_get, 'https://user.com/post')
convert_id = reply['id'].replace('://', ':/')
@ -445,7 +445,7 @@ class ActivityPubTest(TestCase):
)
self.assert_object(reply['id'],
domains=['user.com'],
users=[self.user.key],
source_protocol='activitypub',
status='complete',
delivered=['https://user.com/post'],
@ -501,7 +501,7 @@ class ActivityPubTest(TestCase):
self.assert_object('http://mas.to/note/as2',
source_protocol='activitypub',
as2=expected_as2,
domains=['user.com', 'baz.com'],
users=[self.user.key, Fake(id='baz.com').key],
type='post',
labels=['activity', 'feed'],
object_ids=[NOTE_OBJECT['id']])
@ -548,7 +548,7 @@ class ActivityPubTest(TestCase):
source_protocol='activitypub',
status='complete',
as2=repost,
domains=['user.com'],
users=[self.user.key],
delivered=['https://user.com/orig'],
type='share',
labels=['activity', 'feed', 'notification'],
@ -579,7 +579,7 @@ class ActivityPubTest(TestCase):
source_protocol='activitypub',
status='ignored',
as2=REPOST_FULL,
domains=['user.com', 'baz.com'],
users=[self.user.key, Fake(id='baz.com').key],
type='share',
labels=['activity', 'feed'],
object_ids=[REPOST['object']])
@ -596,9 +596,10 @@ class ActivityPubTest(TestCase):
self.assertEqual(200, got.status_code)
self.assert_object('http://mas.to/like#ok',
domains=['nope.com'],
# no nope.com Web user key since it didn't exist
users=[],
source_protocol='activitypub',
status='complete',
status='ignored',
as2={**LIKE_WITH_ACTOR, 'object': 'http://nope.com/post'},
type='like',
labels=['activity'],
@ -650,6 +651,8 @@ class ActivityPubTest(TestCase):
type='note')
def _test_inbox_mention(self, mention, expected_props, mock_head, mock_get, mock_post):
self.make_user('tar.get')
mock_get.side_effect = [
WEBMENTION_DISCOVERY,
HTML,
@ -673,7 +676,7 @@ class ActivityPubTest(TestCase):
expected_as2 = common.redirect_unwrap(mention)
self.assert_object(mention['id'],
domains=['tar.get', 'masto.foo'],
users=[Web(id='tar.get').key],
source_protocol='activitypub',
status='complete',
as2=expected_as2,
@ -705,7 +708,7 @@ class ActivityPubTest(TestCase):
}, kwargs['data'])
self.assert_object('http://mas.to/like#ok',
domains=['user.com'],
users=[self.user.key],
source_protocol='activitypub',
status='complete',
as2=LIKE_WITH_ACTOR,
@ -733,7 +736,7 @@ class ActivityPubTest(TestCase):
'url': 'https://mas.to/users/swentel#followed-https://user.com/',
}
self.assert_object('https://mas.to/6d1a',
domains=['user.com'],
users=[self.user.key],
source_protocol='activitypub',
status='complete',
as2=follow,
@ -765,7 +768,7 @@ class ActivityPubTest(TestCase):
'url': 'https://mas.to/users/swentel#followed-https://user.com/',
})
self.assert_object('https://mas.to/6d1a',
domains=['user.com'],
users=[self.user.key],
source_protocol='activitypub',
status='complete',
as2=follow,
@ -787,7 +790,7 @@ class ActivityPubTest(TestCase):
'url': 'https://mas.to/users/swentel#followed-https://user.com/',
}
self.assert_object('https://mas.to/6d1a',
domains=['user.com'],
users=[self.user.key],
source_protocol='activitypub',
status='complete',
as2=follow,
@ -980,6 +983,7 @@ class ActivityPubTest(TestCase):
obj = Object.get_by_id(id)
self.assertEqual(['activity'], obj.labels)
self.assertEqual([], obj.users)
self.assertEqual([], obj.domains)
self.assertIsNone(Object.get_by_id(bad_url))
@ -1159,7 +1163,7 @@ class ActivityPubTest(TestCase):
self.assertEqual(200, got.status_code)
self.assert_object('http://mas.to/like#ok',
domains=['user.com'],
users=[self.user.key],
source_protocol='activitypub',
status='complete',
as2=LIKE_WITH_ACTOR,

Wyświetl plik

@ -210,15 +210,17 @@ class FollowTest(TestCase):
follow_id = f'http://localhost/web/alice.com/following#2022-01-02T03:04:05-{input}'
followers = Follower.query().fetch()
followee = ActivityPub(id='https://bar/id').key
self.assert_entities_equal(
Follower(from_=self.user.key, to=ActivityPub(id='https://bar/id').key,
Follower(from_=self.user.key, to=followee,
follow=Object(id=follow_id).key, status='active'),
followers,
ignore=['created', 'updated'])
self.assert_object(follow_id, domains=['alice.com'], status='complete',
labels=['user', 'activity'], source_protocol='ui',
as2=expected_follow, as1=as2.to_as1(expected_follow))
self.assert_object(follow_id, users=[self.user.key, followee],
status='complete', labels=['user', 'activity'],
source_protocol='ui', as2=expected_follow,
as1=as2.to_as1(expected_follow))
self.assertEqual('https://alice.com', session['indieauthed-me'])
@ -258,15 +260,15 @@ class FollowTest(TestCase):
'object': FOLLOWEE,
'to': [as2.PUBLIC_AUDIENCE],
}
followee = ActivityPub(id='https://bar/id').key
follow_obj = self.assert_object(
id, domains=['www.alice.com'], status='complete',
id, users=[user.key, followee], status='complete',
labels=['user', 'activity'], source_protocol='ui', as2=expected_follow,
as1=as2.to_as1(expected_follow))
followers = Follower.query().fetch()
self.assert_entities_equal(
Follower(from_=user.key, to=ActivityPub(id='https://bar/id').key,
follow=follow_obj.key, status='active'),
Follower(from_=user.key, to=followee, follow=follow_obj.key, status='active'),
followers,
ignore=['created', 'updated'])
@ -395,10 +397,8 @@ class UnfollowTest(TestCase):
self.assert_object(
'http://localhost/web/alice.com/following#undo-2022-01-02T03:04:05-https://bar/id',
domains=['alice.com'], status='complete',
source_protocol='ui', labels=['user', 'activity'],
as2=expected_undo,
as1=as2.to_as1(expected_undo))
users=[self.user.key], status='complete', source_protocol='ui',
labels=['user', 'activity'], as2=expected_undo, as1=as2.to_as1(expected_undo))
self.assertEqual('https://alice.com', session['indieauthed-me'])
@ -448,7 +448,7 @@ class UnfollowTest(TestCase):
follower = Follower.query().get()
self.assertEqual('inactive', follower.status)
self.assert_object(id, domains=['www.alice.com'], status='complete',
self.assert_object(id, users=[user.key], status='complete',
source_protocol='ui', labels=['user', 'activity'],
as2=expected_undo, as1=as2.to_as1(expected_undo))

Wyświetl plik

@ -164,7 +164,7 @@ class ObjectTest(TestCase):
def test_actor_link_user(self):
g.user = Fake(id='user.com', actor_as2={"name": "Alice"})
obj = Object(id='x', source_protocol='ui', domains=['user.com'])
obj = Object(id='x', source_protocol='ui', users=[g.user.key])
self.assertIn(
'href="/fake/user.com"><img src="" class="profile"> Alice</a>',
obj.actor_link())

Wyświetl plik

@ -110,7 +110,7 @@ class PagesTest(TestCase):
def test_user_object_bare_string_id(self):
with self.request_context:
Object(id='a', domains=['user.com'], labels=['notification'],
Object(id='a', users=[self.user.key], labels=['notification'],
as2=REPOST_AS2).put()
got = self.client.get('/web/user.com')
@ -118,7 +118,7 @@ class PagesTest(TestCase):
def test_user_object_url_object(self):
with self.request_context:
Object(id='a', domains=['user.com'], labels=['notification'], our_as1={
Object(id='a', users=[self.user.key], labels=['notification'], our_as1={
**REPOST_AS2,
'object': {
'id': 'https://mas.to/toot/id',

Wyświetl plik

@ -45,6 +45,7 @@ class ProtocolTest(TestCase):
def test_receive_reply_not_feed_not_notification(self, mock_get):
Follower.get_or_create(to=Fake.get_or_create(id=ACTOR['id']),
from_=Fake.get_or_create(id='foo.com'))
other_user = self.make_user('user.com', cls=Web)
# user.com webmention discovery
mock_get.return_value = requests_response('<html></html>')
@ -54,7 +55,7 @@ class ProtocolTest(TestCase):
self.assert_object(REPLY['id'],
as2=REPLY,
type='post',
domains=['user.com'],
users=[other_user.key],
# not feed since it's a reply
# not notification since it doesn't involve the user
labels=['activity'],

Wyświetl plik

@ -606,14 +606,14 @@ class WebTest(TestCase):
self.assert_deliveries(mock_post, ['https://mas.to/inbox'], AS2_CREATE)
self.assert_object('https://user.com/reply',
domains=['user.com'],
users=[g.user.key],
source_protocol='web',
mf2=REPLY_MF2,
as1=REPLY_AS1,
type='comment',
)
self.assert_object('https://user.com/reply#bridgy-fed-create',
domains=['user.com'],
users=[g.user.key],
source_protocol='web',
status='complete',
mf2=REPLY_MF2,
@ -767,7 +767,7 @@ class WebTest(TestCase):
mf2 = util.parse_mf2(html)['items'][0]
self.assert_object('https://user.com/repost',
domains=['user.com'],
users=[g.user.key],
source_protocol='web',
status='complete',
mf2=mf2,
@ -825,7 +825,7 @@ class WebTest(TestCase):
mock_post.assert_not_called()
self.assert_object('https://user.com/like',
domains=['user.com'],
users=[g.user.key],
source_protocol='web',
mf2=LIKE_MF2,
as1=microformats2.json_to_object(LIKE_MF2),
@ -930,13 +930,13 @@ class WebTest(TestCase):
self.assert_deliveries(mock_post, inboxes, CREATE_AS2)
self.assert_object('https://user.com/post',
domains=['user.com'],
users=[g.user.key],
mf2=NOTE_MF2,
type='note',
source_protocol='web',
)
self.assert_object('https://user.com/post#bridgy-fed-create',
domains=['user.com'],
users=[g.user.key],
source_protocol='web',
status='complete',
mf2=NOTE_MF2,
@ -953,7 +953,7 @@ class WebTest(TestCase):
with self.request_context:
mf2 = copy.deepcopy(NOTE_MF2)
mf2['properties']['content'] = 'different'
Object(id='https://user.com/post', domains=['user.com'], mf2=mf2).put()
Object(id='https://user.com/post', users=[g.user.key], mf2=mf2).put()
self.make_followers()
@ -981,7 +981,7 @@ class WebTest(TestCase):
}
self.assert_object(
f'https://user.com/post#bridgy-fed-update-2022-01-02T03:04:05+00:00',
domains=['user.com'],
users=[g.user.key],
source_protocol='web',
status='complete',
mf2=NOTE_MF2,
@ -1036,7 +1036,7 @@ class WebTest(TestCase):
self.assert_deliveries(mock_post, ['https://mas.to/inbox'], FOLLOW_AS2)
obj = self.assert_object('https://user.com/follow',
domains=['user.com'],
users=[g.user.key],
source_protocol='web',
status='complete',
mf2=FOLLOW_MF2,
@ -1119,7 +1119,7 @@ class WebTest(TestCase):
FOLLOW_FRAGMENT_AS2)
obj = self.assert_object('https://user.com/follow#2',
domains=['user.com'],
users=[g.user.key],
source_protocol='web',
status='complete',
mf2=FOLLOW_FRAGMENT_MF2,
@ -1180,7 +1180,7 @@ class WebTest(TestCase):
mf2 = util.parse_mf2(html)['items'][0]
as1 = microformats2.json_to_object(mf2)
obj = self.assert_object('https://user.com/follow',
domains=['user.com'],
users=[g.user.key],
source_protocol='web',
status='complete',
mf2=mf2,
@ -1239,7 +1239,7 @@ class WebTest(TestCase):
self.assert_deliveries(mock_post, inboxes, DELETE_AS2)
self.assert_object('https://user.com/post#bridgy-fed-delete',
domains=['user.com'],
users=[g.user.key],
source_protocol='web',
status='complete',
our_as1=DELETE_AS1,
@ -1298,7 +1298,7 @@ class WebTest(TestCase):
self.assert_deliveries(mock_post, ['https://mas.to/inbox'], FOLLOW_AS2)
self.assert_object('https://user.com/follow',
domains=['user.com'],
users=[g.user.key],
source_protocol='web',
status='failed',
mf2=FOLLOW_MF2,
@ -1386,7 +1386,7 @@ class WebTest(TestCase):
},
}
self.assert_object(id,
domains=['user.com'],
users=[g.user.key],
source_protocol='web',
status='complete',
our_as1=expected_as1,

6
web.py
Wyświetl plik

@ -482,7 +482,7 @@ def webmention_task():
inboxes_to_targets = _activitypub_targets(obj)
obj.populate(
domains=[g.user.key.id()],
users=[g.user.key],
source_protocol='web',
)
if not inboxes_to_targets:
@ -519,7 +519,7 @@ def webmention_task():
},
}
obj = Object(id=id, mf2=obj.mf2, our_as1=update_as1, labels=['user'],
domains=[g.user.key.id()], source_protocol='web')
users=[g.user.key], source_protocol='web')
elif obj.new or 'force' in request.form:
logger.info(f'New Object {obj.key.id()}')
@ -534,7 +534,7 @@ def webmention_task():
'object': obj.as1,
}
obj = Object(id=id, mf2=obj.mf2, our_as1=create_as1,
domains=[g.user.key.id()], labels=['user'],
users=[g.user.key], labels=['user'],
source_protocol='web')
else: