kopia lustrzana https://github.com/snarfed/bridgy-fed
Ryan Barrett
rodzic
0d24fafefc
commit
b4841f3e60
|
|
@ -1,5 +1,6 @@
|
|||
"""Bridgy Fed firehose client. Enqueues receive tasks for events for bridged users.
|
||||
"""
|
||||
from collections import namedtuple
|
||||
import itertools
|
||||
import logging
|
||||
import os
|
||||
|
|
@ -18,8 +19,12 @@ from models import Object
|
|||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# contains (str action, dict record or str path) tuples. second element is a
|
||||
# dict record for creates and updates, str path for deletes.
|
||||
# a commit operation. similar to arroba.repo.Write. record is None for deletes.
|
||||
Op = namedtuple('Op', ['action', 'repo', 'path', 'record'],
|
||||
# record is optional
|
||||
defaults=[None])
|
||||
|
||||
# contains Ops
|
||||
new_commits = SimpleQueue()
|
||||
|
||||
|
||||
|
|
@ -52,6 +57,7 @@ def subscribe():
|
|||
continue
|
||||
|
||||
repo = payload.get('repo')
|
||||
assert repo
|
||||
if repo in our_bridged_dids: # from a Bridgy Fed non-Bluesky user; ignore
|
||||
# logger.info(f'Ignoring record from our non-ATProto bridged user {repo}')
|
||||
continue
|
||||
|
|
@ -63,37 +69,36 @@ def subscribe():
|
|||
|
||||
# detect records that reference an ATProto user, eg replies, likes,
|
||||
# reposts, mentions
|
||||
for op in payload['ops']:
|
||||
action = op['action']
|
||||
path = op['path']
|
||||
cid = op['cid']
|
||||
assert action, path
|
||||
for p_op in payload['ops']:
|
||||
op = Op(repo=repo, action=p_op['action'], path=p_op['path'])
|
||||
assert op.action and op.path, (op.action, op.path)
|
||||
cid = p_op['cid']
|
||||
|
||||
is_ours = repo in our_atproto_dids
|
||||
if is_ours and action == 'delete':
|
||||
logger.info(f'Got delete from our ATProto user: {repo} {path}')
|
||||
if is_ours and op.action == 'delete':
|
||||
logger.info(f'Got delete from our ATProto user: {op}')
|
||||
# TODO: also detect deletes of records that *reference* our bridged
|
||||
# users, eg a delete of a follow or like or repost of them.
|
||||
# not easy because we need to getRecord the record to check
|
||||
new_commits.put(('delete', path))
|
||||
new_commits.put(op)
|
||||
continue
|
||||
|
||||
block = blocks.get(op['cid'])
|
||||
block = blocks.get(cid)
|
||||
# our own commits are sometimes missing the record
|
||||
# https://github.com/snarfed/bridgy-fed/issues/1016
|
||||
if not block:
|
||||
continue
|
||||
|
||||
record = block.decoded
|
||||
type = record.get('$type')
|
||||
op = Op(*op[:-1], record=block.decoded)
|
||||
type = op.record.get('$type')
|
||||
if not type:
|
||||
logger.warning('commit record missing $type! {action} {cid}')
|
||||
logger.warning(dag_json.encode(record).decode())
|
||||
logger.warning('commit record missing $type! {op.action} {op.repo} {op.path} {cid}')
|
||||
logger.warning(dag_json.encode(op.record).decode())
|
||||
continue
|
||||
|
||||
if repo in our_atproto_dids:
|
||||
logger.info(f'Got one from our ATProto user: {action} {repo} {path}')
|
||||
new_commits.put((action, record))
|
||||
if is_ours:
|
||||
logger.info(f'Got one from our ATProto user: {op.action} {op.repo} {op.path}')
|
||||
new_commits.put(op)
|
||||
continue
|
||||
|
||||
subjects = []
|
||||
|
|
@ -112,33 +117,33 @@ def subscribe():
|
|||
add(subjects, did)
|
||||
|
||||
if type in ('app.bsky.feed.like', 'app.bsky.feed.repost'):
|
||||
maybe_add(record['subject'])
|
||||
maybe_add(op.record['subject'])
|
||||
|
||||
elif type in ('app.bsky.graph.block', 'app.bsky.graph.follow'):
|
||||
maybe_add(record['subject'])
|
||||
maybe_add(op.record['subject'])
|
||||
|
||||
elif type == 'app.bsky.feed.post':
|
||||
# replies
|
||||
if reply := record.get('reply'):
|
||||
if reply := op.record.get('reply'):
|
||||
for ref in 'parent', 'root':
|
||||
maybe_add(reply[ref])
|
||||
|
||||
# mentions
|
||||
for facet in record.get('facets', []):
|
||||
for facet in op.record.get('facets', []):
|
||||
for feature in facet['features']:
|
||||
if feature['$type'] == 'app.bsky.richtext.facet#mention':
|
||||
maybe_add(feature['did'])
|
||||
|
||||
# quote posts
|
||||
if embed := record.get('embed'):
|
||||
if embed := op.record.get('embed'):
|
||||
if embed['$type'] == 'app.bsky.embed.record':
|
||||
maybe_add(embed['record'])
|
||||
elif embed['$type'] == 'app.bsky.embed.recordWithMedia':
|
||||
maybe_add(embed['record']['record'])
|
||||
|
||||
if subjects:
|
||||
logger.info(f'Got one re our ATProto users {subjects}: {action} {repo} {path}')
|
||||
new_commits.put((action, record))
|
||||
logger.info(f'Got one re our ATProto users {subjects}: {op.action} {op.repo} {op.path}')
|
||||
new_commits.put(op)
|
||||
|
||||
logger.info('Ran out of events! Relay closed connection?')
|
||||
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ from google.cloud.tasks_v2.types import Task
|
|||
from granary.tests.test_bluesky import (
|
||||
ACTOR_PROFILE_BSKY,
|
||||
LIKE_BSKY,
|
||||
POST_AS,
|
||||
POST_BSKY,
|
||||
REPLY_BSKY,
|
||||
REPOST_BSKY,
|
||||
|
|
@ -20,7 +21,7 @@ from oauth_dropins.webutil import util
|
|||
import simple_websocket
|
||||
|
||||
from atproto import ATProto
|
||||
from atproto_firehose import new_commits, subscribe
|
||||
from atproto_firehose import new_commits, Op, subscribe
|
||||
import common
|
||||
from models import Object, PROTOCOLS, Target
|
||||
import protocol
|
||||
|
|
@ -45,13 +46,12 @@ class FakeWebsocketClient:
|
|||
return dag_cbor.encode(header) + dag_cbor.encode(payload)
|
||||
|
||||
@classmethod
|
||||
def setup_receive(cls, record, path='app.bsky.feed.post/abc123',
|
||||
action='create', repo='did:plc:user'):
|
||||
def setup_receive(cls, op):
|
||||
cid = CID.decode('bafkreicqpqncshdd27sgztqgzocd3zhhqnnsv6slvzhs5uz6f57cq6lmtq')
|
||||
if action == 'delete':
|
||||
if op.action == 'delete':
|
||||
block_bytes = b''
|
||||
else:
|
||||
block = Block(decoded=record)
|
||||
block = Block(decoded=op.record)
|
||||
block_bytes = write_car([cid], [block])
|
||||
|
||||
cls.to_receive = [({
|
||||
|
|
@ -61,13 +61,13 @@ class FakeWebsocketClient:
|
|||
'blocks': block_bytes,
|
||||
'commit': cid,
|
||||
'ops': [{
|
||||
'action': action,
|
||||
'cid': None if action == 'delete' else block.cid,
|
||||
'path': path,
|
||||
'action': op.action,
|
||||
'cid': None if op.action == 'delete' else block.cid,
|
||||
'path': op.path,
|
||||
}],
|
||||
'prev': None,
|
||||
'rebase': False,
|
||||
'repo': repo,
|
||||
'repo': op.repo,
|
||||
'rev': 'abc',
|
||||
'seq': 123,
|
||||
'since': 'def',
|
||||
|
|
@ -84,18 +84,29 @@ class ATProtoFirehoseSubscribeTest(TestCase):
|
|||
FakeWebsocketClient.sent = []
|
||||
FakeWebsocketClient.to_receive = []
|
||||
|
||||
assert new_commits.empty()
|
||||
|
||||
self.alice = self.make_user(
|
||||
'eefake:alice', cls=ExplicitEnableFake,
|
||||
copies=[Target(protocol='atproto', uri='did:alice')])
|
||||
|
||||
def assert_enqueues(self, record=None, expected=None, action='create', **kwargs):
|
||||
FakeWebsocketClient.setup_receive(record, action=action, **kwargs)
|
||||
def assert_enqueues(self, record=None, repo='did:plc:user', action='create',
|
||||
path='app.bsky.feed.post/abc123'):
|
||||
FakeWebsocketClient.setup_receive(
|
||||
Op(repo=repo, action=action, path=path, record=record))
|
||||
subscribe()
|
||||
self.assertEqual((action, expected or record), new_commits.get())
|
||||
|
||||
op = new_commits.get()
|
||||
self.assertEqual(repo, op.repo)
|
||||
self.assertEqual(action, op.action)
|
||||
self.assertEqual(path, op.path)
|
||||
self.assertEqual(record, op.record)
|
||||
self.assertTrue(new_commits.empty())
|
||||
|
||||
def assert_doesnt_enqueue(self, record=None, action='create', **kwargs):
|
||||
FakeWebsocketClient.setup_receive(record, action=action, **kwargs)
|
||||
def assert_doesnt_enqueue(self, record=None, repo='did:plc:user', action='create',
|
||||
path='app.bsky.feed.post/abc123'):
|
||||
FakeWebsocketClient.setup_receive(
|
||||
Op(repo=repo, action=action, path=path, record=record))
|
||||
subscribe()
|
||||
self.assertTrue(new_commits.empty())
|
||||
|
||||
|
|
@ -289,7 +300,7 @@ class ATProtoFirehoseSubscribeTest(TestCase):
|
|||
obj_bsky=ACTOR_PROFILE_BSKY)
|
||||
|
||||
path = 'app.bsky.feed.post/abc123'
|
||||
self.assert_enqueues(expected=path, path=path, action='delete')
|
||||
self.assert_enqueues(path=path, action='delete')
|
||||
|
||||
def test_delete_by_other(self):
|
||||
self.assert_doesnt_enqueue(action='delete')
|
||||
|
|
@ -300,11 +311,10 @@ class ATProtoFirehoseSubscribeTest(TestCase):
|
|||
enabled_protocols=['eefake'],
|
||||
obj_bsky=ACTOR_PROFILE_BSKY)
|
||||
|
||||
path = 'app.bsky.feed.post/abc123'
|
||||
self.assert_enqueues(expected=path, path=path, action='delete')
|
||||
self.assert_enqueues(action='delete')
|
||||
|
||||
def test_update_by_other(self):
|
||||
self.assert_doesnt_enqueue(action='delete', path='app.bsky.feed.post/abc123')
|
||||
self.assert_doesnt_enqueue(action='delete')
|
||||
|
||||
def test_update_like_of_our_user(self):
|
||||
self.assert_enqueues(action='update', record={
|
||||
|
|
@ -315,8 +325,8 @@ class ATProtoFirehoseSubscribeTest(TestCase):
|
|||
|
||||
@skip
|
||||
class ATProtoFirehoseHandleTest(TestCase):
|
||||
def test_handle(self):
|
||||
new_commits.put('create')
|
||||
def test_handle_create(self):
|
||||
new_commits.put(('create', POST_BSKY))
|
||||
at_uri = 'at://did:plc:user/app.bsky.feed.like/abc123'
|
||||
user_key = ATProto(id='did:plc:user').key
|
||||
obj = self.assert_object(at_uri, bsky=LIKE_BSKY, status='new',
|
||||
|
|
|
|||
Ładowanie…
Reference in New Issue