From eaa4e5333a5421d15bac06da854036af1114adb8 Mon Sep 17 00:00:00 2001 From: Ryan Barrett Date: Fri, 26 May 2023 16:07:36 -0700 Subject: [PATCH] start to make User subclasses for each protocol #512 --- activitypub.py | 25 ++--- app.py | 5 +- convert.py | 9 +- follow.py | 16 +-- models.py | 162 ++++++++++++------------------ pages.py | 11 ++- protocol.py | 16 +-- redirect.py | 10 +- tests/test_activitypub.py | 21 ++-- tests/test_common.py | 6 +- tests/test_convert.py | 1 + tests/test_follow.py | 2 + tests/test_models.py | 201 +++----------------------------------- tests/test_pages.py | 10 +- tests/test_protocol.py | 37 ++++--- tests/test_webmention.py | 177 ++++++++++++++++++++++++++++++++- tests/testutil.py | 55 ++++++----- ui.py | 3 +- webfinger.py | 2 +- webmention.py | 109 +++++++++++++++++---- xrpc_actor.py | 3 +- xrpc_feed.py | 3 +- xrpc_graph.py | 3 +- 23 files changed, 464 insertions(+), 423 deletions(-) diff --git a/activitypub.py b/activitypub.py index 5796b26..fcb04f3 100644 --- a/activitypub.py +++ b/activitypub.py @@ -1,5 +1,4 @@ -"""Handles requests for ActivityPub endpoints: actors, inbox, etc. -""" +"""ActivityPub protocol implementation.""" from base64 import b64encode from hashlib import sha256 import itertools @@ -29,6 +28,7 @@ from common import ( ) from models import Follower, Object, Target, User from protocol import Protocol +import webmention logger = logging.getLogger(__name__) @@ -43,11 +43,11 @@ _DEFAULT_SIGNATURE_USER = None def default_signature_user(): global _DEFAULT_SIGNATURE_USER if _DEFAULT_SIGNATURE_USER is None: - _DEFAULT_SIGNATURE_USER = User.get_or_create('snarfed.org') + _DEFAULT_SIGNATURE_USER = webmention.Webmention.get_or_create('snarfed.org') return _DEFAULT_SIGNATURE_USER -class ActivityPub(Protocol): +class ActivityPub(User, Protocol): """ActivityPub protocol class.""" LABEL = 'activitypub' @@ -513,11 +513,12 @@ def actor(domain): if tld in TLD_BLOCKLIST: error('', status=404) - g.user = User.get_by_id(domain) + # TODO(#512): parameterize by protocol + g.user = webmention.Webmention.get_by_id(domain) if not g.user: - return f'User {domain} not found', 404 + return f'Web user {domain} not found', 404 elif not g.user.actor_as2: - return f'User {domain} not fully set up', 404 + return f'Web user {domain} not fully set up', 404 # TODO: unify with common.actor() actor = postprocess_as2(g.user.actor_as2) @@ -565,9 +566,10 @@ def inbox(domain=None): # load user if domain: - g.user = User.get_by_id(domain) + # TODO(#512): parameterize by protocol + g.user = webmention.Webmention.get_by_id(domain) if not g.user: - error(f'User {domain} not found', status=404) + error(f'Web user {domain} not found', status=404) ActivityPub.verify_signature(activity) @@ -603,8 +605,9 @@ def follower_collection(domain, collection): https://www.w3.org/TR/activitypub/#collections https://www.w3.org/TR/activitystreams-core/#paging """ - if not User.get_by_id(domain): - return f'User {domain} not found', 404 + # TODO(#512): parameterize by protocol + if not webmention.Webmention.get_by_id(domain): + return f'Web user {domain} not found', 404 # page followers, new_before, new_after = Follower.fetch_page(domain, collection) diff --git a/app.py b/app.py index e09d964..45da507 100644 --- a/app.py +++ b/app.py @@ -6,4 +6,7 @@ registered. from flask_app import app # import all modules to register their Flask handlers -import activitypub, convert, follow, pages, redirect, superfeedr, webfinger, webmention, xrpc_actor, xrpc_feed, xrpc_graph +import activitypub, convert, follow, pages, redirect, superfeedr, ui, webfinger, webmention, xrpc_actor, xrpc_feed, xrpc_graph + +import models +models.reset_protocol_properties() diff --git a/convert.py b/convert.py index 7c7b8d3..8402d99 100644 --- a/convert.py +++ b/convert.py @@ -17,8 +17,7 @@ from oauth_dropins.webutil.flask_util import error from activitypub import ActivityPub from common import CACHE_TIME from flask_app import app, cache -from models import Object -from protocol import protocols +from models import Object, PROTOCOLS from webmention import Webmention logger = logging.getLogger(__name__) @@ -52,7 +51,7 @@ def convert(src, dest, _): error(f'Expected fully qualified URL; got {url}') # load, and maybe fetch. if it's a post/update, redirect to inner object. - obj = protocols[src].load(url) + obj = PROTOCOLS[src].load(url) if not obj.as1: error(f'Stored object for {id} has no data', status=404) @@ -60,7 +59,7 @@ def convert(src, dest, _): if type in ('post', 'update', 'delete'): obj_id = as1.get_object(obj.as1).get('id') if obj_id: - # TODO: protocols[src].load() this instead? + # TODO: PROTOCOLS[src].load() this instead? obj_obj = Object.get_by_id(obj_id) if (obj_obj and obj_obj.as1 and not obj_obj.as1.keys() <= set(['id', 'url', 'objectType'])): @@ -72,7 +71,7 @@ def convert(src, dest, _): return '', 410 # convert and serve - return protocols[dest].serve(obj) + return PROTOCOLS[dest].serve(obj) @app.get('/render') diff --git a/follow.py b/follow.py index 4d35234..badeefa 100644 --- a/follow.py +++ b/follow.py @@ -20,6 +20,7 @@ from activitypub import ActivityPub from flask_app import app import common import models +from webmention import Webmention logger = logging.getLogger(__name__) @@ -80,9 +81,10 @@ def remote_follow(): logger.info(f'Got: {request.values}') domain = request.values['domain'] - g.user = models.User.get_by_id(domain) + # TODO(#512): parameterize by protocol + g.user = Webmention.get_by_id(domain) if not g.user: - error(f'No Bridgy Fed user found for domain {domain}') + error(f'No web user found for domain {domain}') addr = request.values['address'] webfinger = fetch_webfinger(addr) @@ -133,9 +135,10 @@ class FollowCallback(indieauth.Callback): session['indieauthed-me'] = me domain = util.domain_from_link(me) - g.user = models.User.get_by_id(domain) + # TODO(#512): parameterize by protocol + g.user = Webmention.get_by_id(domain) if not g.user: - error(f'No user for domain {domain}') + error(f'No web user for domain {domain}') domain = g.user.key.id() addr = state @@ -220,9 +223,10 @@ class UnfollowCallback(indieauth.Callback): session['indieauthed-me'] = me domain = util.domain_from_link(me) - g.user = models.User.get_by_id(domain) + # TODO(#512): parameterize by protocol + g.user = Webmention.get_by_id(domain) if not g.user: - error(f'No user for domain {domain}') + error(f'No web user for domain {domain}') domain = g.user.key.id() follower = models.Follower.get_by_id(state) diff --git a/models.py b/models.py index 13288e9..24cf21b 100644 --- a/models.py +++ b/models.py @@ -1,7 +1,6 @@ """Datastore model classes.""" import base64 from datetime import timedelta, timezone -import difflib import itertools import json import logging @@ -22,17 +21,13 @@ from oauth_dropins.webutil.flask_util import error from oauth_dropins.webutil.models import ComputedJsonProperty, JsonProperty, StringIdModel from oauth_dropins.webutil.util import json_dumps, json_loads import requests -from werkzeug.exceptions import BadRequest, NotFound import common -# https://github.com/snarfed/bridgy-fed/issues/314 -WWW_DOMAINS = frozenset(( - 'www.jvt.me', -)) -# TODO: eventually load from protocol.protocols instead, if/when we can get -# around the circular import -PROTOCOLS = ('activitypub', 'bluesky', 'ostatus', 'webmention', 'ui') +# maps string label to Protocol subclass. populated by ProtocolUserMeta. +# seed with old and upcoming protocols that don't have their own classes (yet). +PROTOCOLS = {'bluesky': None, 'ostatus': None} + # 2048 bits makes tests slow, so use 1024 for them KEY_BITS = 1024 if DEBUG else 2048 PAGE_SIZE = 20 @@ -53,6 +48,21 @@ OBJECT_EXPIRE_AGE = timedelta(days=90) logger = logging.getLogger(__name__) +class ProtocolUserMeta(type(ndb.Model)): + """:class:`User` metaclass. Registers all subclasses in the PROTOCOLS global.""" + def __new__(meta, name, bases, class_dict): + cls = super().__new__(meta, name, bases, class_dict) + if hasattr(cls, 'LABEL'): + PROTOCOLS[cls.LABEL] = cls + return cls + + +def reset_protocol_properties(): + """Recreates various protocol properties to include choices PROTOCOLS.""" + Target.protocol = ndb.StringProperty(choices=list(PROTOCOLS.keys()), required=True) + Object.source_protocol = ndb.StringProperty(choices=list(PROTOCOLS.keys())) + + def base64_to_long(x): """Converts x from URL safe base64 encoding to a long integer. @@ -69,24 +79,20 @@ def long_to_base64(x): return base64.urlsafe_b64encode(number.long_to_bytes(x)) -class User(StringIdModel): - """Stores a Bridgy Fed user. - - The key name is the domain. +class User(StringIdModel, metaclass=ProtocolUserMeta): + """Abstract base class for a Bridgy Fed user. Stores multiple keypairs needed for the supported protocols. Currently: * RSA keypair for ActivityPub HTTP Signatures - properties: mod, public_exponent, private_exponent + properties: mod, public_exponent, private_exponent, all encoded as + base64url (ie URL-safe base64) strings as described in RFC 4648 and + section 5.1 of the Magic Signatures spec https://tools.ietf.org/html/draft-cavage-http-signatures-12 * P-256 keypair for AT Protocol's signing key property: p256_key, PEM encoded https://atproto.com/guides/overview#account-portability - - The key pair's modulus and exponent properties are all encoded as base64url - (ie URL-safe base64) strings as described in RFC 4648 and section 5.1 of the - Magic Signatures spec. """ mod = ndb.StringProperty(required=True) public_exponent = ndb.StringProperty(required=True) @@ -101,16 +107,18 @@ class User(StringIdModel): created = ndb.DateTimeProperty(auto_now_add=True) updated = ndb.DateTimeProperty(auto_now=True) + @classmethod + def new(cls, **kwargs): + """Try to prevent instantiation. Use subclasses instead.""" + raise NotImplementedError() + + # TODO(#512): move this and is_homepage to webmention.py? @property def homepage(self): return f'https://{self.key.id()}/' - @classmethod - def _get_kind(cls): - return 'MagicKey' - def _post_put_hook(self, future): - logger.info(f'Wrote User {self.key.id()}') + logger.info(f'Wrote {self.key}') @classmethod def get_by_id(cls, id): @@ -121,27 +129,34 @@ class User(StringIdModel): return user - @staticmethod + @classmethod @ndb.transactional() - def get_or_create(domain, **kwargs): + def get_or_create(cls, domain, **kwargs): """Loads and returns a User. Creates it if necessary.""" - user = User.get_by_id(domain) + assert cls != User + user = cls.get_by_id(domain) if user: return user - # originally from django_salmon.magicsigs - # this uses urandom(), and does nontrivial math, so it can take a - # while depending on the amount of randomness available. - rng = Random.new().read - rsa_key = RSA.generate(KEY_BITS, rng) - p256_key = ECC.generate(curve='P-256', - randfunc=random.randbytes if DEBUG else None) - user = User(id=domain, - mod=long_to_base64(rsa_key.n), - public_exponent=long_to_base64(rsa_key.e), - private_exponent=long_to_base64(rsa_key.d), - p256_key=p256_key.export_key(format='PEM'), - **kwargs) + # generate keys for all protocols _except_ our own + # + # these can use urandom() and do nontrivial math, so they can take time + # depending on the amount of randomness available and compute needed. + if cls.LABEL != 'activitypub': + # originally from django_salmon.magicsigs + key = RSA.generate(KEY_BITS, randfunc=random.randbytes if DEBUG else None) + kwargs.update({ + 'mod': long_to_base64(key.n), + 'public_exponent': long_to_base64(key.e), + 'private_exponent': long_to_base64(key.d), + }) + + if cls.LABEL != 'atprotocol': + key = ECC.generate( + curve='P-256', randfunc=random.randbytes if DEBUG else None) + kwargs['p256_key'] = key.export_key(format='PEM') + + user = cls(id=domain, **kwargs) user.put() return user @@ -222,67 +237,6 @@ class User(StringIdModel): return f' {name}' - def verify(self): - """Fetches site a couple ways to check for redirects and h-card. - - Returns: User that was verified. May be different than self! eg if self's - domain started with www and we switch to the root domain. - """ - domain = self.key.id() - logger.info(f'Verifying {domain}') - - if domain.startswith('www.') and domain not in WWW_DOMAINS: - # if root domain redirects to www, use root domain instead - # https://github.com/snarfed/bridgy-fed/issues/314 - root = domain.removeprefix("www.") - root_site = f'https://{root}/' - try: - resp = util.requests_get(root_site, gateway=False) - if resp.ok and self.is_homepage(resp.url): - logger.info(f'{root_site} redirects to {resp.url} ; using {root} instead') - root_user = User.get_or_create(root) - self.use_instead = root_user.key - self.put() - return root_user.verify() - except requests.RequestException: - pass - - # check webfinger redirect - path = f'/.well-known/webfinger?resource=acct:{domain}@{domain}' - self.has_redirects = False - self.redirects_error = None - try: - url = urllib.parse.urljoin(self.homepage, path) - resp = util.requests_get(url, gateway=False) - domain_urls = ([f'https://{domain}/' for domain in common.DOMAINS] + - [common.host_url()]) - expected = [urllib.parse.urljoin(url, path) for url in domain_urls] - if resp.ok: - if resp.url in expected: - self.has_redirects = True - elif resp.url: - diff = '\n'.join(difflib.Differ().compare([resp.url], [expected[0]])) - self.redirects_error = f'Current vs expected:
{diff}
' - else: - lines = [url, f' returned HTTP {resp.status_code}'] - if resp.url != url: - lines[1:1] = [' redirected to:', resp.url] - self.redirects_error = '
' + '\n'.join(lines) + '
' - except requests.RequestException: - pass - - # check home page - try: - import activitypub, webmention # TODO: actually fix these circular imports - obj = webmention.Webmention.load(self.homepage, gateway=True) - self.actor_as2 = activitypub.postprocess_as2(as2.from_as1(obj.as1)) - self.has_hcard = True - except (BadRequest, NotFound): - self.actor_as2 = None - self.has_hcard = False - - return self - class Target(ndb.Model): """Delivery destinations. ActivityPub inboxes, webmention targets, etc. @@ -301,7 +255,9 @@ class Target(ndb.Model): https://googleapis.dev/python/python-ndb/latest/model.html#google.cloud.ndb.model.StructuredProperty """ uri = ndb.StringProperty(required=True) - protocol = ndb.StringProperty(choices=PROTOCOLS, required=True) + # choices is populated in flask_app, after all User subclasses are created, + # so that PROTOCOLS is fully populated + protocol = ndb.StringProperty(choices=[], required=True) class Object(StringIdModel): @@ -315,8 +271,10 @@ class Object(StringIdModel): # domains of the Bridgy Fed users this activity is to or from domains = ndb.StringProperty(repeated=True) status = ndb.StringProperty(choices=STATUSES) + # choices is populated in flask_app, after all User subclasses are created, + # so that PROTOCOLS is fully populated # TODO: remove? is this redundant with the protocol-specific data fields below? - source_protocol = ndb.StringProperty(choices=PROTOCOLS) + source_protocol = ndb.StringProperty(choices=[]) labels = ndb.StringProperty(repeated=True, choices=LABELS) # TODO: switch back to ndb.JsonProperty if/when they fix it for the web console diff --git a/pages.py b/pages.py index 81e7fe5..b122128 100644 --- a/pages.py +++ b/pages.py @@ -18,6 +18,7 @@ from flask_app import app, cache import common from common import DOMAIN_RE from models import fetch_page, Follower, Object, PAGE_SIZE, User +from webmention import Webmention FOLLOWERS_UI_LIMIT = 999 @@ -49,7 +50,7 @@ def docs(): def enter_web_site(): return render_template('enter_web_site.html') - +# TODO(#512): move to webmention.py? @app.post('/web-site') def check_web_site(): url = request.values['url'] @@ -58,7 +59,7 @@ def check_web_site(): flash(f'No domain found in {url}') return render_template('enter_web_site.html') - g.user = User.get_or_create(domain) + g.user = Webmention.get_or_create(domain) try: g.user = g.user.verify() except BaseException as e: @@ -74,7 +75,7 @@ def check_web_site(): @app.get(f'/user/') def user(domain): - g.user = User.get_by_id(domain) + g.user = Webmention.get_by_id(domain) if not g.user: return USER_NOT_FOUND_HTML, 404 elif g.user.key.id() != domain: @@ -109,7 +110,7 @@ def user(domain): @app.get(f'/user//') def followers_or_following(domain, collection): - g.user = User.get_by_id(domain) # g.user is used in template + g.user = Webmention.get_by_id(domain) # g.user is used in template if not g.user: return USER_NOT_FOUND_HTML, 404 @@ -138,7 +139,7 @@ def feed(domain): if format not in ('html', 'atom', 'rss'): error(f'format {format} not supported; expected html, atom, or rss') - g.user = User.get_by_id(domain) + g.user = Webmention.get_by_id(domain) if not g.user: return render_template('user_not_found.html', domain=domain), 404 diff --git a/protocol.py b/protocol.py index 6e5f846..da7e71d 100644 --- a/protocol.py +++ b/protocol.py @@ -10,7 +10,7 @@ from granary import as1, as2 import common from common import error -from models import Follower, Object, Target, User +from models import Follower, Object, Target from oauth_dropins.webutil import util, webmention from oauth_dropins.webutil.util import json_dumps, json_loads @@ -45,19 +45,7 @@ objects_cache_lock = threading.Lock() logger = logging.getLogger(__name__) -# maps string label to Protocol subclass. populated by ProtocolMeta. -protocols = {} - -class ProtocolMeta(type): - """:class:`Protocol` metaclass. Registers all subclasses in the protocols global.""" - def __new__(meta, name, bases, class_dict): - cls = super().__new__(meta, name, bases, class_dict) - if cls.LABEL: - protocols[cls.LABEL] = cls - return cls - - -class Protocol(metaclass=ProtocolMeta): +class Protocol: """Base protocol class. Not to be instantiated; classmethods only. Attributes: diff --git a/redirect.py b/redirect.py index ef510f2..4736028 100644 --- a/redirect.py +++ b/redirect.py @@ -76,16 +76,18 @@ def redir(to): to_domain)) for domain in domains: if domain: - g.user = User.get_by_id(domain) + # TODO(#512): do we need to parameterize this by protocol? or is it + # only for web? + g.user = Webmention.get_by_id(domain) if g.user: - logger.info(f'Found User for domain {domain}') + logger.info(f'Found web user for domain {domain}') break else: if accept_as2: g.external_user = urllib.parse.urljoin(to, '/') - logging.info(f'No User for {g.external_user}') + logging.info(f'No web user for {g.external_user}') else: - return f'No user found for any of {domains}', 404 + return f'No web user found for any of {domains}', 404 if accept_as2: # AS2 requested, fetch and convert and serve diff --git a/tests/test_activitypub.py b/tests/test_activitypub.py index 6ac792b..2c00497 100644 --- a/tests/test_activitypub.py +++ b/tests/test_activitypub.py @@ -25,10 +25,12 @@ from activitypub import ActivityPub from flask_app import app import common import models -from models import Follower, Object, User +from models import Follower, Object import protocol from protocol import Protocol -from . import testutil +from webmention import Webmention + +from .testutil import Fake, TestCase ACTOR = { '@context': 'https://www.w3.org/ns/activitystreams', @@ -222,11 +224,12 @@ NOT_ACCEPTABLE = requests_response(status=406) @patch('requests.post') @patch('requests.get') @patch('requests.head') -class ActivityPubTest(testutil.TestCase): +class ActivityPubTest(TestCase): def setUp(self): super().setUp() - self.user = self.make_user('user.com', has_hcard=True, actor_as2=ACTOR) + self.user = self.make_user('user.com', + has_hcard=True, actor_as2=ACTOR) with self.request_context: self.key_id_obj = Object(id='http://my/key/id', as2={ **ACTOR, @@ -1163,12 +1166,12 @@ class ActivityPubTest(testutil.TestCase): }, resp.json) -class ActivityPubUtilsTest(testutil.TestCase): +class ActivityPubUtilsTest(TestCase): def setUp(self): super().setUp() self.request_context.push() - g.user = self.make_user('user.com', has_hcard=True, actor_as2=ACTOR) - + g.user = self.make_user('user.com', has_hcard=True, + actor_as2=ACTOR) def tearDown(self): self.request_context.pop() super().tearDown() @@ -1205,7 +1208,7 @@ class ActivityPubUtilsTest(testutil.TestCase): })) def test_postprocess_as2_actor_attributedTo(self): - g.user = User(id='site') + g.user = Fake(id='site') self.assert_equals({ 'actor': { 'id': 'baj', @@ -1255,7 +1258,7 @@ class ActivityPubUtilsTest(testutil.TestCase): ], })) - # TODO: make these generic and use FakeProtocol + # TODO: make these generic and use Fake @patch('requests.get') def test_load_http(self, mock_get): mock_get.return_value = AS2 diff --git a/tests/test_common.py b/tests/test_common.py index e5993d1..8b6954e 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -11,10 +11,10 @@ from flask_app import app import common from models import Object, User import protocol -from . import testutil +from .testutil import Fake, TestCase -class CommonTest(testutil.TestCase): +class CommonTest(TestCase): @classmethod def setUpClass(cls): with appengine_config.ndb_client.context(): @@ -24,7 +24,7 @@ class CommonTest(testutil.TestCase): def setUp(self): super().setUp() self.request_context.push() - g.user = User(id='site') + g.user = Fake(id='site') def tearDown(self): self.request_context.pop() diff --git a/tests/test_convert.py b/tests/test_convert.py index 2ec64e1..86983d6 100644 --- a/tests/test_convert.py +++ b/tests/test_convert.py @@ -9,6 +9,7 @@ from models import Object from oauth_dropins.webutil.testutil import requests_response import requests +import app from common import CONTENT_TYPE_HTML from .test_redirect import ( diff --git a/tests/test_follow.py b/tests/test_follow.py index 2a85419..3442167 100644 --- a/tests/test_follow.py +++ b/tests/test_follow.py @@ -14,6 +14,8 @@ from flask_app import app import common from common import redirect_unwrap from models import Follower, Object, User +from webmention import Webmention + from . import testutil WEBFINGER = requests_response({ diff --git a/tests/test_models.py b/tests/test_models.py index d84b7c0..95ad80b 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -14,27 +14,24 @@ from flask_app import app import common from models import AtpNode, Follower, Object, OBJECT_EXPIRE_AGE, User import protocol -from . import testutil from .test_activitypub import ACTOR +from .testutil import Fake, TestCase -class UserTest(testutil.TestCase): + +class UserTest(TestCase): def setUp(self): super().setUp() self.request_context.push() g.user = self.make_user('y.z') - self.full_redir = requests_response( - status=302, - redirected_url='http://localhost/.well-known/webfinger?resource=acct:y.z@y.z') - def tearDown(self): self.request_context.pop() super().tearDown() def test_get_or_create(self): - user = User.get_or_create('a.b') + user = Fake.get_or_create('a.b') assert user.mod assert user.public_exponent @@ -49,15 +46,15 @@ class UserTest(testutil.TestCase): assert isinstance(p256_key, ECC.EccKey) self.assertEqual('NIST P-256', p256_key.curve) - same = User.get_or_create('a.b') + same = Fake.get_or_create('a.b') self.assertEqual(same, user) def test_get_or_create_use_instead(self): - user = User.get_or_create('a.b') + user = Fake.get_or_create('a.b') user.use_instead = g.user.key user.put() - self.assertEqual('y.z', User.get_or_create('a.b').key.id()) + self.assertEqual('y.z', Fake.get_or_create('a.b').key.id()) def test_href(self): href = g.user.href() @@ -90,184 +87,8 @@ class UserTest(testutil.TestCase): def test_actor_id(self): self.assertEqual('http://localhost/y.z', g.user.actor_id()) - def _test_verify(self, redirects, hcard, actor, redirects_error=None): - got = g.user.verify() - self.assertEqual(g.user.key, got.key) - with self.subTest(redirects=redirects, hcard=hcard, actor=actor, - redirects_error=redirects_error): - self.assert_equals(redirects, bool(g.user.has_redirects)) - self.assert_equals(hcard, bool(g.user.has_hcard)) - if actor is None: - self.assertIsNone(g.user.actor_as2) - else: - got = {k: v for k, v in g.user.actor_as2.items() - if k in actor} - self.assert_equals(actor, got) - self.assert_equals(redirects_error, g.user.redirects_error) - - @mock.patch('requests.get') - def test_verify_neither(self, mock_get): - empty = requests_response('') - mock_get.side_effect = [empty, empty] - self._test_verify(False, False, None) - - @mock.patch('requests.get') - def test_verify_redirect_strips_query_params(self, mock_get): - half_redir = requests_response( - status=302, redirected_url='http://localhost/.well-known/webfinger') - no_hcard = requests_response('') - mock_get.side_effect = [half_redir, no_hcard] - self._test_verify(False, False, None, """\ -Current vs expected:
- http://localhost/.well-known/webfinger
-+ https://fed.brid.gy/.well-known/webfinger?resource=acct:y.z@y.z
""") - - @mock.patch('requests.get') - def test_verify_multiple_redirects(self, mock_get): - two_redirs = requests_response( - status=302, redirected_url=[ - 'https://www.y.z/.well-known/webfinger?resource=acct:y.z@y.z', - 'http://localhost/.well-known/webfinger?resource=acct:y.z@y.z', - ]) - no_hcard = requests_response('') - mock_get.side_effect = [two_redirs, no_hcard] - self._test_verify(True, False, None) - - @mock.patch('requests.get') - def test_verify_redirect_404(self, mock_get): - redir_404 = requests_response(status=404, redirected_url='http://this/404s') - no_hcard = requests_response('') - mock_get.side_effect = [redir_404, no_hcard] - self._test_verify(False, False, None, """\ -
https://y.z/.well-known/webfinger?resource=acct:y.z@y.z
-  redirected to:
-http://this/404s
-  returned HTTP 404
""") - - @mock.patch('requests.get') - def test_verify_no_hcard(self, mock_get): - mock_get.side_effect = [ - self.full_redir, - requests_response(""" - -
-

foo bar

-
- -"""), - ] - self._test_verify(True, False, None) - - @mock.patch('requests.get') - def test_verify_non_representative_hcard(self, mock_get): - bad_hcard = requests_response( - 'acct:me@y.z', - url='https://y.z/', - ) - mock_get.side_effect = [self.full_redir, bad_hcard] - self._test_verify(True, False, None) - - @mock.patch('requests.get') - def test_verify_both_work(self, mock_get): - hcard = requests_response(""" - - me - Masto -""", - url='https://y.z/', - ) - mock_get.side_effect = [self.full_redir, hcard] - self._test_verify(True, True, { - 'type': 'Person', - 'name': 'me', - 'url': ['http://localhost/r/https://y.z/', 'acct:myself@y.z'], - 'preferredUsername': 'y.z', - }) - - @mock.patch('requests.get') - def test_verify_www_redirect(self, mock_get): - www_user = self.make_user('www.y.z') - - empty = requests_response('') - mock_get.side_effect = [ - requests_response(status=302, redirected_url='https://www.y.z/'), - empty, empty, - ] - - got = www_user.verify() - self.assertEqual('y.z', got.key.id()) - - root_user = User.get_by_id('y.z') - self.assertEqual(root_user.key, www_user.key.get().use_instead) - self.assertEqual(root_user.key, User.get_or_create('www.y.z').key) - - @mock.patch('requests.get') - def test_verify_actor_rel_me_links(self, mock_get): - mock_get.side_effect = [ - self.full_redir, - requests_response(""" - -
-Mrs. ☕ Foo -should be ignored - - one text - - -
- -""", url='https://y.z/'), - ] - self._test_verify(True, True, { - 'attachment': [{ - 'type': 'PropertyValue', - 'name': 'Mrs. ☕ Foo', - 'value': 'y.z/about-me', - }, { - 'type': 'PropertyValue', - 'name': 'Web site', - 'value': 'y.z', - }, { - 'type': 'PropertyValue', - 'name': 'one text', - 'value': 'one', - }, { - 'type': 'PropertyValue', - 'name': 'two title', - 'value': 'two', - }]}) - - @mock.patch('requests.get') - def test_verify_override_preferredUsername(self, mock_get): - mock_get.side_effect = [ - self.full_redir, - requests_response(""" - - - Nick - - -""", url='https://y.z/'), - ] - self._test_verify(True, True, { - # stays y.z despite user's username. since Mastodon queries Webfinger - # for preferredUsername@fed.brid.gy - # https://github.com/snarfed/bridgy-fed/issues/77#issuecomment-949955109 - 'preferredUsername': 'y.z', - }) - - def test_homepage(self): - self.assertEqual('https://y.z/', g.user.homepage) - - def test_is_homepage(self): - for url in 'y.z', '//y.z', 'http://y.z', 'https://y.z': - self.assertTrue(g.user.is_homepage(url), url) - - for url in None, '', 'y', 'z', 'z.z', 'ftp://y.z', 'http://y', '://y.z': - self.assertFalse(g.user.is_homepage(url), url) - - -class ObjectTest(testutil.TestCase): +class ObjectTest(TestCase): def setUp(self): super().setUp() self.request_context.push() @@ -323,7 +144,7 @@ class ObjectTest(testutil.TestCase): self.assert_multiline_in(expected, obj.actor_link()) def test_actor_link_user(self): - g.user = User(id='user.com', actor_as2={"name": "Alice"}) + g.user = Fake(id='user.com', actor_as2={"name": "Alice"}) obj = Object(id='x', source_protocol='ui', domains=['user.com']) self.assertIn( 'href="/user/user.com"> Alice', @@ -369,7 +190,7 @@ class ObjectTest(testutil.TestCase): self.assertEqual(['user'], obj.labels) -class FollowerTest(testutil.TestCase): +class FollowerTest(TestCase): def setUp(self): super().setUp() @@ -391,7 +212,7 @@ class FollowerTest(testutil.TestCase): self.assertEqual(ACTOR, self.outbound.to_as2()) -class AtpNodeTest(testutil.TestCase): +class AtpNodeTest(TestCase): def test_create(self): AtpNode.create(ACTOR_PROFILE_BSKY) diff --git a/tests/test_pages.py b/tests/test_pages.py index 2fc7a4b..f7cbb22 100644 --- a/tests/test_pages.py +++ b/tests/test_pages.py @@ -18,15 +18,17 @@ from oauth_dropins.webutil.testutil import requests_response from flask_app import app import common from models import Object, Follower, User -from . import testutil +from webmention import Webmention + from .test_webmention import ACTOR_AS2, ACTOR_HTML, ACTOR_MF2, REPOST_AS2 +from .testutil import Fake, TestCase def contents(activities): return [(a.get('object') or a)['content'] for a in activities] -class PagesTest(testutil.TestCase): +class PagesTest(TestCase): EXPECTED = contents([COMMENT, NOTE]) def setUp(self): @@ -109,7 +111,7 @@ class PagesTest(testutil.TestCase): self.assert_equals(302, got.status_code) self.assert_equals('/user/user.com', got.headers['Location']) - user = User.get_by_id('user.com') + user = Webmention.get_by_id('user.com') self.assertTrue(user.has_hcard) self.assertEqual('Person', user.actor_as2['type']) self.assertEqual('http://localhost/user.com', user.actor_as2['id']) @@ -118,7 +120,7 @@ class PagesTest(testutil.TestCase): got = self.client.post('/web-site', data={'url': '!!!'}) self.assert_equals(200, got.status_code) self.assertEqual(['No domain found in !!!'], get_flashed_messages()) - self.assertEqual(1, User.query().count()) + self.assertEqual(1, Webmention.query().count()) @patch('requests.get') def test_check_web_site_fetch_fails(self, mock_get): diff --git a/tests/test_protocol.py b/tests/test_protocol.py index 312bcc0..cb80443 100644 --- a/tests/test_protocol.py +++ b/tests/test_protocol.py @@ -8,12 +8,11 @@ import requests import protocol from protocol import Protocol from flask_app import app -from models import Follower, Object, User +from models import Follower, Object, PROTOCOLS, User from webmention import Webmention from .test_activitypub import ACTOR, REPLY -from . import testutil -from .testutil import FakeProtocol +from .testutil import Fake, TestCase REPLY = { **REPLY, @@ -25,7 +24,7 @@ REPLY = { } -class ProtocolTest(testutil.TestCase): +class ProtocolTest(TestCase): def setUp(self): super().setUp() @@ -38,8 +37,8 @@ class ProtocolTest(testutil.TestCase): super().tearDown() def test_protocols_global(self): - self.assertEqual(FakeProtocol, protocol.protocols['fake']) - self.assertEqual(Webmention, protocol.protocols['webmention']) + self.assertEqual(Fake, PROTOCOLS['fake']) + self.assertEqual(Webmention, PROTOCOLS['webmention']) @patch('requests.get') def test_receive_reply_not_feed_not_notification(self, mock_get): @@ -65,58 +64,58 @@ class ProtocolTest(testutil.TestCase): ) def test_load(self): - FakeProtocol.objects['foo'] = {'x': 'y'} + Fake.objects['foo'] = {'x': 'y'} - loaded = FakeProtocol.load('foo') + loaded = Fake.load('foo') self.assert_equals({'x': 'y'}, loaded.our_as1) self.assertFalse(loaded.changed) self.assertTrue(loaded.new) self.assertIsNotNone(Object.get_by_id('foo')) - self.assertEqual(['foo'], FakeProtocol.fetched) + self.assertEqual(['foo'], Fake.fetched) def test_load_already_stored(self): stored = Object(id='foo', our_as1={'x': 'y'}) stored.put() - loaded = FakeProtocol.load('foo') + loaded = Fake.load('foo') self.assert_equals({'x': 'y'}, loaded.our_as1) self.assertFalse(loaded.changed) self.assertFalse(loaded.new) - self.assertEqual([], FakeProtocol.fetched) + self.assertEqual([], Fake.fetched) @patch('requests.get') def test_load_empty_deleted(self, mock_get): stored = Object(id='foo', deleted=True) stored.put() - loaded = FakeProtocol.load('foo') + loaded = Fake.load('foo') self.assert_entities_equal(stored, loaded) self.assertFalse(loaded.changed) self.assertFalse(loaded.new) - self.assertEqual([], FakeProtocol.fetched) + self.assertEqual([], Fake.fetched) @patch('requests.get') def test_load_refresh_unchanged(self, mock_get): obj = Object(id='foo', our_as1={'x': 'stored'}) obj.put() - FakeProtocol.objects['foo'] = {'x': 'stored'} + Fake.objects['foo'] = {'x': 'stored'} - loaded = FakeProtocol.load('foo', refresh=True) + loaded = Fake.load('foo', refresh=True) self.assert_entities_equal(obj, loaded) self.assertFalse(obj.changed) self.assertFalse(obj.new) - self.assertEqual(['foo'], FakeProtocol.fetched) + self.assertEqual(['foo'], Fake.fetched) @patch('requests.get') def test_load_refresh_changed(self, mock_get): Object(id='foo', our_as1={'content': 'stored'}).put() - FakeProtocol.objects['foo'] = {'content': 'new'} + Fake.objects['foo'] = {'content': 'new'} - loaded = FakeProtocol.load('foo', refresh=True) + loaded = Fake.load('foo', refresh=True) self.assert_equals({'content': 'new'}, loaded.our_as1) self.assertTrue(loaded.changed) self.assertFalse(loaded.new) - self.assertEqual(['foo'], FakeProtocol.fetched) + self.assertEqual(['foo'], Fake.fetched) diff --git a/tests/test_webmention.py b/tests/test_webmention.py index 8747ec3..0b75a21 100644 --- a/tests/test_webmention.py +++ b/tests/test_webmention.py @@ -162,7 +162,13 @@ DELETE_AS2 = { class WebmentionTest(testutil.TestCase): def setUp(self): super().setUp() - self.user = self.make_user('user.com') + g.user = self.user = self.make_user('user.com') + + self.request_context.push() + self.full_redir = requests_response( + status=302, + redirected_url='http://localhost/.well-known/webfinger?resource=acct:user.com@user.com') + self.toot_html = requests_response("""\ @@ -1337,10 +1343,177 @@ class WebmentionTest(testutil.TestCase): labels=['user', 'activity'], ) + def _test_verify(self, redirects, hcard, actor, redirects_error=None): + got = self.user.verify() + self.assertEqual(self.user.key, got.key) + + with self.subTest(redirects=redirects, hcard=hcard, actor=actor, + redirects_error=redirects_error): + self.assert_equals(redirects, bool(self.user.has_redirects)) + self.assert_equals(hcard, bool(self.user.has_hcard)) + if actor is None: + self.assertIsNone(self.user.actor_as2) + else: + got = {k: v for k, v in self.user.actor_as2.items() + if k in actor} + self.assert_equals(actor, got) + self.assert_equals(redirects_error, self.user.redirects_error) + + def test_verify_neither(self, mock_get, _): + empty = requests_response('') + mock_get.side_effect = [empty, empty] + self._test_verify(False, False, None) + + def test_verify_redirect_strips_query_params(self, mock_get, _): + half_redir = requests_response( + status=302, redirected_url='http://localhost/.well-known/webfinger') + no_hcard = requests_response('') + mock_get.side_effect = [half_redir, no_hcard] + self._test_verify(False, False, None, """\ +Current vs expected:
- http://localhost/.well-known/webfinger
++ https://fed.brid.gy/.well-known/webfinger?resource=acct:user.com@user.com
""") + + def test_verify_multiple_redirects(self, mock_get, _): + two_redirs = requests_response( + status=302, redirected_url=[ + 'https://www.user.com/.well-known/webfinger?resource=acct:user.com@user.com', + 'http://localhost/.well-known/webfinger?resource=acct:user.com@user.com', + ]) + no_hcard = requests_response('') + mock_get.side_effect = [two_redirs, no_hcard] + self._test_verify(True, False, None) + + def test_verify_redirect_404(self, mock_get, _): + redir_404 = requests_response(status=404, redirected_url='http://this/404s') + no_hcard = requests_response('') + mock_get.side_effect = [redir_404, no_hcard] + self._test_verify(False, False, None, """\ +
https://user.com/.well-known/webfinger?resource=acct:user.com@user.com
+  redirected to:
+http://this/404s
+  returned HTTP 404
""") + + def test_verify_no_hcard(self, mock_get, _): + mock_get.side_effect = [ + self.full_redir, + requests_response(""" + +
+

foo bar

+
+ +"""), + ] + self._test_verify(True, False, None) + + def test_verify_non_representative_hcard(self, mock_get, _): + bad_hcard = requests_response( + 'acct:me@user.com', + url='https://user.com/', + ) + mock_get.side_effect = [self.full_redir, bad_hcard] + self._test_verify(True, False, None) + + def test_verify_both_work(self, mock_get, _): + hcard = requests_response(""" + + me + Masto +""", + url='https://user.com/', + ) + mock_get.side_effect = [self.full_redir, hcard] + self._test_verify(True, True, { + 'type': 'Person', + 'name': 'me', + 'url': ['http://localhost/r/https://user.com/', 'acct:myself@user.com'], + 'preferredUsername': 'user.com', + }) + + def test_verify_www_redirect(self, mock_get, _): + www_user = self.make_user('www.user.com') + + empty = requests_response('') + mock_get.side_effect = [ + requests_response(status=302, redirected_url='https://www.user.com/'), + empty, empty, + ] + + got = www_user.verify() + self.assertEqual('user.com', got.key.id()) + + root_user = Webmention.get_by_id('user.com') + self.assertEqual(root_user.key, www_user.key.get().use_instead) + self.assertEqual(root_user.key, Webmention.get_or_create('www.user.com').key) + + def test_verify_actor_rel_me_links(self, mock_get, _): + mock_get.side_effect = [ + self.full_redir, + requests_response(""" + +
+Mrs. ☕ Foo +should be ignored + + one text + + +
+ +""", url='https://user.com/'), + ] + self._test_verify(True, True, { + 'attachment': [{ + 'type': 'PropertyValue', + 'name': 'Mrs. ☕ Foo', + 'value': 'user.com/about-me', + }, { + 'type': 'PropertyValue', + 'name': 'Web site', + 'value': 'user.com', + }, { + 'type': 'PropertyValue', + 'name': 'one text', + 'value': 'one', + }, { + 'type': 'PropertyValue', + 'name': 'two title', + 'value': 'two', + }]}) + + def test_verify_override_preferredUsername(self, mock_get, _): + mock_get.side_effect = [ + self.full_redir, + requests_response(""" + + + Nick + + +""", url='https://user.com/'), + ] + self._test_verify(True, True, { + # stays y.z despite user's username. since Mastodon queries Webfinger + # for preferredUsername@fed.brid.gy + # https://github.com/snarfed/bridgy-fed/issues/77#issuecomment-949955109 + 'preferredUsername': 'user.com', + }) + + def test_homepage(self, _, __): + self.assertEqual('https://user.com/', self.user.homepage) + + def test_is_homepage(self, _, __): + for url in 'user.com', '//user.com', 'http://user.com', 'https://user.com': + self.assertTrue(self.user.is_homepage(url), url) + + for url in (None, '', 'user', 'com', 'com.user', 'ftp://user.com', + 'https://user', '://user.com'): + self.assertFalse(self.user.is_homepage(url), url) + @mock.patch('requests.post') @mock.patch('requests.get') -class WebmentionUtilTest(testutil.TestCase): +class WebmentionProtocolTest(testutil.TestCase): def setUp(self): super().setUp() diff --git a/tests/testutil.py b/tests/testutil.py index 4e7294f..9657724 100644 --- a/tests/testutil.py +++ b/tests/testutil.py @@ -25,23 +25,17 @@ import requests # load all Flask handlers import app from flask_app import app, cache, init_globals -import activitypub, common +import activitypub +import common +import models from models import Object, PROTOCOLS, Target, User import protocol - +from webmention import Webmention logger = logging.getLogger(__name__) -# used in TestCase.make_user() to reuse keys across Users since they're -# expensive to generate -requests.post(f'http://{ndb_client.host}/reset') -with ndb_client.context(): - global_user = User.get_or_create('user.com') - -Object.source_protocol = ndb.StringProperty(choices=PROTOCOLS + ('fake',)) - -class FakeProtocol(protocol.Protocol): +class Fake(User, protocol.Protocol): LABEL = 'fake' # maps string ids to dict AS1 objects. send adds objects here, fetch @@ -56,14 +50,14 @@ class FakeProtocol(protocol.Protocol): @classmethod def send(cls, obj, url, log_data=True): - logger.info(f'FakeProtocol.send {url}') + logger.info(f'Fake.send {url}') cls.sent.append((obj, url)) cls.objects[obj.key.id()] = obj @classmethod def fetch(cls, obj): id = obj.key.id() - logger.info(f'FakeProtocol.load {id}') + logger.info(f'Fake.load {id}') cls.fetched.append(id) if id in cls.objects: @@ -74,11 +68,21 @@ class FakeProtocol(protocol.Protocol): @classmethod def serve(cls, obj): - logger.info(f'FakeProtocol.load {obj.key.id()}') - return (f'FakeProtocol object {obj.key.id()}', + logger.info(f'Fake.load {obj.key.id()}') + return (f'Fake object {obj.key.id()}', {'Accept': 'fake/protocol'}) +# used in TestCase.make_user() to reuse keys across Users since they're +# expensive to generate +requests.post(f'http://{ndb_client.host}/reset') +with ndb_client.context(): + global_user = Fake.get_or_create('user.com') + + +models.reset_protocol_properties() + + class TestCase(unittest.TestCase, testutil.Asserts): maxDiff = None @@ -91,9 +95,9 @@ class TestCase(unittest.TestCase, testutil.Asserts): protocol.objects_cache.clear() common.webmention_discover.cache.clear() - FakeProtocol.objects = {} - FakeProtocol.sent = [] - FakeProtocol.fetched = [] + Fake.objects = {} + Fake.sent = [] + Fake.fetched = [] # make random test data deterministic arroba.util._clockid = 17 @@ -122,15 +126,16 @@ class TestCase(unittest.TestCase, testutil.Asserts): self.client.__exit__(None, None, None) super().tearDown() + # TODO(#512): switch default to Fake, start using that more @staticmethod - def make_user(domain, **kwargs): + def make_user(domain, cls=Webmention, **kwargs): """Reuse RSA key across Users because generating it is expensive.""" - user = User(id=domain, - mod=global_user.mod, - public_exponent=global_user.public_exponent, - private_exponent=global_user.private_exponent, - p256_key=global_user.p256_key, - **kwargs) + user = cls(id=domain, + mod=global_user.mod, + public_exponent=global_user.public_exponent, + private_exponent=global_user.private_exponent, + p256_key=global_user.p256_key, + **kwargs) user.put() return user diff --git a/ui.py b/ui.py index 7396e46..a9182c4 100644 --- a/ui.py +++ b/ui.py @@ -2,8 +2,9 @@ Needed for serving /convert/ui/webmention/... requests. """ +from models import User from protocol import Protocol -class UIProtocol(Protocol): +class UIProtocol(User, Protocol): LABEL = 'ui' diff --git a/webfinger.py b/webfinger.py index 1f62a67..cc32117 100644 --- a/webfinger.py +++ b/webfinger.py @@ -45,7 +45,7 @@ class Actor(flask_util.XrdOrJrd): if domain.split('.')[-1] in NON_TLDS: error(f"{domain} doesn't look like a domain", status=404) - g.user = User.get_by_id(domain) + g.user = Webmention.get_by_id(domain) if g.user: actor = g.user.to_as1() or {} homepage = g.user.homepage diff --git a/webmention.py b/webmention.py index 213a4d0..6c95bf3 100644 --- a/webmention.py +++ b/webmention.py @@ -1,7 +1,7 @@ """Handles inbound webmentions.""" +import difflib import logging -import urllib.parse -from urllib.parse import urlencode, urlparse +from urllib.parse import urlencode, urljoin, urlparse import feedparser from flask import g, redirect, request @@ -14,16 +14,15 @@ from oauth_dropins.webutil.appengine_config import tasks_client from oauth_dropins.webutil.appengine_info import APP_ID from oauth_dropins.webutil.flask_util import error, flash from oauth_dropins.webutil.util import json_dumps, json_loads -from oauth_dropins.webutil import webmention +import oauth_dropins.webutil.webmention as webutil_webmention from requests import HTTPError, RequestException, URLRequired -from werkzeug.exceptions import BadGateway, BadRequest, HTTPException +from werkzeug.exceptions import BadGateway, BadRequest, HTTPException, NotFound -from activitypub import ActivityPub +import activitypub from flask_app import app import common -from models import Follower, Object, Target, User -import models -from protocol import Protocol, protocols +from models import Follower, Object, PROTOCOLS, Target, User +from protocol import Protocol logger = logging.getLogger(__name__) @@ -32,11 +31,85 @@ TASKS_LOCATION = 'us-central1' CHAR_AFTER_SPACE = chr(ord(' ') + 1) +# https://github.com/snarfed/bridgy-fed/issues/314 +WWW_DOMAINS = frozenset(( + 'www.jvt.me', +)) -class Webmention(Protocol): - """Webmention protocol implementation.""" + +class Webmention(User, Protocol): + """Webmention user and protocol implementation. + + The key name is the domain. + """ LABEL = 'webmention' + @classmethod + def _get_kind(cls): + return 'MagicKey' + + def verify(self): + """Fetches site a couple ways to check for redirects and h-card. + + + Returns: :class:`Webmention` that was verified. May be different than + self! eg if self's domain started with www and we switch to the root + domain. + """ + domain = self.key.id() + logger.info(f'Verifying {domain}') + + if domain.startswith('www.') and domain not in WWW_DOMAINS: + # if root domain redirects to www, use root domain instead + # https://github.com/snarfed/bridgy-fed/issues/314 + root = domain.removeprefix("www.") + root_site = f'https://{root}/' + try: + resp = util.requests_get(root_site, gateway=False) + if resp.ok and self.is_homepage(resp.url): + logger.info(f'{root_site} redirects to {resp.url} ; using {root} instead') + root_user = Webmention.get_or_create(root) + self.use_instead = root_user.key + self.put() + return root_user.verify() + except RequestException: + pass + + # check webfinger redirect + path = f'/.well-known/webfinger?resource=acct:{domain}@{domain}' + self.has_redirects = False + self.redirects_error = None + try: + url = urljoin(self.homepage, path) + resp = util.requests_get(url, gateway=False) + domain_urls = ([f'https://{domain}/' for domain in common.DOMAINS] + + [common.host_url()]) + expected = [urljoin(url, path) for url in domain_urls] + if resp.ok: + if resp.url in expected: + self.has_redirects = True + elif resp.url: + diff = '\n'.join(difflib.Differ().compare([resp.url], [expected[0]])) + self.redirects_error = f'Current vs expected:
{diff}
' + else: + lines = [url, f' returned HTTP {resp.status_code}'] + if resp.url != url: + lines[1:1] = [' redirected to:', resp.url] + self.redirects_error = '
' + '\n'.join(lines) + '
' + except RequestException: + pass + + # check home page + try: + obj = Webmention.load(self.homepage, gateway=True) + self.actor_as2 = activitypub.postprocess_as2(as2.from_as1(obj.as1)) + self.has_hcard = True + except (BadRequest, NotFound): + self.actor_as2 = None + self.has_hcard = False + + return self + @classmethod def send(cls, obj, url): """Sends a webmention to a given target URL. @@ -48,7 +121,7 @@ class Webmention(Protocol): endpoint = common.webmention_discover(url).endpoint if endpoint: - webmention.send(endpoint, source_url, url) + webutil_webmention.send(endpoint, source_url, url) return True @classmethod @@ -130,7 +203,7 @@ class Webmention(Protocol): """Serves an :class:`Object` as HTML.""" obj_as1 = obj.as1 - from_proto = protocols.get(obj.source_protocol) + from_proto = PROTOCOLS.get(obj.source_protocol) if from_proto: # fill in author/actor if available for field in 'author', 'actor': @@ -167,7 +240,7 @@ def webmention_external(): error(f'Bad URL {source}') domain = util.domain_from_link(source, minimize=False) - g.user = User.get_by_id(domain) + g.user = Webmention.get_by_id(domain) if not g.user: error(f'No user found for domain {domain}') @@ -215,7 +288,7 @@ def webmention_task(): domain = util.domain_from_link(source, minimize=False) logger.info(f'webmention from {domain}') - g.user = User.get_by_id(domain) + g.user = Webmention.get_by_id(domain) if not g.user: error(f'No user found for domain {domain}', status=304) @@ -373,7 +446,7 @@ def webmention_task(): obj.target_as2 = target_as2 try: - last = ActivityPub.send(obj, inbox, log_data=log_data) + last = activitypub.ActivityPub.send(obj, inbox, log_data=log_data) obj.delivered.append(target) last_success = last except BaseException as e: @@ -429,7 +502,7 @@ def _activitypub_targets(obj): # fetch target page as AS2 object try: # TODO: make this generic across protocols - target_stored = ActivityPub.load(target) + target_stored = activitypub.ActivityPub.load(target) target_obj = target_stored.as2 or as2.from_as1(target_stored.as1) except (HTTPError, BadGateway) as e: resp = getattr(e, 'requests_response', None) @@ -455,7 +528,7 @@ def _activitypub_targets(obj): if not inbox_url: # fetch actor as AS object # TODO: make this generic across protocols - actor_obj = ActivityPub.load(actor) + actor_obj = activitypub.ActivityPub.load(actor) actor = actor_obj.as2 or as2.from_as1(actor_obj.as1) inbox_url = actor.get('inbox') @@ -464,7 +537,7 @@ def _activitypub_targets(obj): logger.error('Target actor has no inbox') continue - inbox_url = urllib.parse.urljoin(target, inbox_url) + inbox_url = urljoin(target, inbox_url) inboxes_to_targets[inbox_url] = target_obj if not targets or verb == 'share': diff --git a/xrpc_actor.py b/xrpc_actor.py index bdf92f7..a38541c 100644 --- a/xrpc_actor.py +++ b/xrpc_actor.py @@ -10,6 +10,7 @@ from oauth_dropins.webutil import util from flask_app import xrpc_server from models import User +from webmention import Webmention logger = logging.getLogger(__name__) @@ -24,7 +25,7 @@ def getProfile(input, actor=None): if not actor or not re.match(util.DOMAIN_RE, actor): raise ValueError(f'{actor} is not a domain') - g.user = User.get_by_id(actor) + g.user = Webmention.get_by_id(actor) if not g.user: raise ValueError(f'User {actor} not found') elif not g.user.actor_as2: diff --git a/xrpc_feed.py b/xrpc_feed.py index 161c9e6..e496a47 100644 --- a/xrpc_feed.py +++ b/xrpc_feed.py @@ -10,6 +10,7 @@ from oauth_dropins.webutil import util from flask_app import xrpc_server from models import Object, PAGE_SIZE, User +from webmention import Webmention logger = logging.getLogger(__name__) @@ -22,7 +23,7 @@ def getAuthorFeed(input, author=None, limit=None, before=None): if not author or not re.match(util.DOMAIN_RE, author): raise ValueError(f'{author} is not a domain') - g.user = User.get_by_id(author) + g.user = Webmention.get_by_id(author) if not g.user: raise ValueError(f'User {author} not found') elif not g.user.actor_as2: diff --git a/xrpc_graph.py b/xrpc_graph.py index b6f8d27..de6c52b 100644 --- a/xrpc_graph.py +++ b/xrpc_graph.py @@ -8,6 +8,7 @@ from oauth_dropins.webutil import util from flask_app import xrpc_server import common from models import Follower, User +from webmention import Webmention logger = logging.getLogger(__name__) @@ -25,7 +26,7 @@ def get_followers(query_prop, output_field, user=None, limit=50, before=None): # TODO: what is user? if not user or not re.match(util.DOMAIN_RE, user): raise ValueError(f'{user} is not a domain') - elif not User.get_by_id(user): + elif not Webmention.get_by_id(user): raise ValueError(f'Unknown user {user}') collection = 'followers' if output_field == 'followers' else 'following'