From 2fdc6f29a9c115e9726d2e42b6bbdb21c7993e0b Mon Sep 17 00:00:00 2001 From: Ryan Barrett Date: Thu, 24 Nov 2022 08:20:04 -0800 Subject: [PATCH] sign HTTP GET requests for AS2 objects to support Mastodon's AUTHORIZED_FETCH aka secure mode: https://docs.joinmastodon.org/admin/config/#authorized_fetch fixes #291 --- activitypub.py | 55 ++++------------------ add_webmention.py | 4 +- common.py | 97 +++++++++++++++++++++++++++++++-------- models.py | 5 +- tests/test_activitypub.py | 13 +++--- tests/test_common.py | 15 ++++-- tests/test_webmention.py | 57 +++++++++++++---------- tests/testutil.py | 25 ++++++---- webfinger.py | 2 +- webmention.py | 34 ++++++++------ 10 files changed, 178 insertions(+), 129 deletions(-) diff --git a/activitypub.py b/activitypub.py index 14dd649..6e50668 100644 --- a/activitypub.py +++ b/activitypub.py @@ -1,8 +1,6 @@ """Handles requests for ActivityPub endpoints: actors, inbox, etc. """ -from base64 import b64encode import datetime -from hashlib import sha256 import logging import re @@ -18,7 +16,6 @@ from app import app, cache import common from common import redirect_unwrap, redirect_wrap from models import Activity, Follower, User -from httpsig.requests_auth import HTTPSignatureAuth logger = logging.getLogger(__name__) @@ -40,45 +37,6 @@ SUPPORTED_TYPES = ( ) -def send(activity, inbox_url, user_domain): - """Sends an ActivityPub request to an inbox. - - Args: - activity: dict, AS2 activity - inbox_url: string - user_domain: string, domain of the bridgy fed user sending the request - - Returns: - requests.Response - """ - logger.info(f'Sending AP request from {user_domain}: {json_dumps(activity, indent=2)}') - - # prepare HTTP Signature (required by Mastodon) - # https://w3c.github.io/activitypub/#authorization - # https://tools.ietf.org/html/draft-cavage-http-signatures-07 - # https://github.com/tootsuite/mastodon/issues/4906#issuecomment-328844846 - key_id = request.host_url + user_domain - user = User.get_or_create(user_domain) - auth = HTTPSignatureAuth(secret=user.private_pem(), key_id=key_id, - algorithm='rsa-sha256', sign_header='signature', - headers=('Date', 'Digest', 'Host')) - - # deliver to inbox - body = json_dumps(activity).encode() - headers = { - 'Content-Type': common.CONTENT_TYPE_AS2, - # required for HTTP Signature - # https://tools.ietf.org/html/draft-cavage-http-signatures-07#section-2.1.3 - 'Date': datetime.datetime.utcnow().strftime('%a, %d %b %Y %H:%M:%S GMT'), - # required by Mastodon - # https://github.com/tootsuite/mastodon/pull/14556#issuecomment-674077648 - 'Digest': 'SHA-256=' + b64encode(sha256(body).digest()).decode(), - 'Host': util.domain_from_link(inbox_url, minimize=False), - } - return common.requests_post(inbox_url, data=body, auth=auth, - headers=headers) - - @app.get(f'/') @flask_util.cached(cache, CACHE_TIME) def actor(domain): @@ -102,7 +60,7 @@ def inbox(domain=None): activity = request.json assert activity except (TypeError, ValueError, AssertionError): - error("Couldn't parse body as JSON", exc_info=True) + error(f"Couldn't parse body as JSON", exc_info=True) logger.info(f'Got: {json_dumps(activity, indent=2)}') @@ -119,6 +77,8 @@ def inbox(domain=None): # TODO: verify signature if there is one + user = User.get_or_create(domain) if domain else None + if type == 'Undo' and obj.get('type') == 'Follow': # skip actor fetch below; we don't need it to undo a follow undo_follow(redirect_unwrap(activity)) @@ -148,11 +108,11 @@ def inbox(domain=None): # fetch actor if necessary so we have name, profile photo, etc actor = activity.get('actor') if actor and isinstance(actor, str): - actor = activity['actor'] = common.get_as2(actor).json() + actor = activity['actor'] = common.get_as2(actor, user=user).json() activity_unwrapped = redirect_unwrap(activity) if type == 'Follow': - return accept_follow(activity, activity_unwrapped) + return accept_follow(activity, activity_unwrapped, user) # send webmentions to each target as1 = as2.to_as1(activity) @@ -186,12 +146,13 @@ def inbox(domain=None): return '' -def accept_follow(follow, follow_unwrapped): +def accept_follow(follow, follow_unwrapped, user): """Replies to an AP Follow request with an Accept request. Args: follow: dict, AP Follow activity follow_unwrapped: dict, same, except with redirect URLs unwrapped + user: :class:`User` """ logger.info('Replying to Follow with Accept') @@ -224,7 +185,7 @@ def accept_follow(follow, follow_unwrapped): 'object': followee, } } - resp = send(accept, inbox, user_domain) + resp = common.signed_post(inbox, data=accept, user=user) # send webmention common.send_webmentions(as2.to_as1(follow), proxy=True, protocol='activitypub', diff --git a/add_webmention.py b/add_webmention.py index 1e7f208..1105640 100644 --- a/add_webmention.py +++ b/add_webmention.py @@ -5,7 +5,7 @@ import urllib.parse import flask from flask import request -from oauth_dropins.webutil import flask_util +from oauth_dropins.webutil import flask_util, util from oauth_dropins.webutil.flask_util import error import requests @@ -25,7 +25,7 @@ def add_wm(url=None): error('URL must start with http:// or https://') try: - got = common.requests_get(url) + got = util.requests_get(url) except requests.exceptions.Timeout as e: error(str(e), status=504, exc_info=True) except requests.exceptions.RequestException as e: diff --git a/common.py b/common.py index b7f7616..06b01a3 100644 --- a/common.py +++ b/common.py @@ -1,6 +1,9 @@ # coding=utf-8 """Misc common utilities. """ +from base64 import b64encode +import datetime +from hashlib import sha256 import itertools import logging import os @@ -9,6 +12,7 @@ import urllib.parse from flask import request from granary import as2, microformats2 +from httpsig.requests_auth import HTTPSignatureAuth import mf2util from oauth_dropins.webutil import util, webmention from oauth_dropins.webutil.flask_util import error @@ -72,19 +76,71 @@ DOMAIN_BLOCKLIST = frozenset(( 'twitter.com', ) + DOMAINS) +_DEFAULT_SIGNATURE_USER = None -def requests_get(url, **kwargs): - return _requests_fn(util.requests_get, url, **kwargs) +# alias allows unit tests to mock the function +utcnow = datetime.datetime.utcnow -def requests_post(url, **kwargs): - return _requests_fn(util.requests_post, url, **kwargs) +def default_signature_user(): + global _DEFAULT_SIGNATURE_USER + if _DEFAULT_SIGNATURE_USER is None: + _DEFAULT_SIGNATURE_USER = User.get_or_create('snarfed.org') + return _DEFAULT_SIGNATURE_USER -def _requests_fn(fn, url, parse_json=False, **kwargs): - """Wraps requests.* and adds raise_for_status().""" +def signed_get(url, **kwargs): + return signed_request(util.requests_get, url, **kwargs) + + +def signed_post(url, **kwargs): + return signed_request(util.requests_post, url, **kwargs) + + +def signed_request(fn, url, data=None, user=None, headers=None, **kwargs): + """Wraps requests.* and adds HTTP Signature. + + Args: + fn: :func:`util.requests_get` or :func:`util.requests_get` + url: str + data: optional AS2 object + user: optional :class:`User` to sign request with + kwargs: passed through to requests + + Returns: :class:`requests.Response` + """ + if headers is None: + headers = {} + + # prepare HTTP Signature and headers + if not user: + user = default_signature_user() + + if data: + data = kwargs['data'] = json_dumps(data).encode() + + headers.update({ + # required for HTTP Signature + # https://tools.ietf.org/html/draft-cavage-http-signatures-07#section-2.1.3 + 'Date': utcnow().strftime('%a, %d %b %Y %H:%M:%S GMT'), + # required by Mastodon + # https://github.com/tootsuite/mastodon/pull/14556#issuecomment-674077648 + 'Host': util.domain_from_link(url, minimize=False), + 'Content-Type': common.CONTENT_TYPE_AS2, + # required for HTTP Signature and Mastodon + 'Digest': f'SHA-256={b64encode(sha256(data or b"").digest()).decode()}', + }) + + domain = user.key.id() + logger.info(f"Signing with {domain}'s key") + key_id = request.host_url + domain + auth = HTTPSignatureAuth(secret=user.private_pem(), key_id=key_id, + algorithm='rsa-sha256', sign_header='signature', + headers=('Date', 'Host', 'Digest')) + + # make HTTP request kwargs.setdefault('gateway', True) - resp = fn(url, **kwargs) + resp = fn(url, auth=auth, headers=headers, **kwargs) logger.info(f'Got {resp.status_code} headers: {resp.headers}') type = content_type(resp) @@ -92,26 +148,29 @@ def _requests_fn(fn, url, parse_json=False, **kwargs): (type.startswith('text/') or type.endswith('+json') or type.endswith('/json'))): logger.info(resp.text) - if parse_json: - try: - return resp.json() - except ValueError: - msg = "Couldn't parse response as JSON" - logger.info(msg, exc_info=True) - raise BadGateway(msg) - return resp -def get_as2(url): +def get_as2(url, user=None): """Tries to fetch the given URL as ActivityStreams 2. Uses HTTP content negotiation via the Content-Type header. If the url is HTML and it has a rel-alternate link with an AS2 content type, fetches and returns that URL. + Includes an HTTP Signature with the request. + https://w3c.github.io/activitypub/#authorization + https://tools.ietf.org/html/draft-cavage-http-signatures-07 + https://github.com/mastodon/mastodon/pull/11269 + + Mastodon requires this signature if AUTHORIZED_FETCH aka secure mode is on: + https://docs.joinmastodon.org/admin/config/#authorized_fetch + + If user is not provided, defaults to using @snarfed.org@snarfed.org's key. + Args: url: string + user: :class:`User` used to sign request Returns: :class:`requests.Response` @@ -129,7 +188,7 @@ def get_as2(url): err.requests_response = resp raise err - resp = requests_get(url, headers=CONNEG_HEADERS_AS2_HTML) + resp = signed_get(url, user=user, headers=CONNEG_HEADERS_AS2_HTML) if content_type(resp) in (CONTENT_TYPE_AS2, CONTENT_TYPE_AS2_LD): return resp @@ -139,8 +198,8 @@ def get_as2(url): if not (as2 and as2['href']): _error(resp) - resp = requests_get(urllib.parse.urljoin(resp.url, as2['href']), - headers=CONNEG_HEADERS_AS2) + resp = signed_get(urllib.parse.urljoin(resp.url, as2['href']), + headers=CONNEG_HEADERS_AS2) if content_type(resp) in (CONTENT_TYPE_AS2, CONTENT_TYPE_AS2_LD): return resp diff --git a/models.py b/models.py index 6bfa960..fcb21c7 100644 --- a/models.py +++ b/models.py @@ -11,6 +11,7 @@ from flask import request from google.cloud import ndb from granary import as2, microformats2 from oauth_dropins.webutil.models import StringIdModel +from oauth_dropins.webutil import util from oauth_dropins.webutil.util import json_dumps, json_loads import common @@ -94,8 +95,8 @@ class User(StringIdModel): # check webfinger redirect path = f'/.well-known/webfinger?resource=acct:{domain}@{domain}' try: - resp = common.requests_get(urllib.parse.urljoin(site, path), - allow_redirects=False, gateway=False) + resp = util.requests_get(urllib.parse.urljoin(site, path), + allow_redirects=False, gateway=False) domain_urls = ([f'https://{domain}/' for domain in common.DOMAINS] + [request.host_url]) expected = [urllib.parse.urljoin(url, path) for url in domain_urls] diff --git a/tests/test_activitypub.py b/tests/test_activitypub.py index f5bc2d0..ae67d16 100644 --- a/tests/test_activitypub.py +++ b/tests/test_activitypub.py @@ -364,9 +364,10 @@ class ActivityPubTest(testutil.TestCase): got = self.client.post('/foo.com/inbox', json=LIKE) self.assertEqual(200, got.status_code) - self.assert_req(mock_get, 'http://orig/actor', - headers=common.CONNEG_HEADERS_AS2_HTML) - self.assert_req(mock_get, 'http://orig/post') + mock_get.assert_has_calls(( + self.as2_req('http://orig/actor'), + self.req('http://orig/post'), + )), args, kwargs = mock_post.call_args self.assertEqual(('http://orig/webmention',), args) @@ -398,8 +399,9 @@ class ActivityPubTest(testutil.TestCase): got = self.client.post('/foo.com/inbox', json=FOLLOW_WRAPPED) self.assertEqual(200, got.status_code) - self.assert_req(mock_get, FOLLOW['actor'], - headers=common.CONNEG_HEADERS_AS2_HTML) + mock_get.assert_has_calls(( + self.as2_req(FOLLOW['actor']), + )) # check AP Accept self.assertEqual(2, len(mock_post.call_args_list)) @@ -489,7 +491,6 @@ class ActivityPubTest(testutil.TestCase): self.assertEqual('inactive', followee.key.get().status) self.assertEqual('active', other.key.get().status) - def test_inbox_webmention_discovery_connection_fails(self, mock_head, mock_get, mock_post): mock_get.side_effect = [ diff --git a/tests/test_common.py b/tests/test_common.py index cd72baa..f47fc9f 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -31,12 +31,21 @@ NOT_ACCEPTABLE = requests_response(status=406) class CommonTest(testutil.TestCase): + def setUp(self): + super().setUp() + self.app_context = app.test_request_context('/') + self.app_context.push() + + def tearDown(self): + self.app_context.pop() + super().tearDown() + @mock.patch('requests.get', return_value=AS2) def test_get_as2_direct(self, mock_get): resp = common.get_as2('http://orig') self.assertEqual(AS2, resp) mock_get.assert_has_calls(( - self.req('http://orig', headers=common.CONNEG_HEADERS_AS2_HTML), + self.as2_req('http://orig'), )) @mock.patch('requests.get', side_effect=[HTML_WITH_AS2, AS2]) @@ -44,8 +53,8 @@ class CommonTest(testutil.TestCase): resp = common.get_as2('http://orig') self.assertEqual(AS2, resp) mock_get.assert_has_calls(( - self.req('http://orig', headers=common.CONNEG_HEADERS_AS2_HTML), - self.req('http://as2', headers=common.CONNEG_HEADERS_AS2), + self.as2_req('http://orig'), + self.as2_req('http://as2', headers=common.CONNEG_HEADERS_AS2), )) @mock.patch('requests.get', return_value=HTML) diff --git a/tests/test_webmention.py b/tests/test_webmention.py index 8e049e2..3b61646 100644 --- a/tests/test_webmention.py +++ b/tests/test_webmention.py @@ -24,6 +24,7 @@ from common import ( CONTENT_TYPE_ATOM, CONTENT_TYPE_HTML, CONTENT_TYPE_MAGIC_ENVELOPE, + default_signature_user, ) from models import Follower, User, Activity import webmention @@ -68,7 +69,7 @@ REPOST_AS2 = { class WebmentionTest(testutil.TestCase): def setUp(self): super().setUp() - self.key = User.get_or_create('a') + self.user = User.get_or_create('a') self.orig_html_as2 = requests_response("""\ @@ -309,7 +310,7 @@ class WebmentionTest(testutil.TestCase): kwargs['headers']['Content-Type']) env = utils.parse_magic_envelope(kwargs['data']) - assert magicsigs.verify(env['data'], env['sig'].encode(), key=self.key) + assert magicsigs.verify(env['data'], env['sig'].encode(), key=self.user) return env['data'] @@ -426,9 +427,9 @@ class WebmentionTest(testutil.TestCase): mock_get.assert_has_calls(( self.req('http://a/reply'), - self.req('http://not/fediverse', headers=CONNEG_HEADERS_AS2_HTML), - self.req('http://orig/post', headers=CONNEG_HEADERS_AS2_HTML), - self.req('http://orig/author', headers=CONNEG_HEADERS_AS2_HTML), + self.as2_req('http://not/fediverse'), + self.as2_req('http://orig/post'), + self.as2_req('http://orig/author'), )) args, kwargs = mock_post.call_args @@ -439,7 +440,7 @@ class WebmentionTest(testutil.TestCase): self.assertEqual(CONTENT_TYPE_AS2, headers['Content-Type']) rsa_key = kwargs['auth'].header_signer._rsa._key - self.assertEqual(self.key.private_pem(), rsa_key.exportKey()) + self.assertEqual(self.user.private_pem(), rsa_key.exportKey()) activity = Activity.get_by_id('http://a/reply http://orig/as2') self.assertEqual(['a'], activity.domain) @@ -526,9 +527,9 @@ class WebmentionTest(testutil.TestCase): mock_get.assert_has_calls(( self.req('http://a/reply'), - self.req('http://not/fediverse', headers=CONNEG_HEADERS_AS2_HTML), - self.req('http://orig/post', headers=CONNEG_HEADERS_AS2_HTML), - self.req('http://orig/author', headers=CONNEG_HEADERS_AS2_HTML), + self.as2_req('http://not/fediverse'), + self.as2_req('http://orig/post'), + self.as2_req('http://orig/author'), )) args, kwargs = mock_post.call_args @@ -547,8 +548,8 @@ class WebmentionTest(testutil.TestCase): mock_get.assert_has_calls(( self.req('http://a/repost'), - self.req('http://orig/post', headers=CONNEG_HEADERS_AS2_HTML), - self.req('http://orig/author', headers=CONNEG_HEADERS_AS2_HTML), + self.as2_req('http://orig/post'), + self.as2_req('http://orig/author'), )) args, kwargs = mock_post.call_args @@ -559,7 +560,13 @@ class WebmentionTest(testutil.TestCase): self.assertEqual(CONTENT_TYPE_AS2, headers['Content-Type']) rsa_key = kwargs['auth'].header_signer._rsa._key - self.assertEqual(self.key.private_pem(), rsa_key.exportKey()) + self.assertEqual(self.user.private_pem(), rsa_key.exportKey()) + + for args, kwargs in mock_get.call_args_list[1:]: + with self.subTest(url=args[0]): + rsa_key = kwargs['auth'].header_signer._rsa._key + self.assertEqual(default_signature_user().private_pem(), + rsa_key.exportKey()) activity = Activity.get_by_id('http://a/repost http://orig/as2') self.assertEqual(['a'], activity.domain) @@ -581,10 +588,10 @@ class WebmentionTest(testutil.TestCase): mock_get.assert_has_calls(( self.req('http://a/reply'), - self.req('http://not/fediverse', headers=CONNEG_HEADERS_AS2_HTML), - self.req('http://orig/post', headers=CONNEG_HEADERS_AS2_HTML), - self.req('http://orig/as2', headers=CONNEG_HEADERS_AS2), - self.req('http://orig/author', headers=CONNEG_HEADERS_AS2_HTML), + self.as2_req('http://not/fediverse'), + self.as2_req('http://orig/post'), + self.req('http://orig/as2', auth=mock.ANY, headers=CONNEG_HEADERS_AS2), + self.as2_req('http://orig/author'), )) args, kwargs = mock_post.call_args @@ -773,7 +780,7 @@ class WebmentionTest(testutil.TestCase): mock_get.assert_has_calls(( self.req('http://a/follow'), - self.req('http://followee/', headers=CONNEG_HEADERS_AS2_HTML), + self.as2_req('http://followee/'), )) args, kwargs = mock_post.call_args @@ -784,7 +791,7 @@ class WebmentionTest(testutil.TestCase): self.assertEqual(CONTENT_TYPE_AS2, headers['Content-Type']) rsa_key = kwargs['auth'].header_signer._rsa._key - self.assertEqual(self.key.private_pem(), rsa_key.exportKey()) + self.assertEqual(self.user.private_pem(), rsa_key.exportKey()) activity = Activity.get_by_id('http://a/follow http://followee/') self.assertEqual(['a'], activity.domain) @@ -811,7 +818,7 @@ class WebmentionTest(testutil.TestCase): mock_get.assert_has_calls(( self.req('http://a/follow#2'), - self.req('http://followee/', headers=CONNEG_HEADERS_AS2_HTML), + self.as2_req('http://followee/'), )) args, kwargs = mock_post.call_args @@ -822,7 +829,7 @@ class WebmentionTest(testutil.TestCase): self.assert_equals(CONTENT_TYPE_AS2, headers['Content-Type']) rsa_key = kwargs['auth'].header_signer._rsa._key - self.assert_equals(self.key.private_pem(), rsa_key.exportKey()) + self.assert_equals(self.user.private_pem(), rsa_key.exportKey()) activity = Activity.get_by_id('http://a/follow__2 http://followee/') self.assert_equals(['a'], activity.domain) @@ -863,7 +870,7 @@ class WebmentionTest(testutil.TestCase): mock_get.assert_has_calls(( self.req('http://a/follow'), - self.req('http://followee/', headers=CONNEG_HEADERS_AS2_HTML), + self.as2_req('http://followee/'), )) args, kwargs = mock_post.call_args @@ -874,7 +881,7 @@ class WebmentionTest(testutil.TestCase): self.assertEqual(CONTENT_TYPE_AS2, headers['Content-Type']) rsa_key = kwargs['auth'].header_signer._rsa._key - self.assertEqual(self.key.private_pem(), rsa_key.exportKey()) + self.assertEqual(self.user.private_pem(), rsa_key.exportKey()) activity = Activity.get_by_id('http://a/follow http://followee/') self.assertEqual(['a'], activity.domain) @@ -907,8 +914,8 @@ class WebmentionTest(testutil.TestCase): mock_get.assert_has_calls(( self.req('http://a/reply'), - self.req('http://not/fediverse', headers=CONNEG_HEADERS_AS2_HTML), - self.req('http://orig/post', headers=CONNEG_HEADERS_AS2_HTML), + self.as2_req('http://not/fediverse'), + self.as2_req('http://orig/post'), self.req('http://orig/atom'), )) @@ -951,7 +958,7 @@ class WebmentionTest(testutil.TestCase): mock_get.assert_has_calls(( self.req('http://a/like'), - self.req('http://orig/post', headers=CONNEG_HEADERS_AS2_HTML), + self.as2_req('http://orig/post'), self.req('http://orig/atom'), )) diff --git a/tests/testutil.py b/tests/testutil.py index ce453f1..0805e57 100644 --- a/tests/testutil.py +++ b/tests/testutil.py @@ -1,6 +1,7 @@ """Common test utility code. """ import copy +import datetime import unittest from unittest.mock import ANY, call @@ -11,6 +12,8 @@ import requests import common +NOW = datetime.datetime(2022, 11, 23, 22, 29, 19) + class TestCase(unittest.TestCase, testutil.Asserts): maxDiff = None @@ -20,6 +23,7 @@ class TestCase(unittest.TestCase, testutil.Asserts): app.testing = True cache.clear() self.client = app.test_client() + common.utcnow = lambda: NOW # clear datastore requests.post('http://%s/reset' % ndb_client.host) @@ -32,22 +36,25 @@ class TestCase(unittest.TestCase, testutil.Asserts): def req(self, url, **kwargs): """Returns a mock requests call.""" - existing = kwargs.get('headers', {}) - if existing is not ANY: - kwargs['headers'] = { - 'User-Agent': util.user_agent, - **existing, - } - + kwargs.setdefault('headers', {}).update({ + 'User-Agent': util.user_agent, + }) kwargs.setdefault('timeout', util.HTTP_TIMEOUT) kwargs.setdefault('stream', True) - return call(url, **kwargs) + def as2_req(self, url, **kwargs): + headers = { + 'Date': 'Wed, 23 Nov 2022 22:29:19 GMT', + 'Host': util.domain_from_link(url, minimize=False), + **common.CONNEG_HEADERS_AS2_HTML, + **kwargs.pop('headers', {}), + } + return self.req(url, auth=ANY, headers=headers, **kwargs) + def assert_req(self, mock, url, **kwargs): """Checks a mock requests call.""" kwargs.setdefault('headers', {}).setdefault('User-Agent', 'Bridgy Fed (https://fed.brid.gy/)') kwargs.setdefault('stream', True) kwargs.setdefault('timeout', util.HTTP_TIMEOUT) - mock.assert_any_call(url, **kwargs) diff --git a/webfinger.py b/webfinger.py index 34cbd2c..17c6767 100644 --- a/webfinger.py +++ b/webfinger.py @@ -47,7 +47,7 @@ class Actor(flask_util.XrdOrJrd): urls = [url, urllib.parse.urljoin(url, '/')] + urls for candidate in urls: - resp = common.requests_get(candidate) + resp = util.requests_get(candidate) parsed = util.parse_html(resp) mf2 = util.parse_mf2(parsed, url=resp.url) # logger.debug(f'Parsed mf2 for {resp.url}: {json_dumps(mf2, indent=2)}') diff --git a/webmention.py b/webmention.py index dc56733..b2d3209 100644 --- a/webmention.py +++ b/webmention.py @@ -25,7 +25,7 @@ from werkzeug.exceptions import BadGateway import activitypub from app import app import common -from models import Follower, User, Activity +from models import Activity, Follower, User logger = logging.getLogger(__name__) @@ -39,13 +39,17 @@ class Webmention(View): source_mf2 = None # parsed mf2 dict source_obj = None # parsed AS1 dict target_resp = None # requests.Response + user = None # User def dispatch_request(self): logger.info(f'Params: {list(request.form.items())}') # fetch source page source = flask_util.get_required_param('source') - source_resp = common.requests_get(source) + try: + source_resp = util.requests_get(source) + except ValueError as e: + error(f'Bad source URL: {source}: {e}') self.source_url = source_resp.url or source self.source_domain = urllib.parse.urlparse(self.source_url).netloc.split(':')[0] fragment = urllib.parse.urlparse(self.source_url).fragment @@ -78,6 +82,7 @@ class Webmention(View): self.source_obj = microformats2.json_to_object(entry, fetch_mf2=True) logger.info(f'Converted to AS1: {json_dumps(self.source_obj, indent=2)}') + self.user = User.get_or_create(self.source_domain) for method in self.try_activitypub, self.try_salmon: ret = method() if ret: @@ -95,7 +100,6 @@ class Webmention(View): if not targets: return None - user = User.get_or_create(self.source_domain) error = None last_success = None @@ -110,7 +114,7 @@ class Webmention(View): for resp, inbox in targets: target_obj = json_loads(resp.target_as2) if resp.target_as2 else None source_activity = common.postprocess_as2( - as2.from_as1(self.source_obj), target=target_obj, user=user) + as2.from_as1(self.source_obj), target=target_obj, user=self.user) if resp.status == 'complete': if resp.source_mf2: @@ -131,7 +135,7 @@ class Webmention(View): source_activity['type'] = 'Update' try: - last = activitypub.send(source_activity, inbox, self.source_domain) + last = common.signed_post(inbox, data=source_activity, user=self.user) resp.status = 'complete' last_success = last except BaseException as e: @@ -291,7 +295,7 @@ class Webmention(View): """ # fetch target HTML page, extract Atom rel-alternate link if not self.target_resp: - self.target_resp = common.requests_get(target) + self.target_resp = util.requests_get(target) parsed = util.parse_html(self.target_resp) atom_url = parsed.find('link', rel='alternate', type=common.CONTENT_TYPE_ATOM) @@ -307,7 +311,7 @@ class Webmention(View): atom_url = urllib.parse.urljoin( target, urllib.parse.urljoin(base_url, atom_link['href'])) - feed = common.requests_get(atom_url).text + feed = util.requests_get(atom_url).text parsed = feedparser.parse(feed) entry = parsed.entries[0] logger.info(f'Parsed: {json_dumps(entry, indent=2)}') @@ -344,11 +348,13 @@ class Webmention(View): (author.get('name', ''), parsed.netloc)) try: # TODO: always https? - profile = common.requests_get( + profile = util.requests_get( '%s://%s/.well-known/webfinger?resource=acct:%s' % - (parsed.scheme, parsed.netloc, email), parse_json=True) - endpoint = django_salmon.get_salmon_replies_link(profile) - except requests.HTTPError as e: + (parsed.scheme, parsed.netloc, email)) + endpoint = django_salmon.get_salmon_replies_link(profile.json()) + except ValueError: + logging.warning("Couldn't parse response as JSON") + except requests.HTTPError: pass if not endpoint: @@ -364,13 +370,11 @@ class Webmention(View): # sign reply and wrap in magic envelope domain = urllib.parse.urlparse(self.source_url).netloc - user = User.get_or_create(domain) - logger.info(f'Using key for {domain}: {user}') magic_envelope = magicsigs.magic_envelope( - entry, common.CONTENT_TYPE_ATOM, user).decode() + entry, common.CONTENT_TYPE_ATOM, self.user).decode() logger.info(f'Sending Salmon slap to {endpoint}') - common.requests_post( + util.requests_post( endpoint, data=common.XML_UTF8 + magic_envelope, headers={'Content-Type': common.CONTENT_TYPE_MAGIC_ENVELOPE})