sign HTTP GET requests for AS2 objects

to support Mastodon's AUTHORIZED_FETCH aka secure mode: https://docs.joinmastodon.org/admin/config/#authorized_fetch

fixes #291

activitypub.py

@@ -1,8 +1,6 @@
 """Handles requests for ActivityPub endpoints: actors, inbox, etc.
 """
-from base64 import b64encode
 import datetime
-from hashlib import sha256
 import logging
 import re
 
@@ -18,7 +16,6 @@ from app import app, cache
 import common
 from common import redirect_unwrap, redirect_wrap
 from models import Activity, Follower, User
-from httpsig.requests_auth import HTTPSignatureAuth
 
 logger = logging.getLogger(__name__)
 
@@ -40,45 +37,6 @@ SUPPORTED_TYPES = (
 )
 
 
-def send(activity, inbox_url, user_domain):
-    """Sends an ActivityPub request to an inbox.
-
-    Args:
-      activity: dict, AS2 activity
-      inbox_url: string
-      user_domain: string, domain of the bridgy fed user sending the request
-
-    Returns:
-      requests.Response
-    """
-    logger.info(f'Sending AP request from {user_domain}: {json_dumps(activity, indent=2)}')
-
-    # prepare HTTP Signature (required by Mastodon)
-    # https://w3c.github.io/activitypub/#authorization
-    # https://tools.ietf.org/html/draft-cavage-http-signatures-07
-    # https://github.com/tootsuite/mastodon/issues/4906#issuecomment-328844846
-    key_id = request.host_url + user_domain
-    user = User.get_or_create(user_domain)
-    auth = HTTPSignatureAuth(secret=user.private_pem(), key_id=key_id,
-                             algorithm='rsa-sha256', sign_header='signature',
-                             headers=('Date', 'Digest', 'Host'))
-
-    # deliver to inbox
-    body = json_dumps(activity).encode()
-    headers = {
-        'Content-Type': common.CONTENT_TYPE_AS2,
-        # required for HTTP Signature
-        # https://tools.ietf.org/html/draft-cavage-http-signatures-07#section-2.1.3
-        'Date': datetime.datetime.utcnow().strftime('%a, %d %b %Y %H:%M:%S GMT'),
-        # required by Mastodon
-        # https://github.com/tootsuite/mastodon/pull/14556#issuecomment-674077648
-        'Digest': 'SHA-256=' + b64encode(sha256(body).digest()).decode(),
-        'Host': util.domain_from_link(inbox_url, minimize=False),
-    }
-    return common.requests_post(inbox_url, data=body, auth=auth,
-                                headers=headers)
-
-
 @app.get(f'/<regex("{common.DOMAIN_RE}"):domain>')
 @flask_util.cached(cache, CACHE_TIME)
 def actor(domain):
@@ -102,7 +60,7 @@ def inbox(domain=None):
         activity = request.json
         assert activity
     except (TypeError, ValueError, AssertionError):
-        error("Couldn't parse body as JSON", exc_info=True)
+        error(f"Couldn't parse body as JSON", exc_info=True)
 
     logger.info(f'Got: {json_dumps(activity, indent=2)}')
 
@@ -119,6 +77,8 @@ def inbox(domain=None):
 
     # TODO: verify signature if there is one
 
+    user = User.get_or_create(domain) if domain else None
+
     if type == 'Undo' and obj.get('type') == 'Follow':
         # skip actor fetch below; we don't need it to undo a follow
         undo_follow(redirect_unwrap(activity))
@@ -148,11 +108,11 @@ def inbox(domain=None):
     # fetch actor if necessary so we have name, profile photo, etc
     actor = activity.get('actor')
     if actor and isinstance(actor, str):
-        actor = activity['actor'] = common.get_as2(actor).json()
+        actor = activity['actor'] = common.get_as2(actor, user=user).json()
 
     activity_unwrapped = redirect_unwrap(activity)
     if type == 'Follow':
-        return accept_follow(activity, activity_unwrapped)
+        return accept_follow(activity, activity_unwrapped, user)
 
     # send webmentions to each target
     as1 = as2.to_as1(activity)
@@ -186,12 +146,13 @@ def inbox(domain=None):
     return ''
 
 
-def accept_follow(follow, follow_unwrapped):
+def accept_follow(follow, follow_unwrapped, user):
     """Replies to an AP Follow request with an Accept request.
 
     Args:
       follow: dict, AP Follow activity
       follow_unwrapped: dict, same, except with redirect URLs unwrapped
+      user: :class:`User`
     """
     logger.info('Replying to Follow with Accept')
 
@@ -224,7 +185,7 @@ def accept_follow(follow, follow_unwrapped):
             'object': followee,
         }
     }
-    resp = send(accept, inbox, user_domain)
+    resp = common.signed_post(inbox, data=accept, user=user)
 
     # send webmention
     common.send_webmentions(as2.to_as1(follow), proxy=True, protocol='activitypub',

add_webmention.py

@@ -5,7 +5,7 @@ import urllib.parse
 
 import flask
 from flask import request
-from oauth_dropins.webutil import flask_util
+from oauth_dropins.webutil import flask_util, util
 from oauth_dropins.webutil.flask_util import error
 import requests
 
@@ -25,7 +25,7 @@ def add_wm(url=None):
         error('URL must start with http:// or https://')
 
     try:
-        got = common.requests_get(url)
+        got = util.requests_get(url)
     except requests.exceptions.Timeout as e:
         error(str(e), status=504, exc_info=True)
     except requests.exceptions.RequestException as e:

common.py

@@ -1,6 +1,9 @@
 # coding=utf-8
 """Misc common utilities.
 """
+from base64 import b64encode
+import datetime
+from hashlib import sha256
 import itertools
 import logging
 import os
@@ -9,6 +12,7 @@ import urllib.parse
 
 from flask import request
 from granary import as2, microformats2
+from httpsig.requests_auth import HTTPSignatureAuth
 import mf2util
 from oauth_dropins.webutil import util, webmention
 from oauth_dropins.webutil.flask_util import error
@@ -72,19 +76,71 @@ DOMAIN_BLOCKLIST = frozenset((
     'twitter.com',
 ) + DOMAINS)
 
+_DEFAULT_SIGNATURE_USER = None
 
-def requests_get(url, **kwargs):
+# alias allows unit tests to mock the function
-    return _requests_fn(util.requests_get, url, **kwargs)
+utcnow = datetime.datetime.utcnow
 
 
-def requests_post(url, **kwargs):
+def default_signature_user():
-    return _requests_fn(util.requests_post, url, **kwargs)
+    global _DEFAULT_SIGNATURE_USER
+    if _DEFAULT_SIGNATURE_USER is None:
+        _DEFAULT_SIGNATURE_USER = User.get_or_create('snarfed.org')
+    return _DEFAULT_SIGNATURE_USER
 
 
-def _requests_fn(fn, url, parse_json=False, **kwargs):
+def signed_get(url, **kwargs):
-    """Wraps requests.* and adds raise_for_status()."""
+    return signed_request(util.requests_get, url, **kwargs)
+
+
+def signed_post(url, **kwargs):
+    return signed_request(util.requests_post, url, **kwargs)
+
+
+def signed_request(fn, url, data=None, user=None, headers=None, **kwargs):
+    """Wraps requests.* and adds HTTP Signature.
+
+    Args:
+      fn: :func:`util.requests_get` or  :func:`util.requests_get`
+      url: str
+      data: optional AS2 object
+      user: optional :class:`User` to sign request with
+      kwargs: passed through to requests
+
+    Returns: :class:`requests.Response`
+    """
+    if headers is None:
+        headers = {}
+
+    # prepare HTTP Signature and headers
+    if not user:
+        user = default_signature_user()
+
+    if data:
+        data = kwargs['data'] = json_dumps(data).encode()
+
+    headers.update({
+        # required for HTTP Signature
+        # https://tools.ietf.org/html/draft-cavage-http-signatures-07#section-2.1.3
+        'Date': utcnow().strftime('%a, %d %b %Y %H:%M:%S GMT'),
+        # required by Mastodon
+        # https://github.com/tootsuite/mastodon/pull/14556#issuecomment-674077648
+        'Host': util.domain_from_link(url, minimize=False),
+        'Content-Type': common.CONTENT_TYPE_AS2,
+        # required for HTTP Signature and Mastodon
+        'Digest': f'SHA-256={b64encode(sha256(data or b"").digest()).decode()}',
+    })
+
+    domain = user.key.id()
+    logger.info(f"Signing with {domain}'s key")
+    key_id = request.host_url + domain
+    auth = HTTPSignatureAuth(secret=user.private_pem(), key_id=key_id,
+                             algorithm='rsa-sha256', sign_header='signature',
+                             headers=('Date', 'Host', 'Digest'))
+
+    # make HTTP request
     kwargs.setdefault('gateway', True)
-    resp = fn(url, **kwargs)
+    resp = fn(url, auth=auth, headers=headers, **kwargs)
 
     logger.info(f'Got {resp.status_code} headers: {resp.headers}')
     type = content_type(resp)
@@ -92,26 +148,29 @@ def _requests_fn(fn, url, parse_json=False, **kwargs):
         (type.startswith('text/') or type.endswith('+json') or type.endswith('/json'))):
         logger.info(resp.text)
 
-    if parse_json:
-        try:
-            return resp.json()
-        except ValueError:
-            msg = "Couldn't parse response as JSON"
-            logger.info(msg, exc_info=True)
-            raise BadGateway(msg)
-
     return resp
 
 
-def get_as2(url):
+def get_as2(url, user=None):
     """Tries to fetch the given URL as ActivityStreams 2.
 
     Uses HTTP content negotiation via the Content-Type header. If the url is
     HTML and it has a rel-alternate link with an AS2 content type, fetches and
     returns that URL.
 
+    Includes an HTTP Signature with the request.
+    https://w3c.github.io/activitypub/#authorization
+    https://tools.ietf.org/html/draft-cavage-http-signatures-07
+    https://github.com/mastodon/mastodon/pull/11269
+
+    Mastodon requires this signature if AUTHORIZED_FETCH aka secure mode is on:
+    https://docs.joinmastodon.org/admin/config/#authorized_fetch
+
+    If user is not provided, defaults to using @snarfed.org@snarfed.org's key.
+
     Args:
         url: string
+        user: :class:`User` used to sign request
 
     Returns:
         :class:`requests.Response`
@@ -129,7 +188,7 @@ def get_as2(url):
         err.requests_response = resp
         raise err
 
-    resp = requests_get(url, headers=CONNEG_HEADERS_AS2_HTML)
+    resp = signed_get(url, user=user, headers=CONNEG_HEADERS_AS2_HTML)
     if content_type(resp) in (CONTENT_TYPE_AS2, CONTENT_TYPE_AS2_LD):
         return resp
 
@@ -139,8 +198,8 @@ def get_as2(url):
     if not (as2 and as2['href']):
         _error(resp)
 
-    resp = requests_get(urllib.parse.urljoin(resp.url, as2['href']),
+    resp = signed_get(urllib.parse.urljoin(resp.url, as2['href']),
-                        headers=CONNEG_HEADERS_AS2)
+                      headers=CONNEG_HEADERS_AS2)
     if content_type(resp) in (CONTENT_TYPE_AS2, CONTENT_TYPE_AS2_LD):
         return resp
 

models.py

@@ -11,6 +11,7 @@ from flask import request
 from google.cloud import ndb
 from granary import as2, microformats2
 from oauth_dropins.webutil.models import StringIdModel
+from oauth_dropins.webutil import util
 from oauth_dropins.webutil.util import json_dumps, json_loads
 
 import common
@@ -94,8 +95,8 @@ class User(StringIdModel):
         # check webfinger redirect
         path = f'/.well-known/webfinger?resource=acct:{domain}@{domain}'
         try:
-            resp = common.requests_get(urllib.parse.urljoin(site, path),
+            resp = util.requests_get(urllib.parse.urljoin(site, path),
-                                       allow_redirects=False, gateway=False)
+                                     allow_redirects=False, gateway=False)
             domain_urls = ([f'https://{domain}/' for domain in common.DOMAINS] +
                            [request.host_url])
             expected = [urllib.parse.urljoin(url, path) for url in domain_urls]

tests/test_activitypub.py

@@ -364,9 +364,10 @@ class ActivityPubTest(testutil.TestCase):
         got = self.client.post('/foo.com/inbox', json=LIKE)
         self.assertEqual(200, got.status_code)
 
-        self.assert_req(mock_get, 'http://orig/actor',
+        mock_get.assert_has_calls((
-                        headers=common.CONNEG_HEADERS_AS2_HTML)
+            self.as2_req('http://orig/actor'),
-        self.assert_req(mock_get, 'http://orig/post')
+            self.req('http://orig/post'),
+        )),
 
         args, kwargs = mock_post.call_args
         self.assertEqual(('http://orig/webmention',), args)
@@ -398,8 +399,9 @@ class ActivityPubTest(testutil.TestCase):
         got = self.client.post('/foo.com/inbox', json=FOLLOW_WRAPPED)
         self.assertEqual(200, got.status_code)
 
-        self.assert_req(mock_get, FOLLOW['actor'],
+        mock_get.assert_has_calls((
-                        headers=common.CONNEG_HEADERS_AS2_HTML)
+            self.as2_req(FOLLOW['actor']),
+        ))
 
         # check AP Accept
         self.assertEqual(2, len(mock_post.call_args_list))
@@ -489,7 +491,6 @@ class ActivityPubTest(testutil.TestCase):
         self.assertEqual('inactive', followee.key.get().status)
         self.assertEqual('active', other.key.get().status)
 
-
     def test_inbox_webmention_discovery_connection_fails(self, mock_head,
                                                          mock_get, mock_post):
         mock_get.side_effect = [

tests/test_common.py

@@ -31,12 +31,21 @@ NOT_ACCEPTABLE = requests_response(status=406)
 
 
 class CommonTest(testutil.TestCase):
+    def setUp(self):
+        super().setUp()
+        self.app_context = app.test_request_context('/')
+        self.app_context.push()
+
+    def tearDown(self):
+        self.app_context.pop()
+        super().tearDown()
+
     @mock.patch('requests.get', return_value=AS2)
     def test_get_as2_direct(self, mock_get):
         resp = common.get_as2('http://orig')
         self.assertEqual(AS2, resp)
         mock_get.assert_has_calls((
-            self.req('http://orig', headers=common.CONNEG_HEADERS_AS2_HTML),
+            self.as2_req('http://orig'),
         ))
 
     @mock.patch('requests.get', side_effect=[HTML_WITH_AS2, AS2])
@@ -44,8 +53,8 @@ class CommonTest(testutil.TestCase):
         resp = common.get_as2('http://orig')
         self.assertEqual(AS2, resp)
         mock_get.assert_has_calls((
-            self.req('http://orig', headers=common.CONNEG_HEADERS_AS2_HTML),
+            self.as2_req('http://orig'),
-            self.req('http://as2', headers=common.CONNEG_HEADERS_AS2),
+            self.as2_req('http://as2', headers=common.CONNEG_HEADERS_AS2),
         ))
 
     @mock.patch('requests.get', return_value=HTML)

tests/test_webmention.py

@@ -24,6 +24,7 @@ from common import (
     CONTENT_TYPE_ATOM,
     CONTENT_TYPE_HTML,
     CONTENT_TYPE_MAGIC_ENVELOPE,
+    default_signature_user,
 )
 from models import Follower, User, Activity
 import webmention
@@ -68,7 +69,7 @@ REPOST_AS2 = {
 class WebmentionTest(testutil.TestCase):
     def setUp(self):
         super().setUp()
-        self.key = User.get_or_create('a')
+        self.user = User.get_or_create('a')
 
         self.orig_html_as2 = requests_response("""\
 <html>
@@ -309,7 +310,7 @@ class WebmentionTest(testutil.TestCase):
                          kwargs['headers']['Content-Type'])
 
         env = utils.parse_magic_envelope(kwargs['data'])
-        assert magicsigs.verify(env['data'], env['sig'].encode(), key=self.key)
+        assert magicsigs.verify(env['data'], env['sig'].encode(), key=self.user)
 
         return env['data']
 
@@ -426,9 +427,9 @@ class WebmentionTest(testutil.TestCase):
 
         mock_get.assert_has_calls((
             self.req('http://a/reply'),
-            self.req('http://not/fediverse', headers=CONNEG_HEADERS_AS2_HTML),
+            self.as2_req('http://not/fediverse'),
-            self.req('http://orig/post', headers=CONNEG_HEADERS_AS2_HTML),
+            self.as2_req('http://orig/post'),
-            self.req('http://orig/author', headers=CONNEG_HEADERS_AS2_HTML),
+            self.as2_req('http://orig/author'),
         ))
 
         args, kwargs = mock_post.call_args
@@ -439,7 +440,7 @@ class WebmentionTest(testutil.TestCase):
         self.assertEqual(CONTENT_TYPE_AS2, headers['Content-Type'])
 
         rsa_key = kwargs['auth'].header_signer._rsa._key
-        self.assertEqual(self.key.private_pem(), rsa_key.exportKey())
+        self.assertEqual(self.user.private_pem(), rsa_key.exportKey())
 
         activity = Activity.get_by_id('http://a/reply http://orig/as2')
         self.assertEqual(['a'], activity.domain)
@@ -526,9 +527,9 @@ class WebmentionTest(testutil.TestCase):
 
         mock_get.assert_has_calls((
             self.req('http://a/reply'),
-            self.req('http://not/fediverse', headers=CONNEG_HEADERS_AS2_HTML),
+            self.as2_req('http://not/fediverse'),
-            self.req('http://orig/post', headers=CONNEG_HEADERS_AS2_HTML),
+            self.as2_req('http://orig/post'),
-            self.req('http://orig/author', headers=CONNEG_HEADERS_AS2_HTML),
+            self.as2_req('http://orig/author'),
         ))
 
         args, kwargs = mock_post.call_args
@@ -547,8 +548,8 @@ class WebmentionTest(testutil.TestCase):
 
         mock_get.assert_has_calls((
             self.req('http://a/repost'),
-            self.req('http://orig/post', headers=CONNEG_HEADERS_AS2_HTML),
+            self.as2_req('http://orig/post'),
-            self.req('http://orig/author', headers=CONNEG_HEADERS_AS2_HTML),
+            self.as2_req('http://orig/author'),
         ))
 
         args, kwargs = mock_post.call_args
@@ -559,7 +560,13 @@ class WebmentionTest(testutil.TestCase):
         self.assertEqual(CONTENT_TYPE_AS2, headers['Content-Type'])
 
         rsa_key = kwargs['auth'].header_signer._rsa._key
-        self.assertEqual(self.key.private_pem(), rsa_key.exportKey())
+        self.assertEqual(self.user.private_pem(), rsa_key.exportKey())
+
+        for args, kwargs in mock_get.call_args_list[1:]:
+            with self.subTest(url=args[0]):
+                rsa_key = kwargs['auth'].header_signer._rsa._key
+                self.assertEqual(default_signature_user().private_pem(),
+                                 rsa_key.exportKey())
 
         activity = Activity.get_by_id('http://a/repost http://orig/as2')
         self.assertEqual(['a'], activity.domain)
@@ -581,10 +588,10 @@ class WebmentionTest(testutil.TestCase):
 
         mock_get.assert_has_calls((
             self.req('http://a/reply'),
-            self.req('http://not/fediverse', headers=CONNEG_HEADERS_AS2_HTML),
+            self.as2_req('http://not/fediverse'),
-            self.req('http://orig/post', headers=CONNEG_HEADERS_AS2_HTML),
+            self.as2_req('http://orig/post'),
-            self.req('http://orig/as2', headers=CONNEG_HEADERS_AS2),
+            self.req('http://orig/as2', auth=mock.ANY, headers=CONNEG_HEADERS_AS2),
-            self.req('http://orig/author', headers=CONNEG_HEADERS_AS2_HTML),
+            self.as2_req('http://orig/author'),
         ))
 
         args, kwargs = mock_post.call_args
@@ -773,7 +780,7 @@ class WebmentionTest(testutil.TestCase):
 
         mock_get.assert_has_calls((
             self.req('http://a/follow'),
-            self.req('http://followee/', headers=CONNEG_HEADERS_AS2_HTML),
+            self.as2_req('http://followee/'),
         ))
 
         args, kwargs = mock_post.call_args
@@ -784,7 +791,7 @@ class WebmentionTest(testutil.TestCase):
         self.assertEqual(CONTENT_TYPE_AS2, headers['Content-Type'])
 
         rsa_key = kwargs['auth'].header_signer._rsa._key
-        self.assertEqual(self.key.private_pem(), rsa_key.exportKey())
+        self.assertEqual(self.user.private_pem(), rsa_key.exportKey())
 
         activity = Activity.get_by_id('http://a/follow http://followee/')
         self.assertEqual(['a'], activity.domain)
@@ -811,7 +818,7 @@ class WebmentionTest(testutil.TestCase):
 
         mock_get.assert_has_calls((
             self.req('http://a/follow#2'),
-            self.req('http://followee/', headers=CONNEG_HEADERS_AS2_HTML),
+            self.as2_req('http://followee/'),
         ))
 
         args, kwargs = mock_post.call_args
@@ -822,7 +829,7 @@ class WebmentionTest(testutil.TestCase):
         self.assert_equals(CONTENT_TYPE_AS2, headers['Content-Type'])
 
         rsa_key = kwargs['auth'].header_signer._rsa._key
-        self.assert_equals(self.key.private_pem(), rsa_key.exportKey())
+        self.assert_equals(self.user.private_pem(), rsa_key.exportKey())
 
         activity = Activity.get_by_id('http://a/follow__2 http://followee/')
         self.assert_equals(['a'], activity.domain)
@@ -863,7 +870,7 @@ class WebmentionTest(testutil.TestCase):
 
         mock_get.assert_has_calls((
             self.req('http://a/follow'),
-            self.req('http://followee/', headers=CONNEG_HEADERS_AS2_HTML),
+            self.as2_req('http://followee/'),
         ))
 
         args, kwargs = mock_post.call_args
@@ -874,7 +881,7 @@ class WebmentionTest(testutil.TestCase):
         self.assertEqual(CONTENT_TYPE_AS2, headers['Content-Type'])
 
         rsa_key = kwargs['auth'].header_signer._rsa._key
-        self.assertEqual(self.key.private_pem(), rsa_key.exportKey())
+        self.assertEqual(self.user.private_pem(), rsa_key.exportKey())
 
         activity = Activity.get_by_id('http://a/follow http://followee/')
         self.assertEqual(['a'], activity.domain)
@@ -907,8 +914,8 @@ class WebmentionTest(testutil.TestCase):
 
         mock_get.assert_has_calls((
             self.req('http://a/reply'),
-            self.req('http://not/fediverse', headers=CONNEG_HEADERS_AS2_HTML),
+            self.as2_req('http://not/fediverse'),
-            self.req('http://orig/post', headers=CONNEG_HEADERS_AS2_HTML),
+            self.as2_req('http://orig/post'),
             self.req('http://orig/atom'),
         ))
 
@@ -951,7 +958,7 @@ class WebmentionTest(testutil.TestCase):
 
         mock_get.assert_has_calls((
             self.req('http://a/like'),
-            self.req('http://orig/post', headers=CONNEG_HEADERS_AS2_HTML),
+            self.as2_req('http://orig/post'),
             self.req('http://orig/atom'),
         ))
 

tests/testutil.py

@@ -1,6 +1,7 @@
 """Common test utility code.
 """
 import copy
+import datetime
 import unittest
 from unittest.mock import ANY, call
 
@@ -11,6 +12,8 @@ import requests
 
 import common
 
+NOW = datetime.datetime(2022, 11, 23, 22, 29, 19)
+
 
 class TestCase(unittest.TestCase, testutil.Asserts):
     maxDiff = None
@@ -20,6 +23,7 @@ class TestCase(unittest.TestCase, testutil.Asserts):
         app.testing = True
         cache.clear()
         self.client = app.test_client()
+        common.utcnow = lambda: NOW
 
         # clear datastore
         requests.post('http://%s/reset' % ndb_client.host)
@@ -32,22 +36,25 @@ class TestCase(unittest.TestCase, testutil.Asserts):
 
     def req(self, url, **kwargs):
         """Returns a mock requests call."""
-        existing = kwargs.get('headers', {})
+        kwargs.setdefault('headers', {}).update({
-        if existing is not ANY:
+            'User-Agent': util.user_agent,
-            kwargs['headers'] = {
+        })
-              'User-Agent': util.user_agent,
-              **existing,
-            }
-
         kwargs.setdefault('timeout', util.HTTP_TIMEOUT)
         kwargs.setdefault('stream', True)
-
         return call(url, **kwargs)
 
+    def as2_req(self, url, **kwargs):
+        headers = {
+            'Date': 'Wed, 23 Nov 2022 22:29:19 GMT',
+            'Host': util.domain_from_link(url, minimize=False),
+            **common.CONNEG_HEADERS_AS2_HTML,
+            **kwargs.pop('headers', {}),
+        }
+        return self.req(url, auth=ANY, headers=headers, **kwargs)
+
     def assert_req(self, mock, url, **kwargs):
         """Checks a mock requests call."""
         kwargs.setdefault('headers', {}).setdefault('User-Agent', 'Bridgy Fed (https://fed.brid.gy/)')
         kwargs.setdefault('stream', True)
         kwargs.setdefault('timeout', util.HTTP_TIMEOUT)
-
         mock.assert_any_call(url, **kwargs)

webfinger.py

@@ -47,7 +47,7 @@ class Actor(flask_util.XrdOrJrd):
             urls = [url, urllib.parse.urljoin(url, '/')] + urls
 
         for candidate in urls:
-            resp = common.requests_get(candidate)
+            resp = util.requests_get(candidate)
             parsed = util.parse_html(resp)
             mf2 = util.parse_mf2(parsed, url=resp.url)
             # logger.debug(f'Parsed mf2 for {resp.url}: {json_dumps(mf2, indent=2)}')

webmention.py

@@ -25,7 +25,7 @@ from werkzeug.exceptions import BadGateway
 import activitypub
 from app import app
 import common
-from models import Follower, User, Activity
+from models import Activity, Follower, User
 
 logger = logging.getLogger(__name__)
 
@@ -39,13 +39,17 @@ class Webmention(View):
     source_mf2 = None     # parsed mf2 dict
     source_obj = None     # parsed AS1 dict
     target_resp = None    # requests.Response
+    user = None           # User
 
     def dispatch_request(self):
         logger.info(f'Params: {list(request.form.items())}')
 
         # fetch source page
         source = flask_util.get_required_param('source')
-        source_resp = common.requests_get(source)
+        try:
+            source_resp = util.requests_get(source)
+        except ValueError as e:
+            error(f'Bad source URL: {source}: {e}')
         self.source_url = source_resp.url or source
         self.source_domain = urllib.parse.urlparse(self.source_url).netloc.split(':')[0]
         fragment = urllib.parse.urlparse(self.source_url).fragment
@@ -78,6 +82,7 @@ class Webmention(View):
         self.source_obj = microformats2.json_to_object(entry, fetch_mf2=True)
         logger.info(f'Converted to AS1: {json_dumps(self.source_obj, indent=2)}')
 
+        self.user = User.get_or_create(self.source_domain)
         for method in self.try_activitypub, self.try_salmon:
             ret = method()
             if ret:
@@ -95,7 +100,6 @@ class Webmention(View):
         if not targets:
             return None
 
-        user = User.get_or_create(self.source_domain)
         error = None
         last_success = None
 
@@ -110,7 +114,7 @@ class Webmention(View):
         for resp, inbox in targets:
             target_obj = json_loads(resp.target_as2) if resp.target_as2 else None
             source_activity = common.postprocess_as2(
-                as2.from_as1(self.source_obj), target=target_obj, user=user)
+                as2.from_as1(self.source_obj), target=target_obj, user=self.user)
 
             if resp.status == 'complete':
                 if resp.source_mf2:
@@ -131,7 +135,7 @@ class Webmention(View):
                     source_activity['type'] = 'Update'
 
             try:
-                last = activitypub.send(source_activity, inbox, self.source_domain)
+                last = common.signed_post(inbox, data=source_activity, user=self.user)
                 resp.status = 'complete'
                 last_success = last
             except BaseException as e:
@@ -291,7 +295,7 @@ class Webmention(View):
         """
         # fetch target HTML page, extract Atom rel-alternate link
         if not self.target_resp:
-            self.target_resp = common.requests_get(target)
+            self.target_resp = util.requests_get(target)
 
         parsed = util.parse_html(self.target_resp)
         atom_url = parsed.find('link', rel='alternate', type=common.CONTENT_TYPE_ATOM)
@@ -307,7 +311,7 @@ class Webmention(View):
         atom_url = urllib.parse.urljoin(
             target, urllib.parse.urljoin(base_url, atom_link['href']))
 
-        feed = common.requests_get(atom_url).text
+        feed = util.requests_get(atom_url).text
         parsed = feedparser.parse(feed)
         entry = parsed.entries[0]
         logger.info(f'Parsed: {json_dumps(entry, indent=2)}')
@@ -344,11 +348,13 @@ class Webmention(View):
                 (author.get('name', ''), parsed.netloc))
             try:
                 # TODO: always https?
-                profile = common.requests_get(
+                profile = util.requests_get(
                     '%s://%s/.well-known/webfinger?resource=acct:%s' %
-                    (parsed.scheme, parsed.netloc, email), parse_json=True)
+                    (parsed.scheme, parsed.netloc, email))
-                endpoint = django_salmon.get_salmon_replies_link(profile)
+                endpoint = django_salmon.get_salmon_replies_link(profile.json())
-            except requests.HTTPError as e:
+            except ValueError:
+                logging.warning("Couldn't parse response as JSON")
+            except requests.HTTPError:
                 pass
 
         if not endpoint:
@@ -364,13 +370,11 @@ class Webmention(View):
 
         # sign reply and wrap in magic envelope
         domain = urllib.parse.urlparse(self.source_url).netloc
-        user = User.get_or_create(domain)
-        logger.info(f'Using key for {domain}: {user}')
         magic_envelope = magicsigs.magic_envelope(
-            entry, common.CONTENT_TYPE_ATOM, user).decode()
+            entry, common.CONTENT_TYPE_ATOM, self.user).decode()
 
         logger.info(f'Sending Salmon slap to {endpoint}')
-        common.requests_post(
+        util.requests_post(
             endpoint, data=common.XML_UTF8 + magic_envelope,
             headers={'Content-Type': common.CONTENT_TYPE_MAGIC_ENVELOPE})