unify request handler classes and handle_exception; move away from HOST[_URL]

fixes:
* https://console.cloud.google.com/errors/CN68rO-5sOK9cQ
* https://console.cloud.google.com/errors/CJWCu8b5_ureAg
* https://console.cloud.google.com/errors/CN6W4Zy7irzgOA
* https://console.cloud.google.com/errors/CN278MyjhZbtOQ

activitypub.py

@@ -11,7 +11,6 @@ from oauth_dropins.webutil.handlers import cache_response
 from oauth_dropins.webutil.util import json_dumps, json_loads
 import webapp2
 
-from appengine_config import HOST, HOST_URL
 import common
 from models import Follower, MagicKey
 from httpsig.requests_auth import HTTPSignatureAuth
@@ -66,7 +65,7 @@ def send(activity, inbox_url, user_domain):
     return common.requests_post(inbox_url, json=activity, auth=auth, headers=headers)
 
 
-class ActorHandler(webapp2.RequestHandler):
+class ActorHandler(common.Handler):
     """Serves /[DOMAIN], fetches its mf2, converts to AS Actor, and serves it."""
 
     @cache_response(CACHE_TIME)
@@ -78,17 +77,17 @@ class ActorHandler(webapp2.RequestHandler):
         hcard = mf2util.representative_hcard(mf2, mf2['url'])
         logging.info('Representative h-card: %s', json_dumps(hcard, indent=2))
         if not hcard:
-            common.error(self, """\
+            self.error("""\
 Couldn't find a representative h-card (http://microformats.org/wiki/representative-hcard-parsing) on %s""" % mf2['url'])
 
         key = MagicKey.get_or_create(domain)
-        obj = common.postprocess_as2(as2.from_as1(microformats2.json_to_object(hcard)),
+        obj = self.postprocess_as2(as2.from_as1(microformats2.json_to_object(hcard)),
-                                     key=key)
+                                   key=key)
         obj.update({
-            'inbox': '%s/%s/inbox' % (HOST_URL, domain),
+            'inbox': '%s/%s/inbox' % (self.request.host_url, domain),
-            'outbox': '%s/%s/outbox' % (HOST_URL, domain),
+            'outbox': '%s/%s/outbox' % (self.request.host_url, domain),
-            'following': '%s/%s/following' % (HOST_URL, domain),
+            'following': '%s/%s/following' % (self.request.host_url, domain),
-            'followers': '%s/%s/followers' % (HOST_URL, domain),
+            'followers': '%s/%s/followers' % (self.request.host_url, domain),
         })
         logging.info('Returning: %s', json_dumps(obj, indent=2))
 
@@ -99,9 +98,8 @@ Couldn't find a representative h-card (http://microformats.org/wiki/representati
         self.response.write(json_dumps(obj, indent=2))
 
 
-class InboxHandler(webapp2.RequestHandler):
+class InboxHandler(common.Handler):
     """Accepts POSTs to /[DOMAIN]/inbox and converts to outbound webmentions."""
-
     def post(self, domain):
         logging.info('Got: %s', self.request.body)
 
@@ -110,7 +108,7 @@ class InboxHandler(webapp2.RequestHandler):
             activity = json_loads(self.request.body)
             assert activity
         except (TypeError, ValueError, AssertionError):
-            common.error(self, "Couldn't parse body as JSON", exc_info=True)
+            self.error("Couldn't parse body as JSON", exc_info=True)
 
         obj = activity.get('object') or {}
         if isinstance(obj, str):
@@ -122,14 +120,14 @@ class InboxHandler(webapp2.RequestHandler):
         if type == 'Create':
             type = obj.get('type')
         elif type not in SUPPORTED_TYPES:
-            common.error(self, 'Sorry, %s activities are not supported yet.' % type,
+            self.error('Sorry, %s activities are not supported yet.' % type,
-                         status=501)
+                       status=501)
 
         # TODO: verify signature if there is one
 
         if type == 'Undo' and obj.get('type') == 'Follow':
             # skip actor fetch below; we don't need it to undo a follow
-            return self.undo_follow(common.redirect_unwrap(activity))
+            return self.undo_follow(self.redirect_unwrap(activity))
 
         # fetch actor if necessary so we have name, profile photo, etc
         for elem in obj, activity:
@@ -137,14 +135,14 @@ class InboxHandler(webapp2.RequestHandler):
             if actor and isinstance(actor, str):
                 elem['actor'] = common.get_as2(actor).json()
 
-        activity_unwrapped = common.redirect_unwrap(activity)
+        activity_unwrapped = self.redirect_unwrap(activity)
         if type == 'Follow':
             return self.accept_follow(activity, activity_unwrapped)
 
         # send webmentions to each target
         as1 = as2.to_as1(activity)
-        common.send_webmentions(self, as1, proxy=True, protocol='activitypub',
+        self.send_webmentions(as1, proxy=True, protocol='activitypub',
-                                source_as2=json_dumps(activity_unwrapped))
+                              source_as2=json_dumps(activity_unwrapped))
 
     def accept_follow(self, follow, follow_unwrapped):
         """Replies to an AP Follow request with an Accept request.
@@ -159,12 +157,12 @@ class InboxHandler(webapp2.RequestHandler):
         followee_unwrapped = follow_unwrapped.get('object')
         follower = follow.get('actor')
         if not followee or not followee_unwrapped or not follower:
-            common.error(self, 'Follow activity requires object and actor. Got: %s' % follow)
+            self.error('Follow activity requires object and actor. Got: %s' % follow)
 
         inbox = follower.get('inbox')
         follower_id = follower.get('id')
         if not inbox or not follower_id:
-            common.error(self, 'Follow actor requires id and inbox. Got: %s', follower)
+            self.error('Follow actor requires id and inbox. Got: %s', follower)
 
         # store Follower
         user_domain = util.domain_from_link(followee_unwrapped)
@@ -173,7 +171,7 @@ class InboxHandler(webapp2.RequestHandler):
         # send AP Accept
         accept = {
             '@context': 'https://www.w3.org/ns/activitystreams',
-            'id': util.tag_uri(HOST, 'accept/%s/%s' % (
+            'id': util.tag_uri(self.request.host, 'accept/%s/%s' % (
                 (user_domain, follow.get('id')))),
             'type': 'Accept',
             'actor': followee,
@@ -188,9 +186,8 @@ class InboxHandler(webapp2.RequestHandler):
         self.response.write(resp.text)
 
         # send webmention
-        common.send_webmentions(
+        self.send_webmentions(as2.to_as1(follow), proxy=True, protocol='activitypub',
-            self, as2.to_as1(follow), proxy=True, protocol='activitypub',
+                              source_as2=json_dumps(follow_unwrapped))
-            source_as2=json_dumps(follow_unwrapped))
 
     @ndb.transactional()
     def undo_follow(self, undo_unwrapped):
@@ -205,7 +202,7 @@ class InboxHandler(webapp2.RequestHandler):
         follower = follow.get('actor')
         followee = follow.get('object')
         if not follower or not followee:
-            common.error(self, 'Undo of Follow requires object with actor and object. Got: %s' % follow)
+            self.error('Undo of Follow requires object with actor and object. Got: %s' % follow)
 
         # deactivate Follower
         user_domain = util.domain_from_link(followee)

add_webmention.py

@@ -7,7 +7,6 @@ from oauth_dropins.webutil.handlers import cache_response
 import requests
 import webapp2
 
-from appengine_config import HOST, HOST_URL
 import common
 
 LINK_HEADER = '<%s>; rel="webmention"'
@@ -15,27 +14,27 @@ LINK_HEADER = '<%s>; rel="webmention"'
 CACHE_TIME = datetime.timedelta(seconds=15)
 
 
-class AddWebmentionHandler(webapp2.RequestHandler):
+class AddWebmentionHandler(common.Handler):
     """Proxies HTTP requests and adds Link header to our webmention endpoint."""
 
     @cache_response(CACHE_TIME)
     def get(self, url):
         url = urllib.parse.unquote(url)
         if not url.startswith('http://') and not url.startswith('https://'):
-            common.error(self, 'URL must start with http:// or https://')
+            self.error('URL must start with http:// or https://')
 
         try:
             resp = common.requests_get(url)
         except requests.exceptions.Timeout as e:
-            common.error(self, str(e), status=504, exc_info=True)
+            self.error(str(e), status=504, exc_info=True)
         except requests.exceptions.RequestException as e:
-            common.error(self, str(e), status=502, exc_info=True)
+            self.error(str(e), status=502, exc_info=True)
 
         self.response.status_int = resp.status_code
         self.response.write(resp.content)
 
         endpoint = LINK_HEADER % (str(self.request.get('endpoint')) or
-                                  HOST_URL + '/webmention')
+                                  self.request.host_url + '/webmention')
         self.response.headers.clear()
         self.response.headers.update(resp.headers)
         self.response.headers.add('Link', endpoint)

appengine_config.py

@@ -1,14 +1,6 @@
 """Bridgy App Engine config.
 """
-import os
+# suppress these INFO logs:
-
-from oauth_dropins.webutil.appengine_info import DEBUG, HOST, HOST_URL, SCHEME
-
-if not DEBUG:
-  HOST = 'fed.brid.gy'
-  HOST_URL = '%s://%s' % (SCHEME, HOST)
-
-# suppresses these INFO logs:
 # Sandbox prevented access to file "/usr/local/Caskroom/google-cloud-sdk"
 # If it is a static file, check that `application_readable: true` is set in your app.yaml
 import logging

common.py

@@ -12,7 +12,6 @@ import requests
 from webmentiontools import send
 from webob import exc
 
-from appengine_config import HOST, HOST_URL
 import common
 from models import Response
 
@@ -21,8 +20,6 @@ ACCT_RE = r'(?:acct:)?([^@]+)@' + DOMAIN_RE
 HEADERS = {
     'User-Agent': 'Bridgy Fed (https://fed.brid.gy/)',
 }
-# see redirect_wrap() and  redirect_unwrap()
-REDIRECT_PREFIX = urllib.parse.urljoin(HOST_URL, '/r/')
 XML_UTF8 = "<?xml version='1.0' encoding='UTF-8'?>\n"
 # USERNAME = 'me'
 # USERNAME_EMOJI = '🌎'  # globe
@@ -143,242 +140,255 @@ def content_type(resp):
         return type.split(';')[0]
 
 
-def error(handler, msg, status=None, exc_info=False):
+class Handler(handlers.ModernHandler):
-    if not status:
+    """Common request handler base class with lots of utilities."""
-        status = 400
-    logging.info('Returning %s: %s' % (status, msg), exc_info=exc_info)
-    handler.abort(status, msg)
 
+    def error(self, msg, status=None, exc_info=False):
+        if not status:
+            status = 400
+        logging.info('Returning %s: %s' % (status, msg), exc_info=exc_info)
+        self.abort(status, msg)
 
-def send_webmentions(handler, activity_wrapped, proxy=None, **response_props):
+    def send_webmentions(self, activity_wrapped, proxy=None, **response_props):
-    """Sends webmentions for an incoming Salmon slap or ActivityPub inbox delivery.
+        """Sends webmentions for an incoming Salmon slap or ActivityPub inbox delivery.
-    Args:
+        Args:
-      handler: RequestHandler
+          activity_wrapped: dict, AS1 activity
-      activity_wrapped: dict, AS1 activity
+          response_props: passed through to the newly created Responses
-      response_props: passed through to the newly created Responses
+        """
-    """
+        activity = self.redirect_unwrap(activity_wrapped)
-    activity = common.redirect_unwrap(activity_wrapped)
 
-    verb = activity.get('verb')
+        verb = activity.get('verb')
-    if verb and verb not in SUPPORTED_VERBS:
+        if verb and verb not in SUPPORTED_VERBS:
-        error(handler, '%s activities are not supported yet.' % verb)
+            self.error('%s activities are not supported yet.' % verb)
 
-    # extract source and targets
+        # extract source and targets
-    source = activity.get('url') or activity.get('id')
+        source = activity.get('url') or activity.get('id')
-    obj = activity.get('object')
+        obj = activity.get('object')
-    obj_url = util.get_url(obj)
+        obj_url = util.get_url(obj)
 
-    targets = util.get_list(activity, 'inReplyTo')
+        logging.info('@@@ %s', activity_wrapped)
-    if isinstance(obj, dict):
+        targets = util.get_list(activity, 'inReplyTo')
-        if not source or verb in ('create', 'post', 'update'):
+        if isinstance(obj, dict):
-            source = obj_url or obj.get('id')
+            if not source or verb in ('create', 'post', 'update'):
-        targets.extend(util.get_list(obj, 'inReplyTo'))
+                source = obj_url or obj.get('id')
+            targets.extend(util.get_list(obj, 'inReplyTo'))
 
-    tags = util.get_list(activity_wrapped, 'tags')
+        tags = util.get_list(activity_wrapped, 'tags')
-    obj_wrapped = activity_wrapped.get('object')
+        obj_wrapped = activity_wrapped.get('object')
-    if isinstance(obj_wrapped, dict):
+        if isinstance(obj_wrapped, dict):
-        tags.extend(util.get_list(obj_wrapped, 'tags'))
+            tags.extend(util.get_list(obj_wrapped, 'tags'))
-    for tag in tags:
+        for tag in tags:
-        if tag.get('objectType') == 'mention':
+            if tag.get('objectType') == 'mention':
-            url = tag.get('url')
+                url = tag.get('url')
-            if url and url.startswith(HOST_URL):
+                logging.info('@@@ %s %s', url, self.request.host_url)
-                targets.append(redirect_unwrap(url))
+                if url and url.startswith(self.request.host_url):
+                    targets.append(self.redirect_unwrap(url))
 
-    if verb in ('follow', 'like', 'share'):
+        if verb in ('follow', 'like', 'share'):
-         targets.append(obj_url)
+             targets.append(obj_url)
 
-    targets = util.dedupe_urls(util.get_url(t) for t in targets)
+        targets = util.dedupe_urls(util.get_url(t) for t in targets)
-    if not source:
+        if not source:
-        error(handler, "Couldn't find original post URL")
+            self.error("Couldn't find original post URL")
-    if not targets:
+        if not targets:
-        error(handler, "Couldn't find any target URLs in inReplyTo, object, or mention tags")
+            self.error("Couldn't find any target URLs in inReplyTo, object, or mention tags")
 
-    # send webmentions and store Responses
+        # send webmentions and store Responses
-    errors = []
+        errors = []
-    for target in targets:
+        for target in targets:
-        if util.domain_from_link(target) == util.domain_from_link(source):
+            if util.domain_from_link(target) == util.domain_from_link(source):
-            logging.info('Skipping same-domain webmention from %s to %s',
+                logging.info('Skipping same-domain webmention from %s to %s',
-                         source, target)
+                             source, target)
-            continue
+                continue
 
-        response = Response(source=source, target=target, direction='in',
+            response = Response(source=source, target=target, direction='in',
-                            **response_props)
+                                **response_props)
-        response.put()
+            response.put()
-        wm_source = (response.proxy_url()
+            wm_source = (response.proxy_url(self)
-                     if verb in ('follow', 'like', 'share') or proxy
+                         if verb in ('follow', 'like', 'share') or proxy
-                     else source)
+                         else source)
-        logging.info('Sending webmention from %s to %s', wm_source, target)
+            logging.info('Sending webmention from %s to %s', wm_source, target)
 
-        wm = send.WebmentionSend(wm_source, target)
+            wm = send.WebmentionSend(wm_source, target)
-        if wm.send(headers=HEADERS):
+            if wm.send(headers=HEADERS):
-            logging.info('Success: %s', wm.response)
+                logging.info('Success: %s', wm.response)
-            response.status = 'complete'
+                response.status = 'complete'
-        else:
+            else:
-            logging.warning('Failed: %s', wm.error)
+                logging.warning('Failed: %s', wm.error)
-            errors.append(wm.error)
+                errors.append(wm.error)
-            response.status = 'error'
+                response.status = 'error'
-        response.put()
+            response.put()
 
-    if errors:
+        if errors:
-        msg = 'Errors:\n' + '\n'.join(util.json_dumps(e, indent=2) for e in errors)
+            msg = 'Errors:\n' + '\n'.join(util.json_dumps(e, indent=2) for e in errors)
-        error(handler, msg, status=errors[0].get('http_status'))
+            self.error(msg, status=errors[0].get('http_status'))
 
+    def postprocess_as2(self, activity, target=None, key=None):
+        """Prepare an AS2 object to be served or sent via ActivityPub.
 
-def postprocess_as2(activity, target=None, key=None):
+        Args:
-    """Prepare an AS2 object to be served or sent via ActivityPub.
+          activity: dict, AS2 object or activity
+          target: dict, AS2 object, optional. The target of activity's inReplyTo or
+            Like/Announce/etc object, if any.
+          key: MagicKey, optional. populated into publicKey field if provided.
+        """
+        type = activity.get('type')
 
-    Args:
+        # actor objects
-      activity: dict, AS2 object or activity
+        if type == 'Person':
-      target: dict, AS2 object, optional. The target of activity's inReplyTo or
+            self.postprocess_as2_actor(activity)
-        Like/Announce/etc object, if any.
+            if not activity.get('publicKey'):
-      key: MagicKey, optional. populated into publicKey field if provided.
+                # underspecified, inferred from this issue and Mastodon's implementation:
-    """
+                # https://github.com/w3c/activitypub/issues/203#issuecomment-297553229
-    type = activity.get('type')
+                # https://github.com/tootsuite/mastodon/blob/bc2c263504e584e154384ecc2d804aeb1afb1ba3/app/services/activitypub/process_account_service.rb#L77
+                activity['publicKey'] = {
+                    'id': activity.get('preferredUsername'),
+                    'publicKeyPem': key.public_pem().decode(),
+                }
+            return activity
 
-    # actor objects
+        for actor in (util.get_list(activity, 'attributedTo') +
-    if type == 'Person':
+                      util.get_list(activity, 'actor')):
-        postprocess_as2_actor(activity)
+            self.postprocess_as2_actor(actor)
-        if not activity.get('publicKey'):
+
-            # underspecified, inferred from this issue and Mastodon's implementation:
+        # inReplyTo: singly valued, prefer id over url
-            # https://github.com/w3c/activitypub/issues/203#issuecomment-297553229
+        target_id = target.get('id') if target else None
-            # https://github.com/tootsuite/mastodon/blob/bc2c263504e584e154384ecc2d804aeb1afb1ba3/app/services/activitypub/process_account_service.rb#L77
+        in_reply_to = activity.get('inReplyTo')
-            activity['publicKey'] = {
+        if in_reply_to:
-                'id': activity.get('preferredUsername'),
+            if target_id:
-                'publicKeyPem': key.public_pem().decode(),
+                activity['inReplyTo'] = target_id
+            elif isinstance(in_reply_to, list):
+                if len(in_reply_to) > 1:
+                    logging.warning(
+                        "AS2 doesn't support multiple inReplyTo URLs! "
+                        'Only using the first: %s' % in_reply_to[0])
+                activity['inReplyTo'] = in_reply_to[0]
+
+            # Mastodon evidently requires a Mention tag for replies to generate a
+            # notification to the original post's author. not required for likes,
+            # reposts, etc. details:
+            # https://github.com/snarfed/bridgy-fed/issues/34
+            if target:
+                for to in (util.get_list(target, 'attributedTo') +
+                           util.get_list(target, 'actor')):
+                    if isinstance(to, dict):
+                        to = to.get('url') or to.get('id')
+                    if to:
+                        activity.setdefault('tag', []).append({
+                            'type': 'Mention',
+                            'href': to,
+                        })
+
+        # activity objects (for Like, Announce, etc): prefer id over url
+        obj = activity.get('object')
+        if obj:
+            if isinstance(obj, dict) and not obj.get('id'):
+                obj['id'] = target_id or obj.get('url')
+            elif target_id and obj != target_id:
+                activity['object'] = target_id
+
+        # id is required for most things. default to url if it's not set.
+        if not activity.get('id'):
+            activity['id'] = activity.get('url')
+
+        # TODO: find a better way to check this, sometimes or always?
+        # removed for now since it fires on posts without u-id or u-url, eg
+        # https://chrisbeckstrom.com/2018/12/27/32551/
+        # assert activity.get('id') or (isinstance(obj, dict) and obj.get('id'))
+
+        activity['id'] = self.redirect_wrap(activity.get('id'))
+        activity['url'] = self.redirect_wrap(activity.get('url'))
+
+        # copy image(s) into attachment(s). may be Mastodon-specific.
+        # https://github.com/snarfed/bridgy-fed/issues/33#issuecomment-440965618
+        obj_or_activity = obj if isinstance(obj, dict) else activity
+        obj_or_activity.setdefault('attachment', []).extend(
+            obj_or_activity.get('image', []))
+
+        # cc public and target's author(s) and recipients
+        # https://www.w3.org/TR/activitystreams-vocabulary/#audienceTargeting
+        # https://w3c.github.io/activitypub/#delivery
+        if type in as2.TYPE_TO_VERB or type in ('Article', 'Note'):
+            recips = [AS2_PUBLIC_AUDIENCE]
+            if target:
+                recips += itertools.chain(*(util.get_list(target, field) for field in
+                                            ('actor', 'attributedTo', 'to', 'cc')))
+            activity['cc'] = util.dedupe_urls(util.get_url(recip) or recip.get('id')
+                                              for recip in recips)
+
+        # wrap articles and notes in a Create activity
+        if type in ('Article', 'Note'):
+            activity = {
+                '@context': as2.CONTEXT,
+                'type': 'Create',
+                'object': activity,
             }
-        return activity
 
-    for actor in (util.get_list(activity, 'attributedTo') +
+        return util.trim_nulls(activity)
-                  util.get_list(activity, 'actor')):
-        postprocess_as2_actor(actor)
 
-    # inReplyTo: singly valued, prefer id over url
+    def postprocess_as2_actor(self, actor):
-    target_id = target.get('id') if target else None
+        """Prepare an AS2 actor object to be served or sent via ActivityPub.
-    in_reply_to = activity.get('inReplyTo')
-    if in_reply_to:
-        if target_id:
-            activity['inReplyTo'] = target_id
-        elif isinstance(in_reply_to, list):
-            if len(in_reply_to) > 1:
-                logging.warning(
-                    "AS2 doesn't support multiple inReplyTo URLs! "
-                    'Only using the first: %s' % in_reply_to[0])
-            activity['inReplyTo'] = in_reply_to[0]
 
-        # Mastodon evidently requires a Mention tag for replies to generate a
+        Args:
-        # notification to the original post's author. not required for likes,
+          actor: dict, AS2 actor object
-        # reposts, etc. details:
+        """
-        # https://github.com/snarfed/bridgy-fed/issues/34
+        url = actor.get('url')
-        if target:
+        if url:
-            for to in (util.get_list(target, 'attributedTo') +
+            domain = urllib.parse.urlparse(url).netloc
-                       util.get_list(target, 'actor')):
+            actor.setdefault('preferredUsername', domain)
-                if isinstance(to, dict):
+            actor['id'] = '%s/%s' % (self.request.host_url, domain)
-                    to = to.get('url') or to.get('id')
+            actor['url'] = self.redirect_wrap(url)
-                if to:
-                    activity.setdefault('tag', []).append({
-                        'type': 'Mention',
-                        'href': to,
-                    })
 
-    # activity objects (for Like, Announce, etc): prefer id over url
+        # required by pixelfed. https://github.com/snarfed/bridgy-fed/issues/39
-    obj = activity.get('object')
+        actor.setdefault('summary', '')
-    if obj:
-        if isinstance(obj, dict) and not obj.get('id'):
-            obj['id'] = target_id or obj.get('url')
-        elif target_id and obj != target_id:
-            activity['object'] = target_id
 
-    # id is required for most things. default to url if it's not set.
+    def redirect_wrap(self, url):
-    if not activity.get('id'):
+        """Returns a URL on our domain that redirects to this URL.
-        activity['id'] = activity.get('url')
 
-    # TODO: find a better way to check this, sometimes or always?
+        ...to satisfy Mastodon's non-standard domain matching requirement. :(
-    # removed for now since it fires on posts without u-id or u-url, eg
-    # https://chrisbeckstrom.com/2018/12/27/32551/
-    # assert activity.get('id') or (isinstance(obj, dict) and obj.get('id'))
 
-    activity['id'] = redirect_wrap(activity.get('id'))
+        Args:
-    activity['url'] = redirect_wrap(activity.get('url'))
+          url: string
 
-    # copy image(s) into attachment(s). may be Mastodon-specific.
+        https://github.com/snarfed/bridgy-fed/issues/16#issuecomment-424799599
-    # https://github.com/snarfed/bridgy-fed/issues/33#issuecomment-440965618
+        https://github.com/tootsuite/mastodon/pull/6219#issuecomment-429142747
-    obj_or_activity = obj if isinstance(obj, dict) else activity
-    obj_or_activity.setdefault('attachment', []).extend(
-        obj_or_activity.get('image', []))
 
-    # cc public and target's author(s) and recipients
+        Returns: string, redirect url
-    # https://www.w3.org/TR/activitystreams-vocabulary/#audienceTargeting
+        """
-    # https://w3c.github.io/activitypub/#delivery
+        if not url:
-    if type in as2.TYPE_TO_VERB or type in ('Article', 'Note'):
+            return url
-        recips = [AS2_PUBLIC_AUDIENCE]
-        if target:
-            recips += itertools.chain(*(util.get_list(target, field) for field in
-                                        ('actor', 'attributedTo', 'to', 'cc')))
-        activity['cc'] = util.dedupe_urls(util.get_url(recip) or recip.get('id')
-                                          for recip in recips)
 
-    # wrap articles and notes in a Create activity
+        prefix = urllib.parse.urljoin(self.request.host_url, '/r/')
-    if type in ('Article', 'Note'):
+        if url.startswith(prefix):
-        activity = {
+            return url
-            '@context': as2.CONTEXT,
-            'type': 'Create',
-            'object': activity,
-        }
 
-    return util.trim_nulls(activity)
+        return prefix + url
 
+    def redirect_unwrap(self, val):
+        """Removes our redirect wrapping from a URL, if it's there.
 
-def postprocess_as2_actor(actor):
+        url may be a string, dict, or list. dicts and lists are unwrapped
-    """Prepare an AS2 actor object to be served or sent via ActivityPub.
+        recursively.
 
-    Args:
+        Strings that aren't wrapped URLs are left unchanged.
-      actor: dict, AS2 actor object
-    """
-    url = actor.get('url')
-    if url:
-        domain = urllib.parse.urlparse(url).netloc
-        actor.setdefault('preferredUsername', domain)
-        actor['id'] = '%s/%s' % (HOST_URL, domain)
-        actor['url'] = redirect_wrap(url)
 
-    # required by pixelfed. https://github.com/snarfed/bridgy-fed/issues/39
+        Args:
-    actor.setdefault('summary', '')
+          url: string
 
+        Returns: string, unwrapped url
+        """
+        if isinstance(val, dict):
+            return {k: self.redirect_unwrap(v) for k, v in val.items()}
 
-def redirect_wrap(url):
+        elif isinstance(val, list):
-    """Returns a URL on our domain that redirects to this URL.
+            return [self.redirect_unwrap(v) for v in val]
 
-    ...to satisfy Mastodon's non-standard domain matching requirement. :(
+        elif isinstance(val, str):
+            prefix = urllib.parse.urljoin(self.request.host_url, '/r/')
+            if val.startswith(prefix):
+                return val[len(prefix):]
+            elif val.startswith(self.request.host_url):
+                return util.follow_redirects(
+                    util.domain_from_link(urllib.parse.urlparse(val).path.strip('/'))).url
 
-    https://github.com/snarfed/bridgy-fed/issues/16#issuecomment-424799599
+        return val
-    https://github.com/tootsuite/mastodon/pull/6219#issuecomment-429142747
-    """
-    if not url:
-        return url
-    if url.startswith(REDIRECT_PREFIX):
-        return url
-    return REDIRECT_PREFIX + url
-
-
-def redirect_unwrap(val):
-    """Removes our redirect wrapping from a URL, if it's there.
-
-    url may be a string, dict, or list. dicts and lists are unwrapped
-    recursively.
-
-    Strings that aren't wrapped URLs are left unchanged.
-    """
-    if isinstance(val, dict):
-        return {k: redirect_unwrap(v) for k, v in val.items()}
-
-    elif isinstance(val, list):
-        return [redirect_unwrap(v) for v in val]
-
-    elif isinstance(val, str):
-        if val.startswith(REDIRECT_PREFIX):
-            return val[len(REDIRECT_PREFIX):]
-        elif val.startswith(HOST_URL):
-            return util.follow_redirects(
-                util.domain_from_link(urllib.parse.urlparse(val).path.strip('/'))).url
-
-    return val

models.py

@@ -10,8 +10,6 @@ from django_salmon import magicsigs
 from google.cloud import ndb
 from oauth_dropins.webutil.models import StringIdModel
 
-from appengine_config import HOST, HOST_URL
-
 
 class MagicKey(StringIdModel):
     """Stores a user's public/private key pair used for Magic Signatures.
@@ -105,11 +103,11 @@ class Response(StringIdModel):
     def target(self):
         return self.key.id().split()[1]
 
-    def proxy_url(self):
+    def proxy_url(self, handler):
         """Returns the Bridgy Fed proxy URL to render this response as HTML."""
         if self.source_mf2 or self.source_as2 or self.source_atom:
             source, target = self.key.id().split(' ')
-            return '%s/render?%s' % (HOST_URL, urllib.parse.urlencode({
+            return '%s/render?%s' % (handler.request.host_url, urllib.parse.urlencode({
                 'source': source,
                 'target': target,
             }))

redirect.py

@@ -24,7 +24,7 @@ import common
 CACHE_TIME = datetime.timedelta(seconds=15)
 
 
-class RedirectHandler(webapp2.RequestHandler):
+class RedirectHandler(common.Handler):
     """301 redirects to the embedded fully qualified URL.
 
     e.g. redirects /r/https://foo.com/bar?baz to https://foo.com/bar?baz
@@ -35,7 +35,7 @@ class RedirectHandler(webapp2.RequestHandler):
         assert self.request.path_qs.startswith('/r/')
         to = self.request.path_qs[3:]
         if not to.startswith('http://') and not to.startswith('https://'):
-            common.error(self, 'Expected fully qualified URL; got %s' % to)
+            self.error('Expected fully qualified URL; got %s' % to)
 
         # poor man's conneg, only handle single Accept values, not multiple with
         # priorities.
@@ -57,7 +57,7 @@ class RedirectHandler(webapp2.RequestHandler):
         entry = mf2util.find_first_entry(mf2, ['h-entry'])
         logging.info('Parsed mf2 for %s: %s', mf2['url'], json_dumps(entry, indent=2))
 
-        obj = common.postprocess_as2(as2.from_as1(microformats2.json_to_object(entry)))
+        obj = self.postprocess_as2(as2.from_as1(microformats2.json_to_object(entry)))
         logging.info('Returning: %s', json_dumps(obj, indent=2))
 
         self.response.headers.update({

render.py

@@ -3,17 +3,18 @@
 import datetime
 
 from granary import as2, atom, microformats2
-from oauth_dropins.webutil.handlers import cache_response, ModernHandler
+from oauth_dropins.webutil.handlers import cache_response
 from oauth_dropins.webutil import util
 from oauth_dropins.webutil.util import json_loads
 import webapp2
 
+import common
 from models import Response
 
 CACHE_TIME = datetime.timedelta(minutes=15)
 
 
-class RenderHandler(ModernHandler):
+class RenderHandler(common.Handler):
     """Fetches a stored Response and renders it as HTML."""
 
     @cache_response(CACHE_TIME)

salmon.py

@@ -28,9 +28,8 @@ SUPPORTED_VERBS = (
 )
 
 
-class SlapHandler(webapp2.RequestHandler):
+class SlapHandler(common.Handler):
     """Accepts POSTs to /[ACCT]/salmon and converts to outbound webmentions."""
-
     # TODO: unify with activitypub
     def post(self, username, domain):
         logging.info('Got: %s', self.request.body)
@@ -38,7 +37,7 @@ class SlapHandler(webapp2.RequestHandler):
         try:
             parsed = utils.parse_magic_envelope(self.request.body)
         except ParseError as e:
-            common.error(self, 'Could not parse POST body as XML', exc_info=True)
+            self.error('Could not parse POST body as XML', exc_info=True)
         data = parsed['data']
         logging.info('Decoded: %s', data)
 
@@ -46,23 +45,23 @@ class SlapHandler(webapp2.RequestHandler):
         try:
             activity = atom.atom_to_activity(data)
         except ParseError as e:
-            common.error(self, 'Could not parse envelope data as XML', exc_info=True)
+            self.error('Could not parse envelope data as XML', exc_info=True)
 
         verb = activity.get('verb')
         if verb and verb not in SUPPORTED_VERBS:
-            common.error(self, 'Sorry, %s activities are not supported yet.' % verb,
+            self.error('Sorry, %s activities are not supported yet.' % verb,
-                         status=501)
+                       status=501)
 
         # verify author and signature
         author = util.get_url(activity.get('actor'))
         if ':' not in author:
             author = 'acct:%s' % author
         elif not author.startswith('acct:'):
-            common.error(self, 'Author URI %s has unsupported scheme; expected acct:' % author)
+            self.error('Author URI %s has unsupported scheme; expected acct:' % author)
 
         logging.info('Fetching Salmon key for %s' % author)
         if not magicsigs.verify(author, data, parsed['sig']):
-            common.error(self, 'Could not verify magic signature.')
+            self.error('Could not verify magic signature.')
         logging.info('Verified magic signature.')
 
         # Verify that the timestamp is recent. Required by spec.
@@ -71,11 +70,11 @@ class SlapHandler(webapp2.RequestHandler):
         #
         # updated = utils.parse_updated_from_atom(data)
         # if not utils.verify_timestamp(updated):
-        #     common.error(self, 'Timestamp is more than 1h old.')
+        #     self.error('Timestamp is more than 1h old.')
 
         # send webmentions to each target
         activity = atom.atom_to_activity(data)
-        common.send_webmentions(self, activity, protocol='ostatus', source_atom=data)
+        self.send_webmentions(activity, protocol='ostatus', source_atom=data)
 
 
 ROUTES = [

superfeedr.py

@@ -8,8 +8,10 @@ import logging
 
 import webapp2
 
+import common
 
-class SuperfeedrHandler(webapp2.RequestHandler):
+
+class SuperfeedrHandler(common.Handler):
     """Superfeedr subscription callback handler.
 
     https://documentation.superfeedr.com/publishers.html#subscription-callback

tests/test_activitypub.py

@@ -28,8 +28,7 @@ REPLY_OBJECT = {
     'cc': ['https://www.w3.org/ns/activitystreams#Public'],
 }
 REPLY_OBJECT_WRAPPED = copy.deepcopy(REPLY_OBJECT)
-REPLY_OBJECT_WRAPPED['inReplyTo'] = common.redirect_wrap(
+REPLY_OBJECT_WRAPPED['inReplyTo'] = 'http://localhost:80/r/orig/post'
-    REPLY_OBJECT_WRAPPED['inReplyTo'])
 REPLY = {
     '@context': 'https://www.w3.org/ns/activitystreams',
     'type': 'Create',
@@ -75,7 +74,7 @@ LIKE = {
     'actor': 'http://orig/actor',
 }
 LIKE_WRAPPED = copy.deepcopy(LIKE)
-LIKE_WRAPPED['object'] = common.redirect_wrap(LIKE_WRAPPED['object'])
+LIKE_WRAPPED['object'] = 'http://localhost/r/http://orig/post'
 LIKE_WITH_ACTOR = copy.deepcopy(LIKE)
 LIKE_WITH_ACTOR['actor'] = {
     '@context': 'https://www.w3.org/ns/activitystreams',
@@ -114,7 +113,7 @@ FOLLOW_WRAPPED_WITH_ACTOR['actor'] = FOLLOW_WITH_ACTOR['actor']
 ACCEPT = {
     '@context': 'https://www.w3.org/ns/activitystreams',
     'type': 'Accept',
-    'id': 'tag:localhost:accept/realize.be/https://mastodon.social/6d1a',
+    'id': 'tag:localhost:80:accept/realize.be/https://mastodon.social/6d1a',
     'actor': 'http://localhost/realize.be',
     'object': {
         'type': 'Follow',
@@ -276,7 +275,7 @@ class ActivityPubTest(testutil.TestCase):
         self.assertEqual('in', resp.direction)
         self.assertEqual('activitypub', resp.protocol)
         self.assertEqual('complete', resp.status)
-        self.assertEqual(common.redirect_unwrap(as2), json_loads(resp.source_as2))
+        self.assertEqual(self.handler.redirect_unwrap(as2), json_loads(resp.source_as2))
 
     def test_inbox_like(self, mock_head, mock_get, mock_post):
         mock_head.return_value = requests_response(url='http://orig/post')
@@ -290,7 +289,7 @@ class ActivityPubTest(testutil.TestCase):
         mock_post.return_value = requests_response()
 
         got = application.get_response('/foo.com/inbox', method='POST',
-                               body=json_dumps(LIKE_WRAPPED).encode())
+                                       body=json_dumps(LIKE_WRAPPED).encode())
         self.assertEqual(200, got.status_int)
 
         as2_headers = copy.deepcopy(common.HEADERS)

tests/test_common.py

@@ -29,7 +29,6 @@ NOT_ACCEPTABLE = requests_response(status=406)
 
 
 class CommonTest(testutil.TestCase):
-
     @mock.patch('requests.get', return_value=AS2)
     def test_get_as2_direct(self, mock_get):
         resp = common.get_as2('http://orig')
@@ -63,14 +62,14 @@ class CommonTest(testutil.TestCase):
             resp = common.get_as2('http://orig')
 
     def test_redirect_wrap_empty(self):
-        self.assertIsNone(common.redirect_wrap(None))
+        self.assertIsNone(self.handler.redirect_wrap(None))
-        self.assertEqual('', common.redirect_wrap(''))
+        self.assertEqual('', self.handler.redirect_wrap(''))
 
     def test_postprocess_as2_multiple_in_reply_tos(self):
         self.assertEqual({
             'id': 'http://localhost/r/xyz',
             'inReplyTo': 'foo',
-        }, common.postprocess_as2({
+        }, self.handler.postprocess_as2({
             'id': 'xyz',
             'inReplyTo': ['foo', 'bar'],
         }))

tests/test_models.py

@@ -53,8 +53,8 @@ class ResponseTest(testutil.TestCase):
 
     def test_proxy_url(self):
         resp = Response.get_or_create('abc', 'xyz')
-        self.assertIsNone(resp.proxy_url())
+        self.assertIsNone(resp.proxy_url(self.handler))
 
         resp.source_as2 = 'as2'
         self.assertEqual('http://localhost/render?source=abc&target=xyz',
-                          resp.proxy_url())
+                          resp.proxy_url(self.handler))

tests/testutil.py

@@ -7,6 +7,7 @@ from unittest.mock import ANY, call
 from oauth_dropins.webutil import testutil, util
 from oauth_dropins.webutil.appengine_config import ndb_client
 import requests
+import webapp2
 
 import common
 
@@ -22,6 +23,10 @@ class TestCase(unittest.TestCase, testutil.Asserts):
         self.ndb_context = ndb_client.context()
         self.ndb_context.__enter__()
 
+        self.request = webapp2.Request.blank('/')
+        self.response = webapp2.Response()
+        self.handler = common.Handler(self.request, self.response)
+
     def tearDown(self):
         self.ndb_context.__exit__(None, None, None)
         super(TestCase, self).tearDown()

webfinger.py

@@ -20,7 +20,6 @@ from oauth_dropins.webutil import handlers, util
 from oauth_dropins.webutil.util import json_dumps
 import webapp2
 
-from appengine_config import HOST, HOST_URL
 import common
 import models
 
@@ -28,7 +27,7 @@ CACHE_TIME = datetime.timedelta(seconds=15)
 NON_TLDS = frozenset(('html', 'json', 'php', 'xml'))
 
 
-class UserHandler(handlers.XrdOrJrdHandler):
+class UserHandler(common.Handler, handlers.XrdOrJrdHandler):
     """Fetches a site's home page, converts its mf2 to WebFinger, and serves."""
     JRD_TEMPLATE = False
 
@@ -43,7 +42,7 @@ class UserHandler(handlers.XrdOrJrdHandler):
         assert domain
 
         if domain.split('.')[-1] in NON_TLDS:
-            common.error(self, "%s doesn't look like a domain" % domain, status=404)
+            self.error("%s doesn't look like a domain" % domain, status=404)
 
         # find representative h-card. try url, then url's home page, then domain
         urls = ['http://%s/' % domain]
@@ -60,7 +59,7 @@ class UserHandler(handlers.XrdOrJrdHandler):
                 logging.info('Representative h-card: %s', json_dumps(hcard, indent=2))
                 break
         else:
-            common.error(self, """\
+            self.error("""\
 Couldn't find a representative h-card (http://microformats.org/wiki/representative-hcard-parsing) on %s""" % resp.url)
 
         logging.info('Generating WebFinger data for %s', domain)
@@ -121,14 +120,14 @@ Couldn't find a representative h-card (http://microformats.org/wiki/representati
             {
                 'rel': 'self',
                 'type': common.CONTENT_TYPE_AS2,
-                # use HOST_URL instead of e.g. request.host_url because it
+                # WARNING: in python 2 sometimes request.host_url lost port,
-                # sometimes lost port, e.g. http://localhost:8080 would become
+                # http://localhost:8080 would become just http://localhost. no
-                # just http://localhost. no clue how or why.
+                # clue how or why. pay attention here if that happens again.
-                'href': '%s/%s' % (HOST_URL, domain),
+                'href': '%s/%s' % (self.request.host_url, domain),
             }, {
                 'rel': 'inbox',
                 'type': common.CONTENT_TYPE_AS2,
-                'href': '%s/%s/inbox' % (HOST_URL, domain),
+                'href': '%s/%s/inbox' % (self.request.host_url, domain),
             },
 
             # OStatus
@@ -144,7 +143,7 @@ Couldn't find a representative h-card (http://microformats.org/wiki/representati
                 'href': key.href(),
             }, {
                 'rel': 'salmon',
-                'href': '%s/%s/salmon' % (HOST_URL, domain),
+                'href': '%s/%s/salmon' % (self.request.host_url, domain),
             }]
         })
         logging.info('Returning WebFinger data: %s', json_dumps(data, indent=2))
@@ -152,7 +151,6 @@ Couldn't find a representative h-card (http://microformats.org/wiki/representati
 
 
 class WebfingerHandler(UserHandler):
-
     def is_jrd(self):
         return True
 

webmention.py

@@ -27,7 +27,7 @@ from models import Follower, MagicKey, Response
 SKIP_EMAIL_DOMAINS = frozenset(('localhost', 'snarfed.org'))
 
 
-class WebmentionHandler(webapp2.RequestHandler):
+class WebmentionHandler(common.Handler):
     """Handles inbound webmention, converts to ActivityPub or Salmon."""
     source_url = None     # string
     source_domain = None  # string
@@ -51,12 +51,12 @@ class WebmentionHandler(webapp2.RequestHandler):
         # source's intent to federate to mastodon)
         if (self.request.host_url not in source_resp.text and
             urllib.parse.quote(self.request.host_url, safe='') not in source_resp.text):
-            common.error(self, "Couldn't find link to %s" % self.request.host_url)
+            self.error("Couldn't find link to %s" % self.request.host_url)
 
         # convert source page to ActivityStreams
         entry = mf2util.find_first_entry(self.source_mf2, ['h-entry'])
         if not entry:
-            common.error(self, 'No microformats2 found on %s' % self.source_url)
+            self.error('No microformats2 found on %s' % self.source_url)
 
         logging.info('First entry: %s', json_dumps(entry, indent=2))
         # make sure it has url, since we use that for AS2 id, which is required
@@ -93,7 +93,7 @@ class WebmentionHandler(webapp2.RequestHandler):
 
         for resp, inbox in targets:
             target_obj = json_loads(resp.target_as2) if resp.target_as2 else None
-            source_activity = common.postprocess_as2(
+            source_activity = self.postprocess_as2(
                 as2.from_as1(self.source_obj), target=target_obj, key=key)
 
             if resp.status == 'complete':
@@ -187,9 +187,9 @@ class WebmentionHandler(webapp2.RequestHandler):
                 inbox_url = actor.get('inbox')
                 actor = actor.get('url') or actor.get('id')
             if not inbox_url and not actor:
-                common.error(self, 'Target object has no actor or attributedTo with URL or id.')
+                self.error('Target object has no actor or attributedTo with URL or id.')
             elif not isinstance(actor, str):
-                common.error(self, 'Target actor or attributedTo has unexpected url or id object: %r' % actor)
+                self.error('Target actor or attributedTo has unexpected url or id object: %r' % actor)
 
         if not inbox_url:
             # fetch actor as AS object
@@ -199,7 +199,7 @@ class WebmentionHandler(webapp2.RequestHandler):
         if not inbox_url:
             # TODO: probably need a way to save errors like this so that we can
             # return them if ostatus fails too.
-            # common.error(self, 'Target actor has no inbox')
+            # self.error('Target actor has no inbox')
             return []
 
         inbox_url = urllib.parse.urljoin(target_url, inbox_url)
@@ -239,8 +239,7 @@ class WebmentionHandler(webapp2.RequestHandler):
         parsed = util.parse_html(self.target_resp)
         atom_url = parsed.find('link', rel='alternate', type=common.CONTENT_TYPE_ATOM)
         if not atom_url or not atom_url.get('href'):
-            common.error(self, 'Target post %s has no Atom link' % resp.target(),
+            self.error('Target post %s has no Atom link' % resp.target(), status=400)
-                         status=400)
 
         # fetch Atom target post, extract and inject id into source object
         base_url = ''
@@ -294,7 +293,7 @@ class WebmentionHandler(webapp2.RequestHandler):
                 pass
 
         if not endpoint:
-            common.error(self, 'No salmon endpoint found!', status=400)
+            self.error('No salmon endpoint found!', status=400)
         logging.info('Discovered Salmon endpoint %s', endpoint)
 
         # construct reply Atom object