From df6b0b58ba11ed92be1c798aeff37ba090cda1b8 Mon Sep 17 00:00:00 2001 From: Ryan Barrett Date: Fri, 31 Jan 2020 07:38:58 -0800 Subject: [PATCH] unify request handler classes and handle_exception; move away from HOST[_URL] fixes: * https://console.cloud.google.com/errors/CN68rO-5sOK9cQ * https://console.cloud.google.com/errors/CJWCu8b5_ureAg * https://console.cloud.google.com/errors/CN6W4Zy7irzgOA * https://console.cloud.google.com/errors/CN278MyjhZbtOQ --- activitypub.py | 47 ++--- add_webmention.py | 11 +- appengine_config.py | 10 +- common.py | 422 +++++++++++++++++++------------------- models.py | 6 +- redirect.py | 6 +- render.py | 5 +- salmon.py | 19 +- superfeedr.py | 4 +- tests/test_activitypub.py | 11 +- tests/test_common.py | 7 +- tests/test_models.py | 4 +- tests/testutil.py | 5 + webfinger.py | 20 +- webmention.py | 19 +- 15 files changed, 297 insertions(+), 299 deletions(-) diff --git a/activitypub.py b/activitypub.py index 65469ec..771a5f6 100644 --- a/activitypub.py +++ b/activitypub.py @@ -11,7 +11,6 @@ from oauth_dropins.webutil.handlers import cache_response from oauth_dropins.webutil.util import json_dumps, json_loads import webapp2 -from appengine_config import HOST, HOST_URL import common from models import Follower, MagicKey from httpsig.requests_auth import HTTPSignatureAuth @@ -66,7 +65,7 @@ def send(activity, inbox_url, user_domain): return common.requests_post(inbox_url, json=activity, auth=auth, headers=headers) -class ActorHandler(webapp2.RequestHandler): +class ActorHandler(common.Handler): """Serves /[DOMAIN], fetches its mf2, converts to AS Actor, and serves it.""" @cache_response(CACHE_TIME) @@ -78,17 +77,17 @@ class ActorHandler(webapp2.RequestHandler): hcard = mf2util.representative_hcard(mf2, mf2['url']) logging.info('Representative h-card: %s', json_dumps(hcard, indent=2)) if not hcard: - common.error(self, """\ + self.error("""\ Couldn't find a representative h-card (http://microformats.org/wiki/representative-hcard-parsing) on %s""" % mf2['url']) key = MagicKey.get_or_create(domain) - obj = common.postprocess_as2(as2.from_as1(microformats2.json_to_object(hcard)), - key=key) + obj = self.postprocess_as2(as2.from_as1(microformats2.json_to_object(hcard)), + key=key) obj.update({ - 'inbox': '%s/%s/inbox' % (HOST_URL, domain), - 'outbox': '%s/%s/outbox' % (HOST_URL, domain), - 'following': '%s/%s/following' % (HOST_URL, domain), - 'followers': '%s/%s/followers' % (HOST_URL, domain), + 'inbox': '%s/%s/inbox' % (self.request.host_url, domain), + 'outbox': '%s/%s/outbox' % (self.request.host_url, domain), + 'following': '%s/%s/following' % (self.request.host_url, domain), + 'followers': '%s/%s/followers' % (self.request.host_url, domain), }) logging.info('Returning: %s', json_dumps(obj, indent=2)) @@ -99,9 +98,8 @@ Couldn't find a representative h-card (http://microformats.org/wiki/representati self.response.write(json_dumps(obj, indent=2)) -class InboxHandler(webapp2.RequestHandler): +class InboxHandler(common.Handler): """Accepts POSTs to /[DOMAIN]/inbox and converts to outbound webmentions.""" - def post(self, domain): logging.info('Got: %s', self.request.body) @@ -110,7 +108,7 @@ class InboxHandler(webapp2.RequestHandler): activity = json_loads(self.request.body) assert activity except (TypeError, ValueError, AssertionError): - common.error(self, "Couldn't parse body as JSON", exc_info=True) + self.error("Couldn't parse body as JSON", exc_info=True) obj = activity.get('object') or {} if isinstance(obj, str): @@ -122,14 +120,14 @@ class InboxHandler(webapp2.RequestHandler): if type == 'Create': type = obj.get('type') elif type not in SUPPORTED_TYPES: - common.error(self, 'Sorry, %s activities are not supported yet.' % type, - status=501) + self.error('Sorry, %s activities are not supported yet.' % type, + status=501) # TODO: verify signature if there is one if type == 'Undo' and obj.get('type') == 'Follow': # skip actor fetch below; we don't need it to undo a follow - return self.undo_follow(common.redirect_unwrap(activity)) + return self.undo_follow(self.redirect_unwrap(activity)) # fetch actor if necessary so we have name, profile photo, etc for elem in obj, activity: @@ -137,14 +135,14 @@ class InboxHandler(webapp2.RequestHandler): if actor and isinstance(actor, str): elem['actor'] = common.get_as2(actor).json() - activity_unwrapped = common.redirect_unwrap(activity) + activity_unwrapped = self.redirect_unwrap(activity) if type == 'Follow': return self.accept_follow(activity, activity_unwrapped) # send webmentions to each target as1 = as2.to_as1(activity) - common.send_webmentions(self, as1, proxy=True, protocol='activitypub', - source_as2=json_dumps(activity_unwrapped)) + self.send_webmentions(as1, proxy=True, protocol='activitypub', + source_as2=json_dumps(activity_unwrapped)) def accept_follow(self, follow, follow_unwrapped): """Replies to an AP Follow request with an Accept request. @@ -159,12 +157,12 @@ class InboxHandler(webapp2.RequestHandler): followee_unwrapped = follow_unwrapped.get('object') follower = follow.get('actor') if not followee or not followee_unwrapped or not follower: - common.error(self, 'Follow activity requires object and actor. Got: %s' % follow) + self.error('Follow activity requires object and actor. Got: %s' % follow) inbox = follower.get('inbox') follower_id = follower.get('id') if not inbox or not follower_id: - common.error(self, 'Follow actor requires id and inbox. Got: %s', follower) + self.error('Follow actor requires id and inbox. Got: %s', follower) # store Follower user_domain = util.domain_from_link(followee_unwrapped) @@ -173,7 +171,7 @@ class InboxHandler(webapp2.RequestHandler): # send AP Accept accept = { '@context': 'https://www.w3.org/ns/activitystreams', - 'id': util.tag_uri(HOST, 'accept/%s/%s' % ( + 'id': util.tag_uri(self.request.host, 'accept/%s/%s' % ( (user_domain, follow.get('id')))), 'type': 'Accept', 'actor': followee, @@ -188,9 +186,8 @@ class InboxHandler(webapp2.RequestHandler): self.response.write(resp.text) # send webmention - common.send_webmentions( - self, as2.to_as1(follow), proxy=True, protocol='activitypub', - source_as2=json_dumps(follow_unwrapped)) + self.send_webmentions(as2.to_as1(follow), proxy=True, protocol='activitypub', + source_as2=json_dumps(follow_unwrapped)) @ndb.transactional() def undo_follow(self, undo_unwrapped): @@ -205,7 +202,7 @@ class InboxHandler(webapp2.RequestHandler): follower = follow.get('actor') followee = follow.get('object') if not follower or not followee: - common.error(self, 'Undo of Follow requires object with actor and object. Got: %s' % follow) + self.error('Undo of Follow requires object with actor and object. Got: %s' % follow) # deactivate Follower user_domain = util.domain_from_link(followee) diff --git a/add_webmention.py b/add_webmention.py index d75a3b7..98f6fb5 100644 --- a/add_webmention.py +++ b/add_webmention.py @@ -7,7 +7,6 @@ from oauth_dropins.webutil.handlers import cache_response import requests import webapp2 -from appengine_config import HOST, HOST_URL import common LINK_HEADER = '<%s>; rel="webmention"' @@ -15,27 +14,27 @@ LINK_HEADER = '<%s>; rel="webmention"' CACHE_TIME = datetime.timedelta(seconds=15) -class AddWebmentionHandler(webapp2.RequestHandler): +class AddWebmentionHandler(common.Handler): """Proxies HTTP requests and adds Link header to our webmention endpoint.""" @cache_response(CACHE_TIME) def get(self, url): url = urllib.parse.unquote(url) if not url.startswith('http://') and not url.startswith('https://'): - common.error(self, 'URL must start with http:// or https://') + self.error('URL must start with http:// or https://') try: resp = common.requests_get(url) except requests.exceptions.Timeout as e: - common.error(self, str(e), status=504, exc_info=True) + self.error(str(e), status=504, exc_info=True) except requests.exceptions.RequestException as e: - common.error(self, str(e), status=502, exc_info=True) + self.error(str(e), status=502, exc_info=True) self.response.status_int = resp.status_code self.response.write(resp.content) endpoint = LINK_HEADER % (str(self.request.get('endpoint')) or - HOST_URL + '/webmention') + self.request.host_url + '/webmention') self.response.headers.clear() self.response.headers.update(resp.headers) self.response.headers.add('Link', endpoint) diff --git a/appengine_config.py b/appengine_config.py index 58cc3c1..5b96eae 100644 --- a/appengine_config.py +++ b/appengine_config.py @@ -1,14 +1,6 @@ """Bridgy App Engine config. """ -import os - -from oauth_dropins.webutil.appengine_info import DEBUG, HOST, HOST_URL, SCHEME - -if not DEBUG: - HOST = 'fed.brid.gy' - HOST_URL = '%s://%s' % (SCHEME, HOST) - -# suppresses these INFO logs: +# suppress these INFO logs: # Sandbox prevented access to file "/usr/local/Caskroom/google-cloud-sdk" # If it is a static file, check that `application_readable: true` is set in your app.yaml import logging diff --git a/common.py b/common.py index 4435140..b894ba7 100644 --- a/common.py +++ b/common.py @@ -12,7 +12,6 @@ import requests from webmentiontools import send from webob import exc -from appengine_config import HOST, HOST_URL import common from models import Response @@ -21,8 +20,6 @@ ACCT_RE = r'(?:acct:)?([^@]+)@' + DOMAIN_RE HEADERS = { 'User-Agent': 'Bridgy Fed (https://fed.brid.gy/)', } -# see redirect_wrap() and redirect_unwrap() -REDIRECT_PREFIX = urllib.parse.urljoin(HOST_URL, '/r/') XML_UTF8 = "\n" # USERNAME = 'me' # USERNAME_EMOJI = '🌎' # globe @@ -143,242 +140,255 @@ def content_type(resp): return type.split(';')[0] -def error(handler, msg, status=None, exc_info=False): - if not status: - status = 400 - logging.info('Returning %s: %s' % (status, msg), exc_info=exc_info) - handler.abort(status, msg) +class Handler(handlers.ModernHandler): + """Common request handler base class with lots of utilities.""" + def error(self, msg, status=None, exc_info=False): + if not status: + status = 400 + logging.info('Returning %s: %s' % (status, msg), exc_info=exc_info) + self.abort(status, msg) -def send_webmentions(handler, activity_wrapped, proxy=None, **response_props): - """Sends webmentions for an incoming Salmon slap or ActivityPub inbox delivery. - Args: - handler: RequestHandler - activity_wrapped: dict, AS1 activity - response_props: passed through to the newly created Responses - """ - activity = common.redirect_unwrap(activity_wrapped) + def send_webmentions(self, activity_wrapped, proxy=None, **response_props): + """Sends webmentions for an incoming Salmon slap or ActivityPub inbox delivery. + Args: + activity_wrapped: dict, AS1 activity + response_props: passed through to the newly created Responses + """ + activity = self.redirect_unwrap(activity_wrapped) - verb = activity.get('verb') - if verb and verb not in SUPPORTED_VERBS: - error(handler, '%s activities are not supported yet.' % verb) + verb = activity.get('verb') + if verb and verb not in SUPPORTED_VERBS: + self.error('%s activities are not supported yet.' % verb) - # extract source and targets - source = activity.get('url') or activity.get('id') - obj = activity.get('object') - obj_url = util.get_url(obj) + # extract source and targets + source = activity.get('url') or activity.get('id') + obj = activity.get('object') + obj_url = util.get_url(obj) - targets = util.get_list(activity, 'inReplyTo') - if isinstance(obj, dict): - if not source or verb in ('create', 'post', 'update'): - source = obj_url or obj.get('id') - targets.extend(util.get_list(obj, 'inReplyTo')) + logging.info('@@@ %s', activity_wrapped) + targets = util.get_list(activity, 'inReplyTo') + if isinstance(obj, dict): + if not source or verb in ('create', 'post', 'update'): + source = obj_url or obj.get('id') + targets.extend(util.get_list(obj, 'inReplyTo')) - tags = util.get_list(activity_wrapped, 'tags') - obj_wrapped = activity_wrapped.get('object') - if isinstance(obj_wrapped, dict): - tags.extend(util.get_list(obj_wrapped, 'tags')) - for tag in tags: - if tag.get('objectType') == 'mention': - url = tag.get('url') - if url and url.startswith(HOST_URL): - targets.append(redirect_unwrap(url)) + tags = util.get_list(activity_wrapped, 'tags') + obj_wrapped = activity_wrapped.get('object') + if isinstance(obj_wrapped, dict): + tags.extend(util.get_list(obj_wrapped, 'tags')) + for tag in tags: + if tag.get('objectType') == 'mention': + url = tag.get('url') + logging.info('@@@ %s %s', url, self.request.host_url) + if url and url.startswith(self.request.host_url): + targets.append(self.redirect_unwrap(url)) - if verb in ('follow', 'like', 'share'): - targets.append(obj_url) + if verb in ('follow', 'like', 'share'): + targets.append(obj_url) - targets = util.dedupe_urls(util.get_url(t) for t in targets) - if not source: - error(handler, "Couldn't find original post URL") - if not targets: - error(handler, "Couldn't find any target URLs in inReplyTo, object, or mention tags") + targets = util.dedupe_urls(util.get_url(t) for t in targets) + if not source: + self.error("Couldn't find original post URL") + if not targets: + self.error("Couldn't find any target URLs in inReplyTo, object, or mention tags") - # send webmentions and store Responses - errors = [] - for target in targets: - if util.domain_from_link(target) == util.domain_from_link(source): - logging.info('Skipping same-domain webmention from %s to %s', - source, target) - continue + # send webmentions and store Responses + errors = [] + for target in targets: + if util.domain_from_link(target) == util.domain_from_link(source): + logging.info('Skipping same-domain webmention from %s to %s', + source, target) + continue - response = Response(source=source, target=target, direction='in', - **response_props) - response.put() - wm_source = (response.proxy_url() - if verb in ('follow', 'like', 'share') or proxy - else source) - logging.info('Sending webmention from %s to %s', wm_source, target) + response = Response(source=source, target=target, direction='in', + **response_props) + response.put() + wm_source = (response.proxy_url(self) + if verb in ('follow', 'like', 'share') or proxy + else source) + logging.info('Sending webmention from %s to %s', wm_source, target) - wm = send.WebmentionSend(wm_source, target) - if wm.send(headers=HEADERS): - logging.info('Success: %s', wm.response) - response.status = 'complete' - else: - logging.warning('Failed: %s', wm.error) - errors.append(wm.error) - response.status = 'error' - response.put() + wm = send.WebmentionSend(wm_source, target) + if wm.send(headers=HEADERS): + logging.info('Success: %s', wm.response) + response.status = 'complete' + else: + logging.warning('Failed: %s', wm.error) + errors.append(wm.error) + response.status = 'error' + response.put() - if errors: - msg = 'Errors:\n' + '\n'.join(util.json_dumps(e, indent=2) for e in errors) - error(handler, msg, status=errors[0].get('http_status')) + if errors: + msg = 'Errors:\n' + '\n'.join(util.json_dumps(e, indent=2) for e in errors) + self.error(msg, status=errors[0].get('http_status')) + def postprocess_as2(self, activity, target=None, key=None): + """Prepare an AS2 object to be served or sent via ActivityPub. -def postprocess_as2(activity, target=None, key=None): - """Prepare an AS2 object to be served or sent via ActivityPub. + Args: + activity: dict, AS2 object or activity + target: dict, AS2 object, optional. The target of activity's inReplyTo or + Like/Announce/etc object, if any. + key: MagicKey, optional. populated into publicKey field if provided. + """ + type = activity.get('type') - Args: - activity: dict, AS2 object or activity - target: dict, AS2 object, optional. The target of activity's inReplyTo or - Like/Announce/etc object, if any. - key: MagicKey, optional. populated into publicKey field if provided. - """ - type = activity.get('type') + # actor objects + if type == 'Person': + self.postprocess_as2_actor(activity) + if not activity.get('publicKey'): + # underspecified, inferred from this issue and Mastodon's implementation: + # https://github.com/w3c/activitypub/issues/203#issuecomment-297553229 + # https://github.com/tootsuite/mastodon/blob/bc2c263504e584e154384ecc2d804aeb1afb1ba3/app/services/activitypub/process_account_service.rb#L77 + activity['publicKey'] = { + 'id': activity.get('preferredUsername'), + 'publicKeyPem': key.public_pem().decode(), + } + return activity - # actor objects - if type == 'Person': - postprocess_as2_actor(activity) - if not activity.get('publicKey'): - # underspecified, inferred from this issue and Mastodon's implementation: - # https://github.com/w3c/activitypub/issues/203#issuecomment-297553229 - # https://github.com/tootsuite/mastodon/blob/bc2c263504e584e154384ecc2d804aeb1afb1ba3/app/services/activitypub/process_account_service.rb#L77 - activity['publicKey'] = { - 'id': activity.get('preferredUsername'), - 'publicKeyPem': key.public_pem().decode(), + for actor in (util.get_list(activity, 'attributedTo') + + util.get_list(activity, 'actor')): + self.postprocess_as2_actor(actor) + + # inReplyTo: singly valued, prefer id over url + target_id = target.get('id') if target else None + in_reply_to = activity.get('inReplyTo') + if in_reply_to: + if target_id: + activity['inReplyTo'] = target_id + elif isinstance(in_reply_to, list): + if len(in_reply_to) > 1: + logging.warning( + "AS2 doesn't support multiple inReplyTo URLs! " + 'Only using the first: %s' % in_reply_to[0]) + activity['inReplyTo'] = in_reply_to[0] + + # Mastodon evidently requires a Mention tag for replies to generate a + # notification to the original post's author. not required for likes, + # reposts, etc. details: + # https://github.com/snarfed/bridgy-fed/issues/34 + if target: + for to in (util.get_list(target, 'attributedTo') + + util.get_list(target, 'actor')): + if isinstance(to, dict): + to = to.get('url') or to.get('id') + if to: + activity.setdefault('tag', []).append({ + 'type': 'Mention', + 'href': to, + }) + + # activity objects (for Like, Announce, etc): prefer id over url + obj = activity.get('object') + if obj: + if isinstance(obj, dict) and not obj.get('id'): + obj['id'] = target_id or obj.get('url') + elif target_id and obj != target_id: + activity['object'] = target_id + + # id is required for most things. default to url if it's not set. + if not activity.get('id'): + activity['id'] = activity.get('url') + + # TODO: find a better way to check this, sometimes or always? + # removed for now since it fires on posts without u-id or u-url, eg + # https://chrisbeckstrom.com/2018/12/27/32551/ + # assert activity.get('id') or (isinstance(obj, dict) and obj.get('id')) + + activity['id'] = self.redirect_wrap(activity.get('id')) + activity['url'] = self.redirect_wrap(activity.get('url')) + + # copy image(s) into attachment(s). may be Mastodon-specific. + # https://github.com/snarfed/bridgy-fed/issues/33#issuecomment-440965618 + obj_or_activity = obj if isinstance(obj, dict) else activity + obj_or_activity.setdefault('attachment', []).extend( + obj_or_activity.get('image', [])) + + # cc public and target's author(s) and recipients + # https://www.w3.org/TR/activitystreams-vocabulary/#audienceTargeting + # https://w3c.github.io/activitypub/#delivery + if type in as2.TYPE_TO_VERB or type in ('Article', 'Note'): + recips = [AS2_PUBLIC_AUDIENCE] + if target: + recips += itertools.chain(*(util.get_list(target, field) for field in + ('actor', 'attributedTo', 'to', 'cc'))) + activity['cc'] = util.dedupe_urls(util.get_url(recip) or recip.get('id') + for recip in recips) + + # wrap articles and notes in a Create activity + if type in ('Article', 'Note'): + activity = { + '@context': as2.CONTEXT, + 'type': 'Create', + 'object': activity, } - return activity - for actor in (util.get_list(activity, 'attributedTo') + - util.get_list(activity, 'actor')): - postprocess_as2_actor(actor) + return util.trim_nulls(activity) - # inReplyTo: singly valued, prefer id over url - target_id = target.get('id') if target else None - in_reply_to = activity.get('inReplyTo') - if in_reply_to: - if target_id: - activity['inReplyTo'] = target_id - elif isinstance(in_reply_to, list): - if len(in_reply_to) > 1: - logging.warning( - "AS2 doesn't support multiple inReplyTo URLs! " - 'Only using the first: %s' % in_reply_to[0]) - activity['inReplyTo'] = in_reply_to[0] + def postprocess_as2_actor(self, actor): + """Prepare an AS2 actor object to be served or sent via ActivityPub. - # Mastodon evidently requires a Mention tag for replies to generate a - # notification to the original post's author. not required for likes, - # reposts, etc. details: - # https://github.com/snarfed/bridgy-fed/issues/34 - if target: - for to in (util.get_list(target, 'attributedTo') + - util.get_list(target, 'actor')): - if isinstance(to, dict): - to = to.get('url') or to.get('id') - if to: - activity.setdefault('tag', []).append({ - 'type': 'Mention', - 'href': to, - }) + Args: + actor: dict, AS2 actor object + """ + url = actor.get('url') + if url: + domain = urllib.parse.urlparse(url).netloc + actor.setdefault('preferredUsername', domain) + actor['id'] = '%s/%s' % (self.request.host_url, domain) + actor['url'] = self.redirect_wrap(url) - # activity objects (for Like, Announce, etc): prefer id over url - obj = activity.get('object') - if obj: - if isinstance(obj, dict) and not obj.get('id'): - obj['id'] = target_id or obj.get('url') - elif target_id and obj != target_id: - activity['object'] = target_id + # required by pixelfed. https://github.com/snarfed/bridgy-fed/issues/39 + actor.setdefault('summary', '') - # id is required for most things. default to url if it's not set. - if not activity.get('id'): - activity['id'] = activity.get('url') + def redirect_wrap(self, url): + """Returns a URL on our domain that redirects to this URL. - # TODO: find a better way to check this, sometimes or always? - # removed for now since it fires on posts without u-id or u-url, eg - # https://chrisbeckstrom.com/2018/12/27/32551/ - # assert activity.get('id') or (isinstance(obj, dict) and obj.get('id')) + ...to satisfy Mastodon's non-standard domain matching requirement. :( - activity['id'] = redirect_wrap(activity.get('id')) - activity['url'] = redirect_wrap(activity.get('url')) + Args: + url: string - # copy image(s) into attachment(s). may be Mastodon-specific. - # https://github.com/snarfed/bridgy-fed/issues/33#issuecomment-440965618 - obj_or_activity = obj if isinstance(obj, dict) else activity - obj_or_activity.setdefault('attachment', []).extend( - obj_or_activity.get('image', [])) + https://github.com/snarfed/bridgy-fed/issues/16#issuecomment-424799599 + https://github.com/tootsuite/mastodon/pull/6219#issuecomment-429142747 - # cc public and target's author(s) and recipients - # https://www.w3.org/TR/activitystreams-vocabulary/#audienceTargeting - # https://w3c.github.io/activitypub/#delivery - if type in as2.TYPE_TO_VERB or type in ('Article', 'Note'): - recips = [AS2_PUBLIC_AUDIENCE] - if target: - recips += itertools.chain(*(util.get_list(target, field) for field in - ('actor', 'attributedTo', 'to', 'cc'))) - activity['cc'] = util.dedupe_urls(util.get_url(recip) or recip.get('id') - for recip in recips) + Returns: string, redirect url + """ + if not url: + return url - # wrap articles and notes in a Create activity - if type in ('Article', 'Note'): - activity = { - '@context': as2.CONTEXT, - 'type': 'Create', - 'object': activity, - } + prefix = urllib.parse.urljoin(self.request.host_url, '/r/') + if url.startswith(prefix): + return url - return util.trim_nulls(activity) + return prefix + url + def redirect_unwrap(self, val): + """Removes our redirect wrapping from a URL, if it's there. -def postprocess_as2_actor(actor): - """Prepare an AS2 actor object to be served or sent via ActivityPub. + url may be a string, dict, or list. dicts and lists are unwrapped + recursively. - Args: - actor: dict, AS2 actor object - """ - url = actor.get('url') - if url: - domain = urllib.parse.urlparse(url).netloc - actor.setdefault('preferredUsername', domain) - actor['id'] = '%s/%s' % (HOST_URL, domain) - actor['url'] = redirect_wrap(url) + Strings that aren't wrapped URLs are left unchanged. - # required by pixelfed. https://github.com/snarfed/bridgy-fed/issues/39 - actor.setdefault('summary', '') + Args: + url: string + Returns: string, unwrapped url + """ + if isinstance(val, dict): + return {k: self.redirect_unwrap(v) for k, v in val.items()} -def redirect_wrap(url): - """Returns a URL on our domain that redirects to this URL. + elif isinstance(val, list): + return [self.redirect_unwrap(v) for v in val] - ...to satisfy Mastodon's non-standard domain matching requirement. :( + elif isinstance(val, str): + prefix = urllib.parse.urljoin(self.request.host_url, '/r/') + if val.startswith(prefix): + return val[len(prefix):] + elif val.startswith(self.request.host_url): + return util.follow_redirects( + util.domain_from_link(urllib.parse.urlparse(val).path.strip('/'))).url - https://github.com/snarfed/bridgy-fed/issues/16#issuecomment-424799599 - https://github.com/tootsuite/mastodon/pull/6219#issuecomment-429142747 - """ - if not url: - return url - if url.startswith(REDIRECT_PREFIX): - return url - return REDIRECT_PREFIX + url - - -def redirect_unwrap(val): - """Removes our redirect wrapping from a URL, if it's there. - - url may be a string, dict, or list. dicts and lists are unwrapped - recursively. - - Strings that aren't wrapped URLs are left unchanged. - """ - if isinstance(val, dict): - return {k: redirect_unwrap(v) for k, v in val.items()} - - elif isinstance(val, list): - return [redirect_unwrap(v) for v in val] - - elif isinstance(val, str): - if val.startswith(REDIRECT_PREFIX): - return val[len(REDIRECT_PREFIX):] - elif val.startswith(HOST_URL): - return util.follow_redirects( - util.domain_from_link(urllib.parse.urlparse(val).path.strip('/'))).url - - return val + return val diff --git a/models.py b/models.py index 38eddf7..fde5754 100644 --- a/models.py +++ b/models.py @@ -10,8 +10,6 @@ from django_salmon import magicsigs from google.cloud import ndb from oauth_dropins.webutil.models import StringIdModel -from appengine_config import HOST, HOST_URL - class MagicKey(StringIdModel): """Stores a user's public/private key pair used for Magic Signatures. @@ -105,11 +103,11 @@ class Response(StringIdModel): def target(self): return self.key.id().split()[1] - def proxy_url(self): + def proxy_url(self, handler): """Returns the Bridgy Fed proxy URL to render this response as HTML.""" if self.source_mf2 or self.source_as2 or self.source_atom: source, target = self.key.id().split(' ') - return '%s/render?%s' % (HOST_URL, urllib.parse.urlencode({ + return '%s/render?%s' % (handler.request.host_url, urllib.parse.urlencode({ 'source': source, 'target': target, })) diff --git a/redirect.py b/redirect.py index 25f9a63..952a361 100644 --- a/redirect.py +++ b/redirect.py @@ -24,7 +24,7 @@ import common CACHE_TIME = datetime.timedelta(seconds=15) -class RedirectHandler(webapp2.RequestHandler): +class RedirectHandler(common.Handler): """301 redirects to the embedded fully qualified URL. e.g. redirects /r/https://foo.com/bar?baz to https://foo.com/bar?baz @@ -35,7 +35,7 @@ class RedirectHandler(webapp2.RequestHandler): assert self.request.path_qs.startswith('/r/') to = self.request.path_qs[3:] if not to.startswith('http://') and not to.startswith('https://'): - common.error(self, 'Expected fully qualified URL; got %s' % to) + self.error('Expected fully qualified URL; got %s' % to) # poor man's conneg, only handle single Accept values, not multiple with # priorities. @@ -57,7 +57,7 @@ class RedirectHandler(webapp2.RequestHandler): entry = mf2util.find_first_entry(mf2, ['h-entry']) logging.info('Parsed mf2 for %s: %s', mf2['url'], json_dumps(entry, indent=2)) - obj = common.postprocess_as2(as2.from_as1(microformats2.json_to_object(entry))) + obj = self.postprocess_as2(as2.from_as1(microformats2.json_to_object(entry))) logging.info('Returning: %s', json_dumps(obj, indent=2)) self.response.headers.update({ diff --git a/render.py b/render.py index 8702605..13f991c 100644 --- a/render.py +++ b/render.py @@ -3,17 +3,18 @@ import datetime from granary import as2, atom, microformats2 -from oauth_dropins.webutil.handlers import cache_response, ModernHandler +from oauth_dropins.webutil.handlers import cache_response from oauth_dropins.webutil import util from oauth_dropins.webutil.util import json_loads import webapp2 +import common from models import Response CACHE_TIME = datetime.timedelta(minutes=15) -class RenderHandler(ModernHandler): +class RenderHandler(common.Handler): """Fetches a stored Response and renders it as HTML.""" @cache_response(CACHE_TIME) diff --git a/salmon.py b/salmon.py index 0a1357e..00e7774 100644 --- a/salmon.py +++ b/salmon.py @@ -28,9 +28,8 @@ SUPPORTED_VERBS = ( ) -class SlapHandler(webapp2.RequestHandler): +class SlapHandler(common.Handler): """Accepts POSTs to /[ACCT]/salmon and converts to outbound webmentions.""" - # TODO: unify with activitypub def post(self, username, domain): logging.info('Got: %s', self.request.body) @@ -38,7 +37,7 @@ class SlapHandler(webapp2.RequestHandler): try: parsed = utils.parse_magic_envelope(self.request.body) except ParseError as e: - common.error(self, 'Could not parse POST body as XML', exc_info=True) + self.error('Could not parse POST body as XML', exc_info=True) data = parsed['data'] logging.info('Decoded: %s', data) @@ -46,23 +45,23 @@ class SlapHandler(webapp2.RequestHandler): try: activity = atom.atom_to_activity(data) except ParseError as e: - common.error(self, 'Could not parse envelope data as XML', exc_info=True) + self.error('Could not parse envelope data as XML', exc_info=True) verb = activity.get('verb') if verb and verb not in SUPPORTED_VERBS: - common.error(self, 'Sorry, %s activities are not supported yet.' % verb, - status=501) + self.error('Sorry, %s activities are not supported yet.' % verb, + status=501) # verify author and signature author = util.get_url(activity.get('actor')) if ':' not in author: author = 'acct:%s' % author elif not author.startswith('acct:'): - common.error(self, 'Author URI %s has unsupported scheme; expected acct:' % author) + self.error('Author URI %s has unsupported scheme; expected acct:' % author) logging.info('Fetching Salmon key for %s' % author) if not magicsigs.verify(author, data, parsed['sig']): - common.error(self, 'Could not verify magic signature.') + self.error('Could not verify magic signature.') logging.info('Verified magic signature.') # Verify that the timestamp is recent. Required by spec. @@ -71,11 +70,11 @@ class SlapHandler(webapp2.RequestHandler): # # updated = utils.parse_updated_from_atom(data) # if not utils.verify_timestamp(updated): - # common.error(self, 'Timestamp is more than 1h old.') + # self.error('Timestamp is more than 1h old.') # send webmentions to each target activity = atom.atom_to_activity(data) - common.send_webmentions(self, activity, protocol='ostatus', source_atom=data) + self.send_webmentions(activity, protocol='ostatus', source_atom=data) ROUTES = [ diff --git a/superfeedr.py b/superfeedr.py index 4c10784..5ca544d 100644 --- a/superfeedr.py +++ b/superfeedr.py @@ -8,8 +8,10 @@ import logging import webapp2 +import common -class SuperfeedrHandler(webapp2.RequestHandler): + +class SuperfeedrHandler(common.Handler): """Superfeedr subscription callback handler. https://documentation.superfeedr.com/publishers.html#subscription-callback diff --git a/tests/test_activitypub.py b/tests/test_activitypub.py index 2496095..f5e0627 100644 --- a/tests/test_activitypub.py +++ b/tests/test_activitypub.py @@ -28,8 +28,7 @@ REPLY_OBJECT = { 'cc': ['https://www.w3.org/ns/activitystreams#Public'], } REPLY_OBJECT_WRAPPED = copy.deepcopy(REPLY_OBJECT) -REPLY_OBJECT_WRAPPED['inReplyTo'] = common.redirect_wrap( - REPLY_OBJECT_WRAPPED['inReplyTo']) +REPLY_OBJECT_WRAPPED['inReplyTo'] = 'http://localhost:80/r/orig/post' REPLY = { '@context': 'https://www.w3.org/ns/activitystreams', 'type': 'Create', @@ -75,7 +74,7 @@ LIKE = { 'actor': 'http://orig/actor', } LIKE_WRAPPED = copy.deepcopy(LIKE) -LIKE_WRAPPED['object'] = common.redirect_wrap(LIKE_WRAPPED['object']) +LIKE_WRAPPED['object'] = 'http://localhost/r/http://orig/post' LIKE_WITH_ACTOR = copy.deepcopy(LIKE) LIKE_WITH_ACTOR['actor'] = { '@context': 'https://www.w3.org/ns/activitystreams', @@ -114,7 +113,7 @@ FOLLOW_WRAPPED_WITH_ACTOR['actor'] = FOLLOW_WITH_ACTOR['actor'] ACCEPT = { '@context': 'https://www.w3.org/ns/activitystreams', 'type': 'Accept', - 'id': 'tag:localhost:accept/realize.be/https://mastodon.social/6d1a', + 'id': 'tag:localhost:80:accept/realize.be/https://mastodon.social/6d1a', 'actor': 'http://localhost/realize.be', 'object': { 'type': 'Follow', @@ -276,7 +275,7 @@ class ActivityPubTest(testutil.TestCase): self.assertEqual('in', resp.direction) self.assertEqual('activitypub', resp.protocol) self.assertEqual('complete', resp.status) - self.assertEqual(common.redirect_unwrap(as2), json_loads(resp.source_as2)) + self.assertEqual(self.handler.redirect_unwrap(as2), json_loads(resp.source_as2)) def test_inbox_like(self, mock_head, mock_get, mock_post): mock_head.return_value = requests_response(url='http://orig/post') @@ -290,7 +289,7 @@ class ActivityPubTest(testutil.TestCase): mock_post.return_value = requests_response() got = application.get_response('/foo.com/inbox', method='POST', - body=json_dumps(LIKE_WRAPPED).encode()) + body=json_dumps(LIKE_WRAPPED).encode()) self.assertEqual(200, got.status_int) as2_headers = copy.deepcopy(common.HEADERS) diff --git a/tests/test_common.py b/tests/test_common.py index 8319143..2a2089a 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -29,7 +29,6 @@ NOT_ACCEPTABLE = requests_response(status=406) class CommonTest(testutil.TestCase): - @mock.patch('requests.get', return_value=AS2) def test_get_as2_direct(self, mock_get): resp = common.get_as2('http://orig') @@ -63,14 +62,14 @@ class CommonTest(testutil.TestCase): resp = common.get_as2('http://orig') def test_redirect_wrap_empty(self): - self.assertIsNone(common.redirect_wrap(None)) - self.assertEqual('', common.redirect_wrap('')) + self.assertIsNone(self.handler.redirect_wrap(None)) + self.assertEqual('', self.handler.redirect_wrap('')) def test_postprocess_as2_multiple_in_reply_tos(self): self.assertEqual({ 'id': 'http://localhost/r/xyz', 'inReplyTo': 'foo', - }, common.postprocess_as2({ + }, self.handler.postprocess_as2({ 'id': 'xyz', 'inReplyTo': ['foo', 'bar'], })) diff --git a/tests/test_models.py b/tests/test_models.py index 30045f3..b977729 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -53,8 +53,8 @@ class ResponseTest(testutil.TestCase): def test_proxy_url(self): resp = Response.get_or_create('abc', 'xyz') - self.assertIsNone(resp.proxy_url()) + self.assertIsNone(resp.proxy_url(self.handler)) resp.source_as2 = 'as2' self.assertEqual('http://localhost/render?source=abc&target=xyz', - resp.proxy_url()) + resp.proxy_url(self.handler)) diff --git a/tests/testutil.py b/tests/testutil.py index 13c324b..a3831e9 100644 --- a/tests/testutil.py +++ b/tests/testutil.py @@ -7,6 +7,7 @@ from unittest.mock import ANY, call from oauth_dropins.webutil import testutil, util from oauth_dropins.webutil.appengine_config import ndb_client import requests +import webapp2 import common @@ -22,6 +23,10 @@ class TestCase(unittest.TestCase, testutil.Asserts): self.ndb_context = ndb_client.context() self.ndb_context.__enter__() + self.request = webapp2.Request.blank('/') + self.response = webapp2.Response() + self.handler = common.Handler(self.request, self.response) + def tearDown(self): self.ndb_context.__exit__(None, None, None) super(TestCase, self).tearDown() diff --git a/webfinger.py b/webfinger.py index 3ff03ef..e21de9e 100644 --- a/webfinger.py +++ b/webfinger.py @@ -20,7 +20,6 @@ from oauth_dropins.webutil import handlers, util from oauth_dropins.webutil.util import json_dumps import webapp2 -from appengine_config import HOST, HOST_URL import common import models @@ -28,7 +27,7 @@ CACHE_TIME = datetime.timedelta(seconds=15) NON_TLDS = frozenset(('html', 'json', 'php', 'xml')) -class UserHandler(handlers.XrdOrJrdHandler): +class UserHandler(common.Handler, handlers.XrdOrJrdHandler): """Fetches a site's home page, converts its mf2 to WebFinger, and serves.""" JRD_TEMPLATE = False @@ -43,7 +42,7 @@ class UserHandler(handlers.XrdOrJrdHandler): assert domain if domain.split('.')[-1] in NON_TLDS: - common.error(self, "%s doesn't look like a domain" % domain, status=404) + self.error("%s doesn't look like a domain" % domain, status=404) # find representative h-card. try url, then url's home page, then domain urls = ['http://%s/' % domain] @@ -60,7 +59,7 @@ class UserHandler(handlers.XrdOrJrdHandler): logging.info('Representative h-card: %s', json_dumps(hcard, indent=2)) break else: - common.error(self, """\ + self.error("""\ Couldn't find a representative h-card (http://microformats.org/wiki/representative-hcard-parsing) on %s""" % resp.url) logging.info('Generating WebFinger data for %s', domain) @@ -121,14 +120,14 @@ Couldn't find a representative h-card (http://microformats.org/wiki/representati { 'rel': 'self', 'type': common.CONTENT_TYPE_AS2, - # use HOST_URL instead of e.g. request.host_url because it - # sometimes lost port, e.g. http://localhost:8080 would become - # just http://localhost. no clue how or why. - 'href': '%s/%s' % (HOST_URL, domain), + # WARNING: in python 2 sometimes request.host_url lost port, + # http://localhost:8080 would become just http://localhost. no + # clue how or why. pay attention here if that happens again. + 'href': '%s/%s' % (self.request.host_url, domain), }, { 'rel': 'inbox', 'type': common.CONTENT_TYPE_AS2, - 'href': '%s/%s/inbox' % (HOST_URL, domain), + 'href': '%s/%s/inbox' % (self.request.host_url, domain), }, # OStatus @@ -144,7 +143,7 @@ Couldn't find a representative h-card (http://microformats.org/wiki/representati 'href': key.href(), }, { 'rel': 'salmon', - 'href': '%s/%s/salmon' % (HOST_URL, domain), + 'href': '%s/%s/salmon' % (self.request.host_url, domain), }] }) logging.info('Returning WebFinger data: %s', json_dumps(data, indent=2)) @@ -152,7 +151,6 @@ Couldn't find a representative h-card (http://microformats.org/wiki/representati class WebfingerHandler(UserHandler): - def is_jrd(self): return True diff --git a/webmention.py b/webmention.py index cba2015..ad9e24d 100644 --- a/webmention.py +++ b/webmention.py @@ -27,7 +27,7 @@ from models import Follower, MagicKey, Response SKIP_EMAIL_DOMAINS = frozenset(('localhost', 'snarfed.org')) -class WebmentionHandler(webapp2.RequestHandler): +class WebmentionHandler(common.Handler): """Handles inbound webmention, converts to ActivityPub or Salmon.""" source_url = None # string source_domain = None # string @@ -51,12 +51,12 @@ class WebmentionHandler(webapp2.RequestHandler): # source's intent to federate to mastodon) if (self.request.host_url not in source_resp.text and urllib.parse.quote(self.request.host_url, safe='') not in source_resp.text): - common.error(self, "Couldn't find link to %s" % self.request.host_url) + self.error("Couldn't find link to %s" % self.request.host_url) # convert source page to ActivityStreams entry = mf2util.find_first_entry(self.source_mf2, ['h-entry']) if not entry: - common.error(self, 'No microformats2 found on %s' % self.source_url) + self.error('No microformats2 found on %s' % self.source_url) logging.info('First entry: %s', json_dumps(entry, indent=2)) # make sure it has url, since we use that for AS2 id, which is required @@ -93,7 +93,7 @@ class WebmentionHandler(webapp2.RequestHandler): for resp, inbox in targets: target_obj = json_loads(resp.target_as2) if resp.target_as2 else None - source_activity = common.postprocess_as2( + source_activity = self.postprocess_as2( as2.from_as1(self.source_obj), target=target_obj, key=key) if resp.status == 'complete': @@ -187,9 +187,9 @@ class WebmentionHandler(webapp2.RequestHandler): inbox_url = actor.get('inbox') actor = actor.get('url') or actor.get('id') if not inbox_url and not actor: - common.error(self, 'Target object has no actor or attributedTo with URL or id.') + self.error('Target object has no actor or attributedTo with URL or id.') elif not isinstance(actor, str): - common.error(self, 'Target actor or attributedTo has unexpected url or id object: %r' % actor) + self.error('Target actor or attributedTo has unexpected url or id object: %r' % actor) if not inbox_url: # fetch actor as AS object @@ -199,7 +199,7 @@ class WebmentionHandler(webapp2.RequestHandler): if not inbox_url: # TODO: probably need a way to save errors like this so that we can # return them if ostatus fails too. - # common.error(self, 'Target actor has no inbox') + # self.error('Target actor has no inbox') return [] inbox_url = urllib.parse.urljoin(target_url, inbox_url) @@ -239,8 +239,7 @@ class WebmentionHandler(webapp2.RequestHandler): parsed = util.parse_html(self.target_resp) atom_url = parsed.find('link', rel='alternate', type=common.CONTENT_TYPE_ATOM) if not atom_url or not atom_url.get('href'): - common.error(self, 'Target post %s has no Atom link' % resp.target(), - status=400) + self.error('Target post %s has no Atom link' % resp.target(), status=400) # fetch Atom target post, extract and inject id into source object base_url = '' @@ -294,7 +293,7 @@ class WebmentionHandler(webapp2.RequestHandler): pass if not endpoint: - common.error(self, 'No salmon endpoint found!', status=400) + self.error('No salmon endpoint found!', status=400) logging.info('Discovered Salmon endpoint %s', endpoint) # construct reply Atom object