"""Handles inbound webmentions. TODO tests: * actor/attributedTo could be string URL * salmon rel via webfinger via author.name + domain """ import logging import urllib.parse from urllib.parse import urlencode import django_salmon from django_salmon import magicsigs import feedparser from google.cloud.ndb import Key from granary import as2, atom, microformats2, source import mf2util from oauth_dropins.webutil import util from oauth_dropins.webutil.util import json_dumps, json_loads import requests import webapp2 from webob import exc import activitypub import common from models import Follower, MagicKey, Response SKIP_EMAIL_DOMAINS = frozenset(('localhost', 'snarfed.org')) class WebmentionHandler(common.Handler): """Handles inbound webmention, converts to ActivityPub or Salmon.""" source_url = None # string source_domain = None # string source_mf2 = None # parsed mf2 dict source_obj = None # parsed AS1 dict target_resp = None # requests.Response def post(self): logging.info('Params: %s', list(self.request.params.items())) # fetch source page source = util.get_required_param(self, 'source') source_resp = common.requests_get(source) self.source_url = source_resp.url or source self.source_domain = urllib.parse.urlparse(self.source_url).netloc.split(':')[0] self.source_mf2 = util.parse_mf2(source_resp) # logging.debug('Parsed mf2 for %s: %s', source_resp.url, json_dumps(self.source_mf2 indent=2)) # check for backlink to bridgy fed (for webmention spec and to confirm # source's intent to federate to mastodon) if (self.request.host_url not in source_resp.text and urllib.parse.quote(self.request.host_url, safe='') not in source_resp.text): self.error("Couldn't find link to %s" % self.request.host_url) # convert source page to ActivityStreams entry = mf2util.find_first_entry(self.source_mf2, ['h-entry']) if not entry: self.error('No microformats2 found on %s' % self.source_url) logging.info('First entry: %s', json_dumps(entry, indent=2)) # make sure it has url, since we use that for AS2 id, which is required # for ActivityPub. props = entry.setdefault('properties', {}) if not props.get('url'): props['url'] = [self.source_url] self.source_obj = microformats2.json_to_object(entry, fetch_mf2=True) logging.info('Converted to AS1: %s', json_dumps(self.source_obj, indent=2)) self.try_activitypub() or self.try_salmon() # if self.source_domain not in SKIP_EMAIL_DOMAINS: # try: # msg = 'Bridgy Fed: new webmention from %s' % source # mail.send_mail( # sender='admin@bridgy-federated.appspotmail.com', # to='bridgy-fed@ryanb.org', # subject=msg, body=msg) # except BaseException: # logging.warning('Error sending email', exc_info=True) def try_activitypub(self): """Returns True if we attempted ActivityPub delivery, False otherwise.""" targets = self._activitypub_targets() if not targets: return False key = MagicKey.get_or_create(self.source_domain) error = None # TODO: collect by inbox, add 'to' fields, de-dupe inboxes and recipients for resp, inbox in targets: target_obj = json_loads(resp.target_as2) if resp.target_as2 else None source_activity = self.postprocess_as2( as2.from_as1(self.source_obj), target=target_obj, key=key) if resp.status == 'complete': source_activity['type'] = 'Update' try: last = activitypub.send(source_activity, inbox, self.source_domain) resp.status = 'complete' except BaseException as e: error = e resp.status = 'error' resp.put() # Pass the AP response status code and body through as our response if not error: self.response.status_int = last.status_code self.response.write(last.text) elif isinstance(error, requests.HTTPError): self.response.status_int = error.status_code self.response.write(error.text) else: self.response.write(str(error)) return not error def _single_target(self): """ Returns: string URL, the source's inReplyTo or object (if appropriate) """ target = util.get_first(self.source_obj, 'inReplyTo') if target: return util.get_url(target) if self.source_obj.get('verb') in source.VERBS_WITH_OBJECT: return util.get_url(util.get_first(self.source_obj, 'object')) def _activitypub_targets(self): """ Returns: list of (Response, string inbox URL) """ # if there's an in-reply-to, like-of, or repost-of, that's the target. # otherwise, it's all followers' inboxes. target = self._single_target() if not target: # interpret this as a Create or Update, deliver it to followers inboxes = [] for follower in Follower.query().filter( Follower.key > Key('Follower', self.source_domain + ' '), Follower.key < Key('Follower', self.source_domain + chr(ord(' ') + 1))): if follower.status != 'inactive' and follower.last_follow: actor = json_loads(follower.last_follow).get('actor') if actor and isinstance(actor, dict): inboxes.append(actor.get('endpoints', {}).get('sharedInbox') or actor.get('publicInbox')or actor.get('inbox')) return [(Response.get_or_create( source=self.source_url, target=inbox, direction='out', protocol='activitypub', source_mf2=json_dumps(self.source_mf2)), inbox) for inbox in inboxes if inbox] # fetch target page as AS2 object try: self.target_resp = common.get_as2(target) except (requests.HTTPError, exc.HTTPBadGateway) as e: self.target_resp = getattr(e, 'response', None) if (self.target_resp and self.target_resp.status_code // 100 == 2 and common.content_type(self.target_resp).startswith('text/html')): # TODO: pass e.response to try_salmon()'s target_resp return False # make post() try Salmon else: raise target_url = self.target_resp.url or target resp = Response.get_or_create( source=self.source_url, target=target_url, direction='out', protocol='activitypub', source_mf2=json_dumps(self.source_mf2)) # find target's inbox target_obj = self.target_resp.json() resp.target_as2 = json_dumps(target_obj) inbox_url = target_obj.get('inbox') if not inbox_url: # TODO: test actor/attributedTo and not, with/without inbox actor = (util.get_first(target_obj, 'actor') or util.get_first(target_obj, 'attributedTo')) if isinstance(actor, dict): inbox_url = actor.get('inbox') actor = actor.get('url') or actor.get('id') if not inbox_url and not actor: self.error('Target object has no actor or attributedTo with URL or id.') elif not isinstance(actor, str): self.error('Target actor or attributedTo has unexpected url or id object: %r' % actor) if not inbox_url: # fetch actor as AS object actor = common.get_as2(actor).json() inbox_url = actor.get('inbox') if not inbox_url: # TODO: probably need a way to save errors like this so that we can # return them if ostatus fails too. # self.error('Target actor has no inbox') return [] inbox_url = urllib.parse.urljoin(target_url, inbox_url) return [(resp, inbox_url)] def try_salmon(self): """Returns True if we attempted OStatus delivery. Raises otherwise.""" target = self.target_resp.url if self.target_resp else self._single_target() if not target: logging.warning("No targets or followers. Ignoring.") return False resp = Response.get_or_create( source=self.source_url, target=target, direction='out', source_mf2=json_dumps(self.source_mf2)) resp.protocol = 'ostatus' try: ret = self._try_salmon(resp) resp.status = 'complete' return ret except: resp.status = 'error' raise finally: resp.put() def _try_salmon(self, resp): """ Args: resp: Response """ # fetch target HTML page, extract Atom rel-alternate link if not self.target_resp: self.target_resp = common.requests_get(resp.target()) parsed = util.parse_html(self.target_resp) atom_url = parsed.find('link', rel='alternate', type=common.CONTENT_TYPE_ATOM) if not atom_url or not atom_url.get('href'): self.error('Target post %s has no Atom link' % resp.target(), status=400) # fetch Atom target post, extract and inject id into source object base_url = '' base = parsed.find('base') if base and base.get('href'): base_url = base['href'] atom_link = parsed.find('link', rel='alternate', type=common.CONTENT_TYPE_ATOM) atom_url = urllib.parse.urljoin( resp.target(), urllib.parse.urljoin(base_url, atom_link['href'])) feed = common.requests_get(atom_url).text parsed = feedparser.parse(feed) logging.info('Parsed: %s', json_dumps(parsed, indent=2)) entry = parsed.entries[0] target_id = entry.id in_reply_to = self.source_obj.get('inReplyTo') source_obj_obj = self.source_obj.get('object') if in_reply_to: in_reply_to[0]['id'] = target_id elif isinstance(source_obj_obj, dict): source_obj_obj['id'] = target_id # Mastodon (and maybe others?) require a rel-mentioned link to the # original post's author to make it show up as a reply: # app/services/process_interaction_service.rb # ...so add them as a tag, which atom renders as a rel-mention link. authors = entry.get('authors', None) if authors: url = entry.authors[0].get('href') if url: self.source_obj.setdefault('tags', []).append({'url': url}) # extract and discover salmon endpoint logging.info('Discovering Salmon endpoint in %s', atom_url) endpoint = django_salmon.discover_salmon_endpoint(feed) if not endpoint: # try webfinger parsed = urllib.parse.urlparse(resp.target()) # TODO: test missing email author = entry.get('author_detail', {}) email = author.get('email') or '@'.join( (author.get('name', ''), parsed.netloc)) try: # TODO: always https? profile = common.requests_get( '%s://%s/.well-known/webfinger?resource=acct:%s' % (parsed.scheme, parsed.netloc, email), verify=False) endpoint = django_salmon.get_salmon_replies_link(profile.json()) except requests.HTTPError as e: pass if not endpoint: self.error('No salmon endpoint found!', status=400) logging.info('Discovered Salmon endpoint %s', endpoint) # construct reply Atom object self.source_url = resp.source() activity = self.source_obj if self.source_obj.get('verb') not in source.VERBS_WITH_OBJECT: activity = {'object': self.source_obj} entry = atom.activity_to_atom(activity, xml_base=self.source_url) logging.info('Converted %s to Atom:\n%s', self.source_url, entry) # sign reply and wrap in magic envelope domain = urllib.parse.urlparse(self.source_url).netloc key = MagicKey.get_or_create(domain) logging.info('Using key for %s: %s', domain, key) magic_envelope = magicsigs.magic_envelope( entry, common.CONTENT_TYPE_ATOM, key).decode() logging.info('Sending Salmon slap to %s', endpoint) common.requests_post( endpoint, data=common.XML_UTF8 + magic_envelope, headers={'Content-Type': common.CONTENT_TYPE_MAGIC_ENVELOPE}) return True ROUTES = [ ('/webmention', WebmentionHandler), ]