diff --git a/add_webmention.py b/add_webmention.py index b987979..36b7cf5 100644 --- a/add_webmention.py +++ b/add_webmention.py @@ -5,6 +5,7 @@ import urllib.parse import flask from flask import request +from oauth_dropins.webutil import flask_util import requests from app import app, cache @@ -17,7 +18,7 @@ CACHE_TIME = datetime.timedelta(seconds=15) @app.get(r'/wm/') @cache.cached(timeout=CACHE_TIME.total_seconds(), query_string=True, - response_filter=common.not_5xx) + response_filter=flask_util.not_5xx) def add_wm(url=None): """Proxies HTTP requests and adds Link header to our webmention endpoint.""" url = urllib.parse.unquote(url) diff --git a/app.py b/app.py index e60a319..0882b18 100644 --- a/app.py +++ b/app.py @@ -3,7 +3,13 @@ import logging from flask import Flask from flask_caching import Cache -from oauth_dropins.webutil import appengine_info, appengine_config, handlers, util +from oauth_dropins.webutil import ( + appengine_info, + appengine_config, + flask_util, + handlers, + util, +) from werkzeug.exceptions import HTTPException import common @@ -17,7 +23,7 @@ app.config.from_mapping( SECRET_KEY=util.read('flask_secret_key'), JSONIFY_PRETTYPRINT_REGULAR=True, ) -app.url_map.converters['regex'] = common.RegexConverter +app.url_map.converters['regex'] = flask_util.RegexConverter app.wsgi_app = handlers.ndb_context_middleware( app.wsgi_app, client=appengine_config.ndb_client) @@ -40,10 +46,8 @@ def handle_exception(e): # Add modern headers, but let the response override them -from common import MODERN_HEADERS - def default_modern_headers(resp): - for name, value in MODERN_HEADERS.items(): + for name, value in flask_util.MODERN_HEADERS.items(): resp.headers.setdefault(name, value) return resp diff --git a/common.py b/common.py index f1f0928..c81e6eb 100644 --- a/common.py +++ b/common.py @@ -7,14 +7,11 @@ import os import re import urllib.parse -from flask import render_template, request -from flask.views import View +from flask import request from granary import as2 from oauth_dropins.webutil import util, webmention import requests from webob import exc -from werkzeug.exceptions import abort -from werkzeug.routing import BaseConverter from models import Response @@ -50,24 +47,6 @@ CONNEG_HEADERS_AS2_HTML = { 'Accept': CONNEG_HEADERS_AS2['Accept'] + ', %s; q=0.7' % CONTENT_TYPE_HTML, } -# Modern HTTP headers for CORS, CSP, other security, etc. -MODERN_HEADERS = { - 'Access-Control-Allow-Headers': '*', - 'Access-Control-Allow-Methods': '*', - 'Access-Control-Allow-Origin': '*', - # see https://content-security-policy.com/ - 'Content-Security-Policy': - "script-src https: localhost:8080 my.dev.com:8080 'unsafe-inline'; " - "frame-ancestors 'self'; " - "report-uri /csp-report; ", - # 16070400 seconds is 6 months - 'Strict-Transport-Security': 'max-age=16070400; preload', - 'X-Content-Type-Options': 'nosniff', - # https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Frame-Options - 'X-Frame-Options': 'SAMEORIGIN', - 'X-XSS-Protection': '1; mode=block', -} - SUPPORTED_VERBS = ( 'checkin', 'create', @@ -87,40 +66,6 @@ OTHER_DOMAINS = ( DOMAINS = (PRIMARY_DOMAIN,) + OTHER_DOMAINS -def not_5xx(resp): - """Returns True if resp is an HTTP 5xx, False otherwise. - - Useful to pass to `@cache.cached`'s `response_filter` kwarg to avoid caching - 5xxes. - - Args: - resp: :class:`flask.Response` - - Returns: boolean - """ - return (isinstance(resp, tuple) and len(resp) > 1 and util.is_int(resp[1]) and - resp[1] // 100 != 5) - - -class RegexConverter(BaseConverter): - """Regexp URL route for Werkzeug/Flask. - - Based on https://github.com/rhyselsmore/flask-reggie. - - Usage: - - @app.route('/') - - Install with: - - app = Flask(...) - app.url_map.converters['regex'] = RegexConverter - """ - def __init__(self, url_map, *items): - super(RegexConverter, self).__init__(url_map) - self.regex = items[0] - - def requests_get(url, **kwargs): return _requests_fn(util.requests_get, url, **kwargs) @@ -202,18 +147,6 @@ def content_type(resp): return type.split(';')[0] -def get_required_param(name): - try: - val = request.args.get(name) or request.form.get(name) - except (UnicodeDecodeError, UnicodeEncodeError) as e: - abort(400, f"Couldn't decode parameters as UTF-8: {e}") - - if not val: - abort(400, f'Missing required parameter: {name}') - - return val - - def error(msg, status=None, exc_info=False): if not status: status = 400 @@ -474,79 +407,3 @@ def redirect_unwrap(val): return util.follow_redirects(domain).url return val - - -class XrdOrJrd(View): - """Renders and serves an XRD or JRD file. - - JRD is served if the request path ends in .jrd or .json, or the format query - parameter is 'jrd' or 'json', or the request's Accept header includes 'jrd' or - 'json'. - - XRD is served if the request path ends in .xrd or .xml, or the format query - parameter is 'xml' or 'xrd', or the request's Accept header includes 'xml' or - 'xrd'. - - Otherwise, defaults to DEFAULT_TYPE. - - Subclasses must override :meth:`template_prefix()` and - :meth:`template_vars()`. URL route variables are passed through to - :meth:`template_vars()` as keyword args. - - Class members: - DEFAULT_TYPE: either JRD or XRD, which type to return by default if the - request doesn't ask for one explicitly with the Accept header. - - """ - JRD = 'jrd' - XRD = 'xrd' - DEFAULT_TYPE = JRD # either JRD or XRD - - def template_prefix(self): - """Returns template filename, without extension.""" - raise NotImplementedError() - - def template_vars(self, **kwargs): - """Returns a dict with template variables. - - URL route variables are passed through as kwargs. - """ - raise NotImplementedError() - - def _type(self): - """Returns XRD or JRD.""" - format = request.args.get('format', '').lower() - ext = os.path.splitext(request.path)[1] - - if ext in ('.jrd', '.json') or format in ('jrd', 'json'): - return self.JRD - elif ext in ('.xrd', '.xml') or format in ('xrd', 'xml'): - return self.XRD - - # We don't do full content negotiation (Accept Header parsing); we just - # check whether jrd/json and xrd/xml are in the header, and if they both - # are, which one comes first. :/ - # https://developer.mozilla.org/en-US/docs/Web/HTTP/Content_negotiation - accept = request.headers.get('Accept', '').lower() - jrd = re.search(r'jrd|json', accept) - xrd = re.search(r'xrd|xml', accept) - if jrd and (not xrd or jrd.start() < xrd.start()): - return self.JRD - elif xrd and (not jrd or xrd.start() < jrd.start()): - return self.XRD - - assert self.DEFAULT_TYPE in (self.JRD, self.XRD) - return self.DEFAULT_TYPE - - def dispatch_request(self, **kwargs): - data = self.template_vars(**kwargs) - if not isinstance(data, dict): - return data - - # Content-Types are from https://tools.ietf.org/html/rfc7033#section-10.2 - if self._type() == self.JRD: - return data, {'Content-Type': 'application/jrd+json'} - else: - template = f'{self.template_prefix()}.{self._type()}' - return (render_template(template, **data), - {'Content-Type': 'application/xrd+xml; charset=utf-8'}) diff --git a/logs.py b/logs.py index 2ebd6fa..b24b646 100644 --- a/logs.py +++ b/logs.py @@ -12,12 +12,11 @@ from flask import render_template from google.cloud import ndb from google.cloud.logging import Client import humanize -from oauth_dropins.webutil import util +from oauth_dropins.webutil import flask_util, util from oauth_dropins.webutil.appengine_info import APP_ID import webapp2 from app import app, cache -import common from common import error from models import Response @@ -174,7 +173,7 @@ def log(): start_time: float, seconds since the epoch key: string that should appear in the first app log """ - start_time = common.get_required_param('start_time') + start_time = flask_util.get_required_param('start_time') if not util.is_float(start_time): return error("Couldn't convert start_time to float: %r" % start_time) @@ -184,7 +183,7 @@ def log(): client = Client() project = 'projects/%s' % APP_ID - key = urllib.parse.unquote_plus(common.get_required_param('key')) + key = urllib.parse.unquote_plus(flask_util.get_required_param('key')) # first, find the individual stdout log message to get the trace id timestamp_filter = 'timestamp>="%s" timestamp<="%s"' % ( diff --git a/redirect.py b/redirect.py index 8404030..462cae4 100644 --- a/redirect.py +++ b/redirect.py @@ -16,7 +16,7 @@ import urllib.parse from flask import redirect, request from granary import as2, microformats2 import mf2util -from oauth_dropins.webutil import util +from oauth_dropins.webutil import flask_util, util from oauth_dropins.webutil.util import json_dumps from werkzeug.exceptions import abort @@ -30,7 +30,7 @@ CACHE_TIME = datetime.timedelta(seconds=15) @app.get(r'/r/') @cache.cached(timeout=CACHE_TIME.total_seconds(), query_string=True, - response_filter=common.not_5xx) + response_filter=flask_util.not_5xx) def redir(to=None): """301 redirect to the embedded fully qualified URL. diff --git a/render.py b/render.py index ada884c..cb63436 100644 --- a/render.py +++ b/render.py @@ -4,6 +4,7 @@ import datetime from flask import request from granary import as2, atom, microformats2 +from oauth_dropins.webutil import flask_util from oauth_dropins.webutil.util import json_loads from app import app, cache @@ -16,11 +17,11 @@ CACHE_TIME = datetime.timedelta(minutes=15) @app.get('/render') @cache.cached(timeout=CACHE_TIME.total_seconds(), query_string=True, - response_filter=common.not_5xx) + response_filter=flask_util.not_5xx) def render(): """Fetches a stored Response and renders it as HTML.""" - source = common.get_required_param('source') - target = common.get_required_param('target') + source = flask_util.get_required_param('source') + target = flask_util.get_required_param('target') id = f'{source} {target}' resp = Response.get_by_id(id) diff --git a/tests/test_common.py b/tests/test_common.py index 99a3fe6..c58ee66 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -4,7 +4,6 @@ import logging import os from unittest import mock -from flask import Flask, request from oauth_dropins.webutil import util from oauth_dropins.webutil.testutil import requests_response import requests @@ -77,85 +76,3 @@ class CommonTest(testutil.TestCase): 'id': 'xyz', 'inReplyTo': ['foo', 'bar'], })) - - def test_regex_converter(self): - app = Flask('test_regex_converter') - app.url_map.converters['regex'] = common.RegexConverter - - @app.route('/') - def fn(letters): - return '' - - with app.test_client() as client: - resp = client.get('/def') - self.assertEqual(200, resp.status_code) - self.assertEqual('def', request.view_args['letters']) - - resp = client.get('/xyz') - self.assertEqual(404, resp.status_code) - - -class XrdOrJrdTest(testutil.TestCase): - def setUp(self): - super().setUp() - - class View(common.XrdOrJrd): - def template_prefix(self): - return 'test_template' - - def template_vars(self, **kwargs): - return {'foo': 'bar'} - - self.View = View - - self.app = Flask('XrdOrJrdTest') - self.app.template_folder = os.path.dirname(__file__) - - view_func = View.as_view('XrdOrJrdTest') - self.app.add_url_rule('/', view_func=view_func) - self.app.add_url_rule('/', view_func=view_func) - - self.client = self.app.test_client() - - def assert_jrd(self, resp, expected={'foo': 'bar'}): - self.assertEqual(200, resp.status_code) - self.assertEqual('application/jrd+json', resp.headers['Content-Type']) - self.assertEqual(expected, resp.json) - - def assert_xrd(self, resp, expected='bar'): - self.assertEqual(200, resp.status_code) - self.assertEqual('application/xrd+xml; charset=utf-8', - resp.headers['Content-Type']) - self.assertEqual(expected, resp.get_data(as_text=True)) - - def test_xrd_or_jrd_handler_default_jrd(self): - self.assert_jrd(self.client.get('/')) - for resp in (self.client.get('/x.xrd'), - self.client.get('/x.xml'), - self.client.get('/?format=xrd'), - self.client.get('/?format=xml'), - self.client.get('/', headers={'Accept': 'application/xrd+xml'}), - self.client.get('/', headers={'Accept': 'application/xml'}), - ): - self.assert_xrd(resp) - - def test_xrd_or_jrd_handler_default_xrd(self): - self.View.DEFAULT_TYPE = common.XrdOrJrd.XRD - - self.assert_xrd(self.client.get('/')) - for resp in (self.client.get('/x.jrd'), - self.client.get('/x.json'), - self.client.get('/?format=jrd'), - self.client.get('/?format=json'), - self.client.get('/', headers={'Accept': 'application/jrd+json'}), - self.client.get('/', headers={'Accept': 'application/json'}), - ): - self.assert_jrd(resp) - - def test_xrd_or_jrd_handler_accept_header_order(self): - self.assert_jrd(self.client.get('/', headers={ - 'Accept': 'application/jrd+json,application/xrd+xml', - })) - self.assert_xrd(self.client.get('/', headers={ - 'Accept': 'application/xrd+xml,application/jrd+json', - })) diff --git a/webfinger.py b/webfinger.py index f6653c0..21a2d10 100644 --- a/webfinger.py +++ b/webfinger.py @@ -14,7 +14,7 @@ from flask import render_template, request from flask.views import View from granary.microformats2 import get_text import mf2util -from oauth_dropins.webutil import handlers, util +from oauth_dropins.webutil import flask_util, handlers, util from oauth_dropins.webutil.util import json_dumps import webapp2 @@ -32,7 +32,7 @@ NON_TLDS = frozenset(('html', 'json', 'php', 'xml')) # CACHE_TIME.total_seconds(), # make_cache_key=lambda domain: f'{request.path} {request.headers.get("Accept")}') -class User(common.XrdOrJrd): +class User(flask_util.XrdOrJrd): """Fetches a site's home page, converts its mf2 to WebFinger, and serves.""" def template_prefix(self): return 'webfinger_user' @@ -156,7 +156,7 @@ class Webfinger(User): https://tools.ietf.org/html/rfc7033#section-4 """ def template_vars(self): - resource = common.get_required_param('resource') + resource = flask_util.get_required_param('resource') try: user, domain = util.parse_acct_uri(resource) if domain in common.DOMAINS: @@ -171,13 +171,13 @@ class Webfinger(User): return super().template_vars(domain=domain, url=url) -class HostMeta(common.XrdOrJrd): +class HostMeta(flask_util.XrdOrJrd): """Renders and serves the /.well-known/host-meta file. Supports both JRD and XRD; defaults to XRD. https://tools.ietf.org/html/rfc6415#section-3 """ - DEFAULT_TYPE = common.XrdOrJrd.XRD + DEFAULT_TYPE = flask_util.XrdOrJrd.XRD def template_prefix(self): return 'host-meta' diff --git a/webmention.py b/webmention.py index 4f9931c..25754c5 100644 --- a/webmention.py +++ b/webmention.py @@ -16,7 +16,7 @@ from flask.views import View from google.cloud.ndb import Key from granary import as2, atom, microformats2, source import mf2util -from oauth_dropins.webutil import util +from oauth_dropins.webutil import flask_util, util from oauth_dropins.webutil.util import json_dumps, json_loads import requests import webapp2 @@ -43,7 +43,7 @@ class Webmention(View): logging.info(f'Params: {list(request.form.items())}') # fetch source page - source = common.get_required_param('source') + source = flask_util.get_required_param('source') source_resp = common.requests_get(source) self.source_url = source_resp.url or source self.source_domain = urllib.parse.urlparse(self.source_url).netloc.split(':')[0]