move Flask utils to oauth_dropins.webutil.flask_util

corresponds to snarfed/webutil@5574bb23fa

add_webmention.py

@@ -5,6 +5,7 @@ import urllib.parse
 
 import flask
 from flask import request
+from oauth_dropins.webutil import flask_util
 import requests
 
 from app import app, cache
@@ -17,7 +18,7 @@ CACHE_TIME = datetime.timedelta(seconds=15)
 
 @app.get(r'/wm/<path:url>')
 @cache.cached(timeout=CACHE_TIME.total_seconds(), query_string=True,
-              response_filter=common.not_5xx)
+              response_filter=flask_util.not_5xx)
 def add_wm(url=None):
     """Proxies HTTP requests and adds Link header to our webmention endpoint."""
     url = urllib.parse.unquote(url)

app.py

@@ -3,7 +3,13 @@ import logging
 
 from flask import Flask
 from flask_caching import Cache
-from oauth_dropins.webutil import appengine_info, appengine_config, handlers, util
+from oauth_dropins.webutil import (
+    appengine_info,
+    appengine_config,
+    flask_util,
+    handlers,
+    util,
+)
 from werkzeug.exceptions import HTTPException
 
 import common
@@ -17,7 +23,7 @@ app.config.from_mapping(
     SECRET_KEY=util.read('flask_secret_key'),
     JSONIFY_PRETTYPRINT_REGULAR=True,
 )
-app.url_map.converters['regex'] = common.RegexConverter
+app.url_map.converters['regex'] = flask_util.RegexConverter
 app.wsgi_app = handlers.ndb_context_middleware(
     app.wsgi_app, client=appengine_config.ndb_client)
 
@@ -40,10 +46,8 @@ def handle_exception(e):
 
 
 # Add modern headers, but let the response override them
-from common import MODERN_HEADERS
-
 def default_modern_headers(resp):
-    for name, value in MODERN_HEADERS.items():
+    for name, value in flask_util.MODERN_HEADERS.items():
         resp.headers.setdefault(name, value)
 
     return resp

common.py

@@ -7,14 +7,11 @@ import os
 import re
 import urllib.parse
 
-from flask import render_template, request
-from flask.views import View
+from flask import request
 from granary import as2
 from oauth_dropins.webutil import util, webmention
 import requests
 from webob import exc
-from werkzeug.exceptions import abort
-from werkzeug.routing import BaseConverter
 
 from models import Response
 
@@ -50,24 +47,6 @@ CONNEG_HEADERS_AS2_HTML = {
     'Accept': CONNEG_HEADERS_AS2['Accept'] + ', %s; q=0.7' % CONTENT_TYPE_HTML,
 }
 
-# Modern HTTP headers for CORS, CSP, other security, etc.
-MODERN_HEADERS = {
-    'Access-Control-Allow-Headers': '*',
-    'Access-Control-Allow-Methods': '*',
-    'Access-Control-Allow-Origin': '*',
-    # see https://content-security-policy.com/
-    'Content-Security-Policy':
-      "script-src https: localhost:8080 my.dev.com:8080 'unsafe-inline'; "
-      "frame-ancestors 'self'; "
-      "report-uri /csp-report; ",
-    # 16070400 seconds is 6 months
-    'Strict-Transport-Security': 'max-age=16070400; preload',
-    'X-Content-Type-Options': 'nosniff',
-    # https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Frame-Options
-    'X-Frame-Options': 'SAMEORIGIN',
-    'X-XSS-Protection': '1; mode=block',
-}
-
 SUPPORTED_VERBS = (
     'checkin',
     'create',
@@ -87,40 +66,6 @@ OTHER_DOMAINS = (
 DOMAINS = (PRIMARY_DOMAIN,) + OTHER_DOMAINS
 
 
-def not_5xx(resp):
-    """Returns True if resp is an HTTP 5xx, False otherwise.
-
-    Useful to pass to `@cache.cached`'s `response_filter` kwarg to avoid caching
-    5xxes.
-
-    Args:
-      resp: :class:`flask.Response`
-
-    Returns: boolean
-    """
-    return (isinstance(resp, tuple) and len(resp) > 1 and util.is_int(resp[1]) and
-            resp[1] // 100 != 5)
-
-
-class RegexConverter(BaseConverter):
-    """Regexp URL route for Werkzeug/Flask.
-
-    Based on https://github.com/rhyselsmore/flask-reggie.
-
-    Usage:
-
-      @app.route('/<regex("(abc|def)"):letters>')
-
-    Install with:
-
-      app = Flask(...)
-      app.url_map.converters['regex'] = RegexConverter
-    """
-    def __init__(self, url_map, *items):
-        super(RegexConverter, self).__init__(url_map)
-        self.regex = items[0]
-
-
 def requests_get(url, **kwargs):
     return _requests_fn(util.requests_get, url, **kwargs)
 
@@ -202,18 +147,6 @@ def content_type(resp):
         return type.split(';')[0]
 
 
-def get_required_param(name):
-    try:
-        val = request.args.get(name) or request.form.get(name)
-    except (UnicodeDecodeError, UnicodeEncodeError) as e:
-        abort(400, f"Couldn't decode parameters as UTF-8: {e}")
-
-    if not val:
-        abort(400, f'Missing required parameter: {name}')
-
-    return val
-
-
 def error(msg, status=None, exc_info=False):
     if not status:
         status = 400
@@ -474,79 +407,3 @@ def redirect_unwrap(val):
             return util.follow_redirects(domain).url
 
     return val
-
-
-class XrdOrJrd(View):
-    """Renders and serves an XRD or JRD file.
-
-    JRD is served if the request path ends in .jrd or .json, or the format query
-    parameter is 'jrd' or 'json', or the request's Accept header includes 'jrd' or
-    'json'.
-
-    XRD is served if the request path ends in .xrd or .xml, or the format query
-    parameter is 'xml' or 'xrd', or the request's Accept header includes 'xml' or
-    'xrd'.
-
-    Otherwise, defaults to DEFAULT_TYPE.
-
-    Subclasses must override :meth:`template_prefix()` and
-    :meth:`template_vars()`. URL route variables are passed through to
-    :meth:`template_vars()` as keyword args.
-
-    Class members:
-      DEFAULT_TYPE: either JRD or XRD, which type to return by default if the
-        request doesn't ask for one explicitly with the Accept header.
-
-    """
-    JRD = 'jrd'
-    XRD = 'xrd'
-    DEFAULT_TYPE = JRD  # either JRD or XRD
-
-    def template_prefix(self):
-        """Returns template filename, without extension."""
-        raise NotImplementedError()
-
-    def template_vars(self, **kwargs):
-        """Returns a dict with template variables.
-
-        URL route variables are passed through as kwargs.
-        """
-        raise NotImplementedError()
-
-    def _type(self):
-        """Returns XRD or JRD."""
-        format = request.args.get('format', '').lower()
-        ext = os.path.splitext(request.path)[1]
-
-        if ext in ('.jrd', '.json') or format in ('jrd', 'json'):
-            return self.JRD
-        elif ext in ('.xrd', '.xml') or format in ('xrd', 'xml'):
-            return self.XRD
-
-        # We don't do full content negotiation (Accept Header parsing); we just
-        # check whether jrd/json and xrd/xml are in the header, and if they both
-        # are, which one comes first. :/
-        # https://developer.mozilla.org/en-US/docs/Web/HTTP/Content_negotiation
-        accept = request.headers.get('Accept', '').lower()
-        jrd = re.search(r'jrd|json', accept)
-        xrd = re.search(r'xrd|xml', accept)
-        if jrd and (not xrd or jrd.start() < xrd.start()):
-            return self.JRD
-        elif xrd and (not jrd or xrd.start() < jrd.start()):
-            return self.XRD
-
-        assert self.DEFAULT_TYPE in (self.JRD, self.XRD)
-        return self.DEFAULT_TYPE
-
-    def dispatch_request(self, **kwargs):
-        data = self.template_vars(**kwargs)
-        if not isinstance(data, dict):
-            return data
-
-        # Content-Types are from https://tools.ietf.org/html/rfc7033#section-10.2
-        if self._type() == self.JRD:
-            return data, {'Content-Type': 'application/jrd+json'}
-        else:
-            template = f'{self.template_prefix()}.{self._type()}'
-            return (render_template(template, **data),
-                    {'Content-Type': 'application/xrd+xml; charset=utf-8'})

logs.py

@@ -12,12 +12,11 @@ from flask import render_template
 from google.cloud import ndb
 from google.cloud.logging import Client
 import humanize
-from oauth_dropins.webutil import util
+from oauth_dropins.webutil import flask_util, util
 from oauth_dropins.webutil.appengine_info import APP_ID
 import webapp2
 
 from app import app, cache
-import common
 from common import error
 from models import Response
 
@@ -174,7 +173,7 @@ def log():
       start_time: float, seconds since the epoch
       key: string that should appear in the first app log
     """
-    start_time = common.get_required_param('start_time')
+    start_time = flask_util.get_required_param('start_time')
     if not util.is_float(start_time):
       return error("Couldn't convert start_time to float: %r" % start_time)
 
@@ -184,7 +183,7 @@ def log():
 
     client = Client()
     project = 'projects/%s' % APP_ID
-    key = urllib.parse.unquote_plus(common.get_required_param('key'))
+    key = urllib.parse.unquote_plus(flask_util.get_required_param('key'))
 
     # first, find the individual stdout log message to get the trace id
     timestamp_filter = 'timestamp>="%s" timestamp<="%s"' % (

redirect.py

@@ -16,7 +16,7 @@ import urllib.parse
 from flask import redirect, request
 from granary import as2, microformats2
 import mf2util
-from oauth_dropins.webutil import util
+from oauth_dropins.webutil import flask_util, util
 from oauth_dropins.webutil.util import json_dumps
 from werkzeug.exceptions import abort
 
@@ -30,7 +30,7 @@ CACHE_TIME = datetime.timedelta(seconds=15)
 
 @app.get(r'/r/<path:to>')
 @cache.cached(timeout=CACHE_TIME.total_seconds(), query_string=True,
-              response_filter=common.not_5xx)
+              response_filter=flask_util.not_5xx)
 def redir(to=None):
     """301 redirect to the embedded fully qualified URL.
 

render.py

@@ -4,6 +4,7 @@ import datetime
 
 from flask import request
 from granary import as2, atom, microformats2
+from oauth_dropins.webutil import flask_util
 from oauth_dropins.webutil.util import json_loads
 
 from app import app, cache
@@ -16,11 +17,11 @@ CACHE_TIME = datetime.timedelta(minutes=15)
 
 @app.get('/render')
 @cache.cached(timeout=CACHE_TIME.total_seconds(), query_string=True,
-              response_filter=common.not_5xx)
+              response_filter=flask_util.not_5xx)
 def render():
     """Fetches a stored Response and renders it as HTML."""
-    source = common.get_required_param('source')
-    target = common.get_required_param('target')
+    source = flask_util.get_required_param('source')
+    target = flask_util.get_required_param('target')
 
     id = f'{source} {target}'
     resp = Response.get_by_id(id)

tests/test_common.py

@@ -4,7 +4,6 @@ import logging
 import os
 from unittest import mock
 
-from flask import Flask, request
 from oauth_dropins.webutil import util
 from oauth_dropins.webutil.testutil import requests_response
 import requests
@@ -77,85 +76,3 @@ class CommonTest(testutil.TestCase):
                 'id': 'xyz',
                 'inReplyTo': ['foo', 'bar'],
             }))
-
-    def test_regex_converter(self):
-        app = Flask('test_regex_converter')
-        app.url_map.converters['regex'] = common.RegexConverter
-
-        @app.route('/<regex("abc|def"):letters>')
-        def fn(letters):
-            return ''
-
-        with app.test_client() as client:
-            resp = client.get('/def')
-            self.assertEqual(200, resp.status_code)
-            self.assertEqual('def', request.view_args['letters'])
-
-            resp = client.get('/xyz')
-            self.assertEqual(404, resp.status_code)
-
-
-class XrdOrJrdTest(testutil.TestCase):
-    def setUp(self):
-        super().setUp()
-
-        class View(common.XrdOrJrd):
-            def template_prefix(self):
-                return 'test_template'
-
-            def template_vars(self, **kwargs):
-                return {'foo': 'bar'}
-
-        self.View = View
-
-        self.app = Flask('XrdOrJrdTest')
-        self.app.template_folder = os.path.dirname(__file__)
-
-        view_func = View.as_view('XrdOrJrdTest')
-        self.app.add_url_rule('/', view_func=view_func)
-        self.app.add_url_rule('/<path>', view_func=view_func)
-
-        self.client = self.app.test_client()
-
-    def assert_jrd(self, resp, expected={'foo': 'bar'}):
-        self.assertEqual(200, resp.status_code)
-        self.assertEqual('application/jrd+json', resp.headers['Content-Type'])
-        self.assertEqual(expected, resp.json)
-
-    def assert_xrd(self, resp, expected='<XRD><Foo>bar</Foo></XRD>'):
-        self.assertEqual(200, resp.status_code)
-        self.assertEqual('application/xrd+xml; charset=utf-8',
-                         resp.headers['Content-Type'])
-        self.assertEqual(expected, resp.get_data(as_text=True))
-
-    def test_xrd_or_jrd_handler_default_jrd(self):
-        self.assert_jrd(self.client.get('/'))
-        for resp in (self.client.get('/x.xrd'),
-                     self.client.get('/x.xml'),
-                     self.client.get('/?format=xrd'),
-                     self.client.get('/?format=xml'),
-                     self.client.get('/', headers={'Accept': 'application/xrd+xml'}),
-                     self.client.get('/', headers={'Accept': 'application/xml'}),
-                     ):
-            self.assert_xrd(resp)
-
-    def test_xrd_or_jrd_handler_default_xrd(self):
-        self.View.DEFAULT_TYPE = common.XrdOrJrd.XRD
-
-        self.assert_xrd(self.client.get('/'))
-        for resp in (self.client.get('/x.jrd'),
-                     self.client.get('/x.json'),
-                     self.client.get('/?format=jrd'),
-                     self.client.get('/?format=json'),
-                     self.client.get('/', headers={'Accept': 'application/jrd+json'}),
-                     self.client.get('/', headers={'Accept': 'application/json'}),
-                     ):
-            self.assert_jrd(resp)
-
-    def test_xrd_or_jrd_handler_accept_header_order(self):
-        self.assert_jrd(self.client.get('/', headers={
-            'Accept': 'application/jrd+json,application/xrd+xml',
-        }))
-        self.assert_xrd(self.client.get('/', headers={
-            'Accept': 'application/xrd+xml,application/jrd+json',
-        }))

webfinger.py

@@ -14,7 +14,7 @@ from flask import render_template, request
 from flask.views import View
 from granary.microformats2 import get_text
 import mf2util
-from oauth_dropins.webutil import handlers, util
+from oauth_dropins.webutil import flask_util, handlers, util
 from oauth_dropins.webutil.util import json_dumps
 import webapp2
 
@@ -32,7 +32,7 @@ NON_TLDS = frozenset(('html', 'json', 'php', 'xml'))
 #     CACHE_TIME.total_seconds(),
 #     make_cache_key=lambda domain: f'{request.path} {request.headers.get("Accept")}')
 
-class User(common.XrdOrJrd):
+class User(flask_util.XrdOrJrd):
     """Fetches a site's home page, converts its mf2 to WebFinger, and serves."""
     def template_prefix(self):
         return 'webfinger_user'
@@ -156,7 +156,7 @@ class Webfinger(User):
     https://tools.ietf.org/html/rfc7033#section-4
     """
     def template_vars(self):
-        resource = common.get_required_param('resource')
+        resource = flask_util.get_required_param('resource')
         try:
             user, domain = util.parse_acct_uri(resource)
             if domain in common.DOMAINS:
@@ -171,13 +171,13 @@ class Webfinger(User):
         return super().template_vars(domain=domain, url=url)
 
 
-class HostMeta(common.XrdOrJrd):
+class HostMeta(flask_util.XrdOrJrd):
     """Renders and serves the /.well-known/host-meta file.
 
     Supports both JRD and XRD; defaults to XRD.
     https://tools.ietf.org/html/rfc6415#section-3
     """
-    DEFAULT_TYPE = common.XrdOrJrd.XRD
+    DEFAULT_TYPE = flask_util.XrdOrJrd.XRD
 
     def template_prefix(self):
         return 'host-meta'

webmention.py

@@ -16,7 +16,7 @@ from flask.views import View
 from google.cloud.ndb import Key
 from granary import as2, atom, microformats2, source
 import mf2util
-from oauth_dropins.webutil import util
+from oauth_dropins.webutil import flask_util, util
 from oauth_dropins.webutil.util import json_dumps, json_loads
 import requests
 import webapp2
@@ -43,7 +43,7 @@ class Webmention(View):
         logging.info(f'Params: {list(request.form.items())}')
 
         # fetch source page
-        source = common.get_required_param('source')
+        source = flask_util.get_required_param('source')
         source_resp = common.requests_get(source)
         self.source_url = source_resp.url or source
         self.source_domain = urllib.parse.urlparse(self.source_url).netloc.split(':')[0]