start to make User subclasses for each protocol

#512

activitypub.py

@@ -1,5 +1,4 @@
-"""Handles requests for ActivityPub endpoints: actors, inbox, etc.
-"""
+"""ActivityPub protocol implementation."""
 from base64 import b64encode
 from hashlib import sha256
 import itertools
@@ -29,6 +28,7 @@ from common import (
 )
 from models import Follower, Object, Target, User
 from protocol import Protocol
+import webmention
 
 logger = logging.getLogger(__name__)
 
@@ -43,11 +43,11 @@ _DEFAULT_SIGNATURE_USER = None
 def default_signature_user():
     global _DEFAULT_SIGNATURE_USER
     if _DEFAULT_SIGNATURE_USER is None:
-        _DEFAULT_SIGNATURE_USER = User.get_or_create('snarfed.org')
+        _DEFAULT_SIGNATURE_USER = webmention.Webmention.get_or_create('snarfed.org')
     return _DEFAULT_SIGNATURE_USER
 
 
-class ActivityPub(Protocol):
+class ActivityPub(User, Protocol):
     """ActivityPub protocol class."""
     LABEL = 'activitypub'
 
@@ -513,11 +513,12 @@ def actor(domain):
     if tld in TLD_BLOCKLIST:
         error('', status=404)
 
-    g.user = User.get_by_id(domain)
+    # TODO(#512): parameterize by protocol
+    g.user = webmention.Webmention.get_by_id(domain)
     if not g.user:
-        return f'User {domain} not found', 404
+        return f'Web user {domain} not found', 404
     elif not g.user.actor_as2:
-        return f'User {domain} not fully set up', 404
+        return f'Web user {domain} not fully set up', 404
 
     # TODO: unify with common.actor()
     actor = postprocess_as2(g.user.actor_as2)
@@ -565,9 +566,10 @@ def inbox(domain=None):
 
     # load user
     if domain:
-        g.user = User.get_by_id(domain)
+        # TODO(#512): parameterize by protocol
+        g.user = webmention.Webmention.get_by_id(domain)
         if not g.user:
-            error(f'User {domain} not found', status=404)
+            error(f'Web user {domain} not found', status=404)
 
     ActivityPub.verify_signature(activity)
 
@@ -603,8 +605,9 @@ def follower_collection(domain, collection):
     https://www.w3.org/TR/activitypub/#collections
     https://www.w3.org/TR/activitystreams-core/#paging
     """
-    if not User.get_by_id(domain):
-        return f'User {domain} not found', 404
+    # TODO(#512): parameterize by protocol
+    if not webmention.Webmention.get_by_id(domain):
+        return f'Web user {domain} not found', 404
 
     # page
     followers, new_before, new_after = Follower.fetch_page(domain, collection)

app.py

@@ -6,4 +6,7 @@ registered.
 from flask_app import app
 
 # import all modules to register their Flask handlers
-import activitypub, convert, follow, pages, redirect, superfeedr, webfinger, webmention, xrpc_actor, xrpc_feed, xrpc_graph
+import activitypub, convert, follow, pages, redirect, superfeedr, ui, webfinger, webmention, xrpc_actor, xrpc_feed, xrpc_graph
+
+import models
+models.reset_protocol_properties()

convert.py

@@ -17,8 +17,7 @@ from oauth_dropins.webutil.flask_util import error
 from activitypub import ActivityPub
 from common import CACHE_TIME
 from flask_app import app, cache
-from models import Object
-from protocol import protocols
+from models import Object, PROTOCOLS
 from webmention import Webmention
 
 logger = logging.getLogger(__name__)
@@ -52,7 +51,7 @@ def convert(src, dest, _):
         error(f'Expected fully qualified URL; got {url}')
 
     # load, and maybe fetch. if it's a post/update, redirect to inner object.
-    obj = protocols[src].load(url)
+    obj = PROTOCOLS[src].load(url)
     if not obj.as1:
         error(f'Stored object for {id} has no data', status=404)
 
@@ -60,7 +59,7 @@ def convert(src, dest, _):
     if type in ('post', 'update', 'delete'):
         obj_id = as1.get_object(obj.as1).get('id')
         if obj_id:
-            # TODO: protocols[src].load() this instead?
+            # TODO: PROTOCOLS[src].load() this instead?
             obj_obj = Object.get_by_id(obj_id)
             if (obj_obj and obj_obj.as1 and
                 not obj_obj.as1.keys() <= set(['id', 'url', 'objectType'])):
@@ -72,7 +71,7 @@ def convert(src, dest, _):
         return '', 410
 
     # convert and serve
-    return protocols[dest].serve(obj)
+    return PROTOCOLS[dest].serve(obj)
 
 
 @app.get('/render')

follow.py

@@ -20,6 +20,7 @@ from activitypub import ActivityPub
 from flask_app import app
 import common
 import models
+from webmention import Webmention
 
 logger = logging.getLogger(__name__)
 
@@ -80,9 +81,10 @@ def remote_follow():
     logger.info(f'Got: {request.values}')
 
     domain = request.values['domain']
-    g.user = models.User.get_by_id(domain)
+    # TODO(#512): parameterize by protocol
+    g.user = Webmention.get_by_id(domain)
     if not g.user:
-        error(f'No Bridgy Fed user found for domain {domain}')
+        error(f'No web user found for domain {domain}')
 
     addr = request.values['address']
     webfinger = fetch_webfinger(addr)
@@ -133,9 +135,10 @@ class FollowCallback(indieauth.Callback):
         session['indieauthed-me'] = me
 
         domain = util.domain_from_link(me)
-        g.user = models.User.get_by_id(domain)
+        # TODO(#512): parameterize by protocol
+        g.user = Webmention.get_by_id(domain)
         if not g.user:
-            error(f'No user for domain {domain}')
+            error(f'No web user for domain {domain}')
         domain = g.user.key.id()
 
         addr = state
@@ -220,9 +223,10 @@ class UnfollowCallback(indieauth.Callback):
         session['indieauthed-me'] = me
 
         domain = util.domain_from_link(me)
-        g.user = models.User.get_by_id(domain)
+        # TODO(#512): parameterize by protocol
+        g.user = Webmention.get_by_id(domain)
         if not g.user:
-            error(f'No user for domain {domain}')
+            error(f'No web user for domain {domain}')
         domain = g.user.key.id()
 
         follower = models.Follower.get_by_id(state)

models.py

@@ -1,7 +1,6 @@
 """Datastore model classes."""
 import base64
 from datetime import timedelta, timezone
-import difflib
 import itertools
 import json
 import logging
@@ -22,17 +21,13 @@ from oauth_dropins.webutil.flask_util import error
 from oauth_dropins.webutil.models import ComputedJsonProperty, JsonProperty, StringIdModel
 from oauth_dropins.webutil.util import json_dumps, json_loads
 import requests
-from werkzeug.exceptions import BadRequest, NotFound
 
 import common
 
-# https://github.com/snarfed/bridgy-fed/issues/314
-WWW_DOMAINS = frozenset((
-    'www.jvt.me',
-))
-# TODO: eventually load from protocol.protocols instead, if/when we can get
-# around the circular import
-PROTOCOLS = ('activitypub', 'bluesky', 'ostatus', 'webmention', 'ui')
+# maps string label to Protocol subclass. populated by ProtocolUserMeta.
+# seed with old and upcoming protocols that don't have their own classes (yet).
+PROTOCOLS = {'bluesky': None, 'ostatus': None}
+
 # 2048 bits makes tests slow, so use 1024 for them
 KEY_BITS = 1024 if DEBUG else 2048
 PAGE_SIZE = 20
@@ -53,6 +48,21 @@ OBJECT_EXPIRE_AGE = timedelta(days=90)
 logger = logging.getLogger(__name__)
 
 
+class ProtocolUserMeta(type(ndb.Model)):
+    """:class:`User` metaclass. Registers all subclasses in the PROTOCOLS global."""
+    def __new__(meta, name, bases, class_dict):
+        cls = super().__new__(meta, name, bases, class_dict)
+        if hasattr(cls, 'LABEL'):
+            PROTOCOLS[cls.LABEL] = cls
+        return cls
+
+
+def reset_protocol_properties():
+    """Recreates various protocol properties to include choices PROTOCOLS."""
+    Target.protocol = ndb.StringProperty(choices=list(PROTOCOLS.keys()), required=True)
+    Object.source_protocol = ndb.StringProperty(choices=list(PROTOCOLS.keys()))
+
+
 def base64_to_long(x):
     """Converts x from URL safe base64 encoding to a long integer.
 
@@ -69,24 +79,20 @@ def long_to_base64(x):
     return base64.urlsafe_b64encode(number.long_to_bytes(x))
 
 
-class User(StringIdModel):
-    """Stores a Bridgy Fed user.
-
-    The key name is the domain.
+class User(StringIdModel, metaclass=ProtocolUserMeta):
+    """Abstract base class for a Bridgy Fed user.
 
     Stores multiple keypairs needed for the supported protocols. Currently:
 
     * RSA keypair for ActivityPub HTTP Signatures
-      properties: mod, public_exponent, private_exponent
+      properties: mod, public_exponent, private_exponent, all encoded as
+        base64url (ie URL-safe base64) strings as described in RFC 4648 and
+        section 5.1 of the Magic Signatures spec
       https://tools.ietf.org/html/draft-cavage-http-signatures-12
 
     * P-256 keypair for AT Protocol's signing key
       property: p256_key, PEM encoded
       https://atproto.com/guides/overview#account-portability
-
-    The key pair's modulus and exponent properties are all encoded as base64url
-    (ie URL-safe base64) strings as described in RFC 4648 and section 5.1 of the
-    Magic Signatures spec.
     """
     mod = ndb.StringProperty(required=True)
     public_exponent = ndb.StringProperty(required=True)
@@ -101,16 +107,18 @@ class User(StringIdModel):
     created = ndb.DateTimeProperty(auto_now_add=True)
     updated = ndb.DateTimeProperty(auto_now=True)
 
+    @classmethod
+    def new(cls, **kwargs):
+        """Try to prevent instantiation. Use subclasses instead."""
+        raise NotImplementedError()
+
+    # TODO(#512): move this and is_homepage to webmention.py?
     @property
     def homepage(self):
         return f'https://{self.key.id()}/'
 
-    @classmethod
-    def _get_kind(cls):
-        return 'MagicKey'
-
     def _post_put_hook(self, future):
-        logger.info(f'Wrote User {self.key.id()}')
+        logger.info(f'Wrote {self.key}')
 
     @classmethod
     def get_by_id(cls, id):
@@ -121,27 +129,34 @@ class User(StringIdModel):
 
         return user
 
-    @staticmethod
+    @classmethod
     @ndb.transactional()
-    def get_or_create(domain, **kwargs):
+    def get_or_create(cls, domain, **kwargs):
         """Loads and returns a User. Creates it if necessary."""
-        user = User.get_by_id(domain)
+        assert cls != User
+        user = cls.get_by_id(domain)
         if user:
             return user
 
-        # originally from django_salmon.magicsigs
-        # this uses urandom(), and does nontrivial math, so it can take a
-        # while depending on the amount of randomness available.
-        rng = Random.new().read
-        rsa_key = RSA.generate(KEY_BITS, rng)
-        p256_key = ECC.generate(curve='P-256',
-                                randfunc=random.randbytes if DEBUG else None)
-        user = User(id=domain,
-                    mod=long_to_base64(rsa_key.n),
-                    public_exponent=long_to_base64(rsa_key.e),
-                    private_exponent=long_to_base64(rsa_key.d),
-                    p256_key=p256_key.export_key(format='PEM'),
-                    **kwargs)
+        # generate keys for all protocols _except_ our own
+        #
+        # these can use urandom() and do nontrivial math, so they can take time
+        # depending on the amount of randomness available and compute needed.
+        if cls.LABEL != 'activitypub':
+            # originally from django_salmon.magicsigs
+            key = RSA.generate(KEY_BITS, randfunc=random.randbytes if DEBUG else None)
+            kwargs.update({
+                    'mod': long_to_base64(key.n),
+                    'public_exponent': long_to_base64(key.e),
+                    'private_exponent': long_to_base64(key.d),
+            })
+
+        if cls.LABEL != 'atprotocol':
+            key = ECC.generate(
+                curve='P-256', randfunc=random.randbytes if DEBUG else None)
+            kwargs['p256_key'] = key.export_key(format='PEM')
+
+        user = cls(id=domain, **kwargs)
         user.put()
         return user
 
@@ -222,67 +237,6 @@ class User(StringIdModel):
 
         return f'<a class="h-card u-author" href="/user/{domain}"><img src="{img}" class="profile"> {name}</a>'
 
-    def verify(self):
-        """Fetches site a couple ways to check for redirects and h-card.
-
-        Returns: User that was verified. May be different than self! eg if self's
-          domain started with www and we switch to the root domain.
-        """
-        domain = self.key.id()
-        logger.info(f'Verifying {domain}')
-
-        if domain.startswith('www.') and domain not in WWW_DOMAINS:
-            # if root domain redirects to www, use root domain instead
-            # https://github.com/snarfed/bridgy-fed/issues/314
-            root = domain.removeprefix("www.")
-            root_site = f'https://{root}/'
-            try:
-                resp = util.requests_get(root_site, gateway=False)
-                if resp.ok and self.is_homepage(resp.url):
-                    logger.info(f'{root_site} redirects to {resp.url} ; using {root} instead')
-                    root_user = User.get_or_create(root)
-                    self.use_instead = root_user.key
-                    self.put()
-                    return root_user.verify()
-            except requests.RequestException:
-                pass
-
-        # check webfinger redirect
-        path = f'/.well-known/webfinger?resource=acct:{domain}@{domain}'
-        self.has_redirects = False
-        self.redirects_error = None
-        try:
-            url = urllib.parse.urljoin(self.homepage, path)
-            resp = util.requests_get(url, gateway=False)
-            domain_urls = ([f'https://{domain}/' for domain in common.DOMAINS] +
-                           [common.host_url()])
-            expected = [urllib.parse.urljoin(url, path) for url in domain_urls]
-            if resp.ok:
-                if resp.url in expected:
-                    self.has_redirects = True
-                elif resp.url:
-                    diff = '\n'.join(difflib.Differ().compare([resp.url], [expected[0]]))
-                    self.redirects_error = f'Current vs expected:<pre>{diff}</pre>'
-            else:
-                lines = [url, f'  returned HTTP {resp.status_code}']
-                if resp.url != url:
-                    lines[1:1] = ['  redirected to:', resp.url]
-                self.redirects_error = '<pre>' + '\n'.join(lines) + '</pre>'
-        except requests.RequestException:
-            pass
-
-        # check home page
-        try:
-            import activitypub, webmention  # TODO: actually fix these circular imports
-            obj = webmention.Webmention.load(self.homepage, gateway=True)
-            self.actor_as2 = activitypub.postprocess_as2(as2.from_as1(obj.as1))
-            self.has_hcard = True
-        except (BadRequest, NotFound):
-            self.actor_as2 = None
-            self.has_hcard = False
-
-        return self
-
 
 class Target(ndb.Model):
     """Delivery destinations. ActivityPub inboxes, webmention targets, etc.
@@ -301,7 +255,9 @@ class Target(ndb.Model):
     https://googleapis.dev/python/python-ndb/latest/model.html#google.cloud.ndb.model.StructuredProperty
     """
     uri = ndb.StringProperty(required=True)
-    protocol = ndb.StringProperty(choices=PROTOCOLS, required=True)
+    # choices is populated in flask_app, after all User subclasses are created,
+    # so that PROTOCOLS is fully populated
+    protocol = ndb.StringProperty(choices=[], required=True)
 
 
 class Object(StringIdModel):
@@ -315,8 +271,10 @@ class Object(StringIdModel):
     # domains of the Bridgy Fed users this activity is to or from
     domains = ndb.StringProperty(repeated=True)
     status = ndb.StringProperty(choices=STATUSES)
+    # choices is populated in flask_app, after all User subclasses are created,
+    # so that PROTOCOLS is fully populated
     # TODO: remove? is this redundant with the protocol-specific data fields below?
-    source_protocol = ndb.StringProperty(choices=PROTOCOLS)
+    source_protocol = ndb.StringProperty(choices=[])
     labels = ndb.StringProperty(repeated=True, choices=LABELS)
 
     # TODO: switch back to ndb.JsonProperty if/when they fix it for the web console

pages.py

@@ -18,6 +18,7 @@ from flask_app import app, cache
 import common
 from common import DOMAIN_RE
 from models import fetch_page, Follower, Object, PAGE_SIZE, User
+from webmention import Webmention
 
 FOLLOWERS_UI_LIMIT = 999
 
@@ -49,7 +50,7 @@ def docs():
 def enter_web_site():
     return render_template('enter_web_site.html')
 
-
+# TODO(#512): move to webmention.py?
 @app.post('/web-site')
 def check_web_site():
     url = request.values['url']
@@ -58,7 +59,7 @@ def check_web_site():
         flash(f'No domain found in {url}')
         return render_template('enter_web_site.html')
 
-    g.user = User.get_or_create(domain)
+    g.user = Webmention.get_or_create(domain)
     try:
         g.user = g.user.verify()
     except BaseException as e:
@@ -74,7 +75,7 @@ def check_web_site():
 
 @app.get(f'/user/<regex("{DOMAIN_RE}"):domain>')
 def user(domain):
-    g.user = User.get_by_id(domain)
+    g.user = Webmention.get_by_id(domain)
     if not g.user:
         return USER_NOT_FOUND_HTML, 404
     elif g.user.key.id() != domain:
@@ -109,7 +110,7 @@ def user(domain):
 
 @app.get(f'/user/<regex("{DOMAIN_RE}"):domain>/<any(followers,following):collection>')
 def followers_or_following(domain, collection):
-    g.user = User.get_by_id(domain)  # g.user is used in template
+    g.user = Webmention.get_by_id(domain)  # g.user is used in template
     if not g.user:
         return USER_NOT_FOUND_HTML, 404
 
@@ -138,7 +139,7 @@ def feed(domain):
     if format not in ('html', 'atom', 'rss'):
         error(f'format {format} not supported; expected html, atom, or rss')
 
-    g.user = User.get_by_id(domain)
+    g.user = Webmention.get_by_id(domain)
     if not g.user:
         return render_template('user_not_found.html', domain=domain), 404
 

protocol.py

@@ -10,7 +10,7 @@ from granary import as1, as2
 
 import common
 from common import error
-from models import Follower, Object, Target, User
+from models import Follower, Object, Target
 from oauth_dropins.webutil import util, webmention
 from oauth_dropins.webutil.util import json_dumps, json_loads
 
@@ -45,19 +45,7 @@ objects_cache_lock = threading.Lock()
 logger = logging.getLogger(__name__)
 
 
-# maps string label to Protocol subclass. populated by ProtocolMeta.
-protocols = {}
-
-class ProtocolMeta(type):
-    """:class:`Protocol` metaclass. Registers all subclasses in the protocols global."""
-    def __new__(meta, name, bases, class_dict):
-        cls = super().__new__(meta, name, bases, class_dict)
-        if cls.LABEL:
-            protocols[cls.LABEL] = cls
-        return cls
-
-
-class Protocol(metaclass=ProtocolMeta):
+class Protocol:
     """Base protocol class. Not to be instantiated; classmethods only.
 
     Attributes:

redirect.py

@@ -76,16 +76,18 @@ def redir(to):
                    to_domain))
     for domain in domains:
         if domain:
-            g.user = User.get_by_id(domain)
+            # TODO(#512): do we need to parameterize this by protocol? or is it
+            # only for web?
+            g.user = Webmention.get_by_id(domain)
             if g.user:
-                logger.info(f'Found User for domain {domain}')
+                logger.info(f'Found web user for domain {domain}')
                 break
     else:
         if accept_as2:
             g.external_user = urllib.parse.urljoin(to, '/')
-            logging.info(f'No User for {g.external_user}')
+            logging.info(f'No web user for {g.external_user}')
         else:
-            return f'No user found for any of {domains}', 404
+            return f'No web user found for any of {domains}', 404
 
     if accept_as2:
         # AS2 requested, fetch and convert and serve

tests/test_activitypub.py

@@ -25,10 +25,12 @@ from activitypub import ActivityPub
 from flask_app import app
 import common
 import models
-from models import Follower, Object, User
+from models import Follower, Object
 import protocol
 from protocol import Protocol
-from . import testutil
+from webmention import Webmention
+
+from .testutil import Fake, TestCase
 
 ACTOR = {
     '@context': 'https://www.w3.org/ns/activitystreams',
@@ -222,11 +224,12 @@ NOT_ACCEPTABLE = requests_response(status=406)
 @patch('requests.post')
 @patch('requests.get')
 @patch('requests.head')
-class ActivityPubTest(testutil.TestCase):
+class ActivityPubTest(TestCase):
 
     def setUp(self):
         super().setUp()
-        self.user = self.make_user('user.com', has_hcard=True, actor_as2=ACTOR)
+        self.user = self.make_user('user.com',
+                                   has_hcard=True, actor_as2=ACTOR)
         with self.request_context:
             self.key_id_obj = Object(id='http://my/key/id', as2={
                 **ACTOR,
@@ -1163,12 +1166,12 @@ class ActivityPubTest(testutil.TestCase):
         }, resp.json)
 
 
-class ActivityPubUtilsTest(testutil.TestCase):
+class ActivityPubUtilsTest(TestCase):
     def setUp(self):
         super().setUp()
         self.request_context.push()
-        g.user = self.make_user('user.com', has_hcard=True, actor_as2=ACTOR)
-
+        g.user = self.make_user('user.com', has_hcard=True,
+                                actor_as2=ACTOR)
     def tearDown(self):
         self.request_context.pop()
         super().tearDown()
@@ -1205,7 +1208,7 @@ class ActivityPubUtilsTest(testutil.TestCase):
         }))
 
     def test_postprocess_as2_actor_attributedTo(self):
-        g.user = User(id='site')
+        g.user = Fake(id='site')
         self.assert_equals({
             'actor': {
                 'id': 'baj',
@@ -1255,7 +1258,7 @@ class ActivityPubUtilsTest(testutil.TestCase):
             ],
         }))
 
-    # TODO: make these generic and use FakeProtocol
+    # TODO: make these generic and use Fake
     @patch('requests.get')
     def test_load_http(self, mock_get):
         mock_get.return_value = AS2

tests/test_common.py

@@ -11,10 +11,10 @@ from flask_app import app
 import common
 from models import Object, User
 import protocol
-from . import testutil
+from .testutil import Fake, TestCase
 
 
-class CommonTest(testutil.TestCase):
+class CommonTest(TestCase):
     @classmethod
     def setUpClass(cls):
         with appengine_config.ndb_client.context():
@@ -24,7 +24,7 @@ class CommonTest(testutil.TestCase):
     def setUp(self):
         super().setUp()
         self.request_context.push()
-        g.user = User(id='site')
+        g.user = Fake(id='site')
 
     def tearDown(self):
         self.request_context.pop()

tests/test_convert.py

@@ -9,6 +9,7 @@ from models import Object
 from oauth_dropins.webutil.testutil import requests_response
 import requests
 
+import app
 from common import CONTENT_TYPE_HTML
 
 from .test_redirect import (

tests/test_follow.py

@@ -14,6 +14,8 @@ from flask_app import app
 import common
 from common import redirect_unwrap
 from models import Follower, Object, User
+from webmention import Webmention
+
 from . import testutil
 
 WEBFINGER = requests_response({

tests/test_models.py

@@ -14,27 +14,24 @@ from flask_app import app
 import common
 from models import AtpNode, Follower, Object, OBJECT_EXPIRE_AGE, User
 import protocol
-from . import testutil
 
 from .test_activitypub import ACTOR
+from .testutil import Fake, TestCase
 
-class UserTest(testutil.TestCase):
+
+class UserTest(TestCase):
 
     def setUp(self):
         super().setUp()
         self.request_context.push()
         g.user = self.make_user('y.z')
 
-        self.full_redir = requests_response(
-            status=302,
-            redirected_url='http://localhost/.well-known/webfinger?resource=acct:y.z@y.z')
-
     def tearDown(self):
         self.request_context.pop()
         super().tearDown()
 
     def test_get_or_create(self):
-        user = User.get_or_create('a.b')
+        user = Fake.get_or_create('a.b')
 
         assert user.mod
         assert user.public_exponent
@@ -49,15 +46,15 @@ class UserTest(testutil.TestCase):
         assert isinstance(p256_key, ECC.EccKey)
         self.assertEqual('NIST P-256', p256_key.curve)
 
-        same = User.get_or_create('a.b')
+        same = Fake.get_or_create('a.b')
         self.assertEqual(same, user)
 
     def test_get_or_create_use_instead(self):
-        user = User.get_or_create('a.b')
+        user = Fake.get_or_create('a.b')
         user.use_instead = g.user.key
         user.put()
 
-        self.assertEqual('y.z', User.get_or_create('a.b').key.id())
+        self.assertEqual('y.z', Fake.get_or_create('a.b').key.id())
 
     def test_href(self):
         href = g.user.href()
@@ -90,184 +87,8 @@ class UserTest(testutil.TestCase):
     def test_actor_id(self):
         self.assertEqual('http://localhost/y.z', g.user.actor_id())
 
-    def _test_verify(self, redirects, hcard, actor, redirects_error=None):
-        got = g.user.verify()
-        self.assertEqual(g.user.key, got.key)
 
-        with self.subTest(redirects=redirects, hcard=hcard, actor=actor,
-                          redirects_error=redirects_error):
-            self.assert_equals(redirects, bool(g.user.has_redirects))
-            self.assert_equals(hcard, bool(g.user.has_hcard))
-            if actor is None:
-                self.assertIsNone(g.user.actor_as2)
-            else:
-                got = {k: v for k, v in g.user.actor_as2.items()
-                       if k in actor}
-                self.assert_equals(actor, got)
-            self.assert_equals(redirects_error, g.user.redirects_error)
-
-    @mock.patch('requests.get')
-    def test_verify_neither(self, mock_get):
-        empty = requests_response('')
-        mock_get.side_effect = [empty, empty]
-        self._test_verify(False, False, None)
-
-    @mock.patch('requests.get')
-    def test_verify_redirect_strips_query_params(self, mock_get):
-        half_redir = requests_response(
-            status=302, redirected_url='http://localhost/.well-known/webfinger')
-        no_hcard = requests_response('<html><body></body></html>')
-        mock_get.side_effect = [half_redir, no_hcard]
-        self._test_verify(False, False, None, """\
-Current vs expected:<pre>- http://localhost/.well-known/webfinger
-+ https://fed.brid.gy/.well-known/webfinger?resource=acct:y.z@y.z</pre>""")
-
-    @mock.patch('requests.get')
-    def test_verify_multiple_redirects(self, mock_get):
-        two_redirs = requests_response(
-            status=302, redirected_url=[
-                'https://www.y.z/.well-known/webfinger?resource=acct:y.z@y.z',
-                'http://localhost/.well-known/webfinger?resource=acct:y.z@y.z',
-            ])
-        no_hcard = requests_response('<html><body></body></html>')
-        mock_get.side_effect = [two_redirs, no_hcard]
-        self._test_verify(True, False, None)
-
-    @mock.patch('requests.get')
-    def test_verify_redirect_404(self, mock_get):
-        redir_404 = requests_response(status=404, redirected_url='http://this/404s')
-        no_hcard = requests_response('<html><body></body></html>')
-        mock_get.side_effect = [redir_404, no_hcard]
-        self._test_verify(False, False, None, """\
-<pre>https://y.z/.well-known/webfinger?resource=acct:y.z@y.z
-  redirected to:
-http://this/404s
-  returned HTTP 404</pre>""")
-
-    @mock.patch('requests.get')
-    def test_verify_no_hcard(self, mock_get):
-        mock_get.side_effect = [
-            self.full_redir,
-            requests_response("""
-<body>
-<div class="h-entry">
-  <p class="e-content">foo bar</p>
-</div>
-</body>
-"""),
-        ]
-        self._test_verify(True, False, None)
-
-    @mock.patch('requests.get')
-    def test_verify_non_representative_hcard(self, mock_get):
-        bad_hcard = requests_response(
-            '<html><body><a class="h-card u-url" href="https://a.b/">acct:me@y.z</a></body></html>',
-            url='https://y.z/',
-        )
-        mock_get.side_effect = [self.full_redir, bad_hcard]
-        self._test_verify(True, False, None)
-
-    @mock.patch('requests.get')
-    def test_verify_both_work(self, mock_get):
-        hcard = requests_response("""
-<html><body class="h-card">
-  <a class="u-url p-name" href="/">me</a>
-  <a class="u-url" href="acct:myself@y.z">Masto</a>
-</body></html>""",
-            url='https://y.z/',
-        )
-        mock_get.side_effect = [self.full_redir, hcard]
-        self._test_verify(True, True, {
-            'type': 'Person',
-            'name': 'me',
-            'url': ['http://localhost/r/https://y.z/', 'acct:myself@y.z'],
-            'preferredUsername': 'y.z',
-        })
-
-    @mock.patch('requests.get')
-    def test_verify_www_redirect(self, mock_get):
-        www_user = self.make_user('www.y.z')
-
-        empty = requests_response('')
-        mock_get.side_effect = [
-            requests_response(status=302, redirected_url='https://www.y.z/'),
-            empty, empty,
-        ]
-
-        got = www_user.verify()
-        self.assertEqual('y.z', got.key.id())
-
-        root_user = User.get_by_id('y.z')
-        self.assertEqual(root_user.key, www_user.key.get().use_instead)
-        self.assertEqual(root_user.key, User.get_or_create('www.y.z').key)
-
-    @mock.patch('requests.get')
-    def test_verify_actor_rel_me_links(self, mock_get):
-        mock_get.side_effect = [
-            self.full_redir,
-            requests_response("""
-<body>
-<div class="h-card">
-<a class="u-url" rel="me" href="/about-me">Mrs. ☕ Foo</a>
-<a class="u-url" rel="me" href="/">should be ignored</a>
-<a class="u-url" rel="me" href="http://one" title="one title">
-  one text
-</a>
-<a class="u-url" rel="me" href="https://two" title=" two title "> </a>
-</div>
-</body>
-""", url='https://y.z/'),
-        ]
-        self._test_verify(True, True, {
-            'attachment': [{
-            'type': 'PropertyValue',
-            'name': 'Mrs. ☕ Foo',
-            'value': '<a rel="me" href="https://y.z/about-me">y.z/about-me</a>',
-        }, {
-            'type': 'PropertyValue',
-            'name': 'Web site',
-            'value': '<a rel="me" href="https://y.z/">y.z</a>',
-        }, {
-            'type': 'PropertyValue',
-            'name': 'one text',
-            'value': '<a rel="me" href="http://one">one</a>',
-        }, {
-            'type': 'PropertyValue',
-            'name': 'two title',
-            'value': '<a rel="me" href="https://two">two</a>',
-        }]})
-
-    @mock.patch('requests.get')
-    def test_verify_override_preferredUsername(self, mock_get):
-        mock_get.side_effect = [
-            self.full_redir,
-            requests_response("""
-<body>
-<a class="h-card u-url" rel="me" href="/about-me">
-  <span class="p-nickname">Nick</span>
-</a>
-</body>
-""", url='https://y.z/'),
-        ]
-        self._test_verify(True, True, {
-            # stays y.z despite user's username. since Mastodon queries Webfinger
-            # for preferredUsername@fed.brid.gy
-            # https://github.com/snarfed/bridgy-fed/issues/77#issuecomment-949955109
-            'preferredUsername': 'y.z',
-        })
-
-    def test_homepage(self):
-        self.assertEqual('https://y.z/', g.user.homepage)
-
-    def test_is_homepage(self):
-        for url in 'y.z', '//y.z', 'http://y.z', 'https://y.z':
-            self.assertTrue(g.user.is_homepage(url), url)
-
-        for url in None, '', 'y', 'z', 'z.z', 'ftp://y.z', 'http://y', '://y.z':
-            self.assertFalse(g.user.is_homepage(url), url)
-
-
-class ObjectTest(testutil.TestCase):
+class ObjectTest(TestCase):
     def setUp(self):
         super().setUp()
         self.request_context.push()
@@ -323,7 +144,7 @@ class ObjectTest(testutil.TestCase):
             self.assert_multiline_in(expected, obj.actor_link())
 
     def test_actor_link_user(self):
-        g.user = User(id='user.com', actor_as2={"name": "Alice"})
+        g.user = Fake(id='user.com', actor_as2={"name": "Alice"})
         obj = Object(id='x', source_protocol='ui', domains=['user.com'])
         self.assertIn(
             'href="/user/user.com"><img src="" class="profile"> Alice</a>',
@@ -369,7 +190,7 @@ class ObjectTest(testutil.TestCase):
         self.assertEqual(['user'], obj.labels)
 
 
-class FollowerTest(testutil.TestCase):
+class FollowerTest(TestCase):
 
     def setUp(self):
         super().setUp()
@@ -391,7 +212,7 @@ class FollowerTest(testutil.TestCase):
         self.assertEqual(ACTOR, self.outbound.to_as2())
 
 
-class AtpNodeTest(testutil.TestCase):
+class AtpNodeTest(TestCase):
 
     def test_create(self):
         AtpNode.create(ACTOR_PROFILE_BSKY)

tests/test_pages.py

@@ -18,15 +18,17 @@ from oauth_dropins.webutil.testutil import requests_response
 from flask_app import app
 import common
 from models import Object, Follower, User
-from . import testutil
+from webmention import Webmention
+
 from .test_webmention import ACTOR_AS2, ACTOR_HTML, ACTOR_MF2, REPOST_AS2
+from .testutil import Fake, TestCase
 
 
 def contents(activities):
     return [(a.get('object') or a)['content'] for a in activities]
 
 
-class PagesTest(testutil.TestCase):
+class PagesTest(TestCase):
     EXPECTED = contents([COMMENT, NOTE])
 
     def setUp(self):
@@ -109,7 +111,7 @@ class PagesTest(testutil.TestCase):
         self.assert_equals(302, got.status_code)
         self.assert_equals('/user/user.com', got.headers['Location'])
 
-        user = User.get_by_id('user.com')
+        user = Webmention.get_by_id('user.com')
         self.assertTrue(user.has_hcard)
         self.assertEqual('Person', user.actor_as2['type'])
         self.assertEqual('http://localhost/user.com', user.actor_as2['id'])
@@ -118,7 +120,7 @@ class PagesTest(testutil.TestCase):
         got = self.client.post('/web-site', data={'url': '!!!'})
         self.assert_equals(200, got.status_code)
         self.assertEqual(['No domain found in !!!'], get_flashed_messages())
-        self.assertEqual(1, User.query().count())
+        self.assertEqual(1, Webmention.query().count())
 
     @patch('requests.get')
     def test_check_web_site_fetch_fails(self, mock_get):

tests/test_protocol.py

@@ -8,12 +8,11 @@ import requests
 import protocol
 from protocol import Protocol
 from flask_app import app
-from models import Follower, Object, User
+from models import Follower, Object, PROTOCOLS, User
 from webmention import Webmention
 
 from .test_activitypub import ACTOR, REPLY
-from . import testutil
-from .testutil import FakeProtocol
+from .testutil import Fake, TestCase
 
 REPLY = {
     **REPLY,
@@ -25,7 +24,7 @@ REPLY = {
 }
 
 
-class ProtocolTest(testutil.TestCase):
+class ProtocolTest(TestCase):
 
     def setUp(self):
         super().setUp()
@@ -38,8 +37,8 @@ class ProtocolTest(testutil.TestCase):
         super().tearDown()
 
     def test_protocols_global(self):
-        self.assertEqual(FakeProtocol, protocol.protocols['fake'])
-        self.assertEqual(Webmention, protocol.protocols['webmention'])
+        self.assertEqual(Fake, PROTOCOLS['fake'])
+        self.assertEqual(Webmention, PROTOCOLS['webmention'])
 
     @patch('requests.get')
     def test_receive_reply_not_feed_not_notification(self, mock_get):
@@ -65,58 +64,58 @@ class ProtocolTest(testutil.TestCase):
                            )
 
     def test_load(self):
-        FakeProtocol.objects['foo'] = {'x': 'y'}
+        Fake.objects['foo'] = {'x': 'y'}
 
-        loaded = FakeProtocol.load('foo')
+        loaded = Fake.load('foo')
         self.assert_equals({'x': 'y'}, loaded.our_as1)
         self.assertFalse(loaded.changed)
         self.assertTrue(loaded.new)
 
         self.assertIsNotNone(Object.get_by_id('foo'))
-        self.assertEqual(['foo'], FakeProtocol.fetched)
+        self.assertEqual(['foo'], Fake.fetched)
 
     def test_load_already_stored(self):
         stored = Object(id='foo', our_as1={'x': 'y'})
         stored.put()
 
-        loaded = FakeProtocol.load('foo')
+        loaded = Fake.load('foo')
         self.assert_equals({'x': 'y'}, loaded.our_as1)
         self.assertFalse(loaded.changed)
         self.assertFalse(loaded.new)
 
-        self.assertEqual([], FakeProtocol.fetched)
+        self.assertEqual([], Fake.fetched)
 
     @patch('requests.get')
     def test_load_empty_deleted(self, mock_get):
         stored = Object(id='foo', deleted=True)
         stored.put()
 
-        loaded = FakeProtocol.load('foo')
+        loaded = Fake.load('foo')
         self.assert_entities_equal(stored, loaded)
         self.assertFalse(loaded.changed)
         self.assertFalse(loaded.new)
 
-        self.assertEqual([], FakeProtocol.fetched)
+        self.assertEqual([], Fake.fetched)
 
     @patch('requests.get')
     def test_load_refresh_unchanged(self, mock_get):
         obj = Object(id='foo', our_as1={'x': 'stored'})
         obj.put()
-        FakeProtocol.objects['foo'] = {'x': 'stored'}
+        Fake.objects['foo'] = {'x': 'stored'}
 
-        loaded = FakeProtocol.load('foo', refresh=True)
+        loaded = Fake.load('foo', refresh=True)
         self.assert_entities_equal(obj, loaded)
         self.assertFalse(obj.changed)
         self.assertFalse(obj.new)
-        self.assertEqual(['foo'], FakeProtocol.fetched)
+        self.assertEqual(['foo'], Fake.fetched)
 
     @patch('requests.get')
     def test_load_refresh_changed(self, mock_get):
         Object(id='foo', our_as1={'content': 'stored'}).put()
-        FakeProtocol.objects['foo'] = {'content': 'new'}
+        Fake.objects['foo'] = {'content': 'new'}
 
-        loaded = FakeProtocol.load('foo', refresh=True)
+        loaded = Fake.load('foo', refresh=True)
         self.assert_equals({'content': 'new'}, loaded.our_as1)
         self.assertTrue(loaded.changed)
         self.assertFalse(loaded.new)
-        self.assertEqual(['foo'], FakeProtocol.fetched)
+        self.assertEqual(['foo'], Fake.fetched)

tests/test_webmention.py

@@ -162,7 +162,13 @@ DELETE_AS2 = {
 class WebmentionTest(testutil.TestCase):
     def setUp(self):
         super().setUp()
-        self.user = self.make_user('user.com')
+        g.user = self.user = self.make_user('user.com')
+
+        self.request_context.push()
+        self.full_redir = requests_response(
+            status=302,
+            redirected_url='http://localhost/.well-known/webfinger?resource=acct:user.com@user.com')
+
         self.toot_html = requests_response("""\
 <html>
 <meta>
@@ -1337,10 +1343,177 @@ class WebmentionTest(testutil.TestCase):
                            labels=['user', 'activity'],
                            )
 
+    def _test_verify(self, redirects, hcard, actor, redirects_error=None):
+        got = self.user.verify()
+        self.assertEqual(self.user.key, got.key)
+
+        with self.subTest(redirects=redirects, hcard=hcard, actor=actor,
+                          redirects_error=redirects_error):
+            self.assert_equals(redirects, bool(self.user.has_redirects))
+            self.assert_equals(hcard, bool(self.user.has_hcard))
+            if actor is None:
+                self.assertIsNone(self.user.actor_as2)
+            else:
+                got = {k: v for k, v in self.user.actor_as2.items()
+                       if k in actor}
+                self.assert_equals(actor, got)
+            self.assert_equals(redirects_error, self.user.redirects_error)
+
+    def test_verify_neither(self, mock_get, _):
+        empty = requests_response('')
+        mock_get.side_effect = [empty, empty]
+        self._test_verify(False, False, None)
+
+    def test_verify_redirect_strips_query_params(self, mock_get, _):
+        half_redir = requests_response(
+            status=302, redirected_url='http://localhost/.well-known/webfinger')
+        no_hcard = requests_response('<html><body></body></html>')
+        mock_get.side_effect = [half_redir, no_hcard]
+        self._test_verify(False, False, None, """\
+Current vs expected:<pre>- http://localhost/.well-known/webfinger
++ https://fed.brid.gy/.well-known/webfinger?resource=acct:user.com@user.com</pre>""")
+
+    def test_verify_multiple_redirects(self, mock_get, _):
+        two_redirs = requests_response(
+            status=302, redirected_url=[
+                'https://www.user.com/.well-known/webfinger?resource=acct:user.com@user.com',
+                'http://localhost/.well-known/webfinger?resource=acct:user.com@user.com',
+            ])
+        no_hcard = requests_response('<html><body></body></html>')
+        mock_get.side_effect = [two_redirs, no_hcard]
+        self._test_verify(True, False, None)
+
+    def test_verify_redirect_404(self, mock_get, _):
+        redir_404 = requests_response(status=404, redirected_url='http://this/404s')
+        no_hcard = requests_response('<html><body></body></html>')
+        mock_get.side_effect = [redir_404, no_hcard]
+        self._test_verify(False, False, None, """\
+<pre>https://user.com/.well-known/webfinger?resource=acct:user.com@user.com
+  redirected to:
+http://this/404s
+  returned HTTP 404</pre>""")
+
+    def test_verify_no_hcard(self, mock_get, _):
+        mock_get.side_effect = [
+            self.full_redir,
+            requests_response("""
+<body>
+<div class="h-entry">
+  <p class="e-content">foo bar</p>
+</div>
+</body>
+"""),
+        ]
+        self._test_verify(True, False, None)
+
+    def test_verify_non_representative_hcard(self, mock_get, _):
+        bad_hcard = requests_response(
+            '<html><body><a class="h-card u-url" href="https://a.b/">acct:me@user.com</a></body></html>',
+            url='https://user.com/',
+        )
+        mock_get.side_effect = [self.full_redir, bad_hcard]
+        self._test_verify(True, False, None)
+
+    def test_verify_both_work(self, mock_get, _):
+        hcard = requests_response("""
+<html><body class="h-card">
+  <a class="u-url p-name" href="/">me</a>
+  <a class="u-url" href="acct:myself@user.com">Masto</a>
+</body></html>""",
+            url='https://user.com/',
+        )
+        mock_get.side_effect = [self.full_redir, hcard]
+        self._test_verify(True, True, {
+            'type': 'Person',
+            'name': 'me',
+            'url': ['http://localhost/r/https://user.com/', 'acct:myself@user.com'],
+            'preferredUsername': 'user.com',
+        })
+
+    def test_verify_www_redirect(self, mock_get, _):
+        www_user = self.make_user('www.user.com')
+
+        empty = requests_response('')
+        mock_get.side_effect = [
+            requests_response(status=302, redirected_url='https://www.user.com/'),
+            empty, empty,
+        ]
+
+        got = www_user.verify()
+        self.assertEqual('user.com', got.key.id())
+
+        root_user = Webmention.get_by_id('user.com')
+        self.assertEqual(root_user.key, www_user.key.get().use_instead)
+        self.assertEqual(root_user.key, Webmention.get_or_create('www.user.com').key)
+
+    def test_verify_actor_rel_me_links(self, mock_get, _):
+        mock_get.side_effect = [
+            self.full_redir,
+            requests_response("""
+<body>
+<div class="h-card">
+<a class="u-url" rel="me" href="/about-me">Mrs. ☕ Foo</a>
+<a class="u-url" rel="me" href="/">should be ignored</a>
+<a class="u-url" rel="me" href="http://one" title="one title">
+  one text
+</a>
+<a class="u-url" rel="me" href="https://two" title=" two title "> </a>
+</div>
+</body>
+""", url='https://user.com/'),
+        ]
+        self._test_verify(True, True, {
+            'attachment': [{
+            'type': 'PropertyValue',
+            'name': 'Mrs. ☕ Foo',
+            'value': '<a rel="me" href="https://user.com/about-me">user.com/about-me</a>',
+        }, {
+            'type': 'PropertyValue',
+            'name': 'Web site',
+            'value': '<a rel="me" href="https://user.com/">user.com</a>',
+        }, {
+            'type': 'PropertyValue',
+            'name': 'one text',
+            'value': '<a rel="me" href="http://one">one</a>',
+        }, {
+            'type': 'PropertyValue',
+            'name': 'two title',
+            'value': '<a rel="me" href="https://two">two</a>',
+        }]})
+
+    def test_verify_override_preferredUsername(self, mock_get, _):
+        mock_get.side_effect = [
+            self.full_redir,
+            requests_response("""
+<body>
+<a class="h-card u-url" rel="me" href="/about-me">
+  <span class="p-nickname">Nick</span>
+</a>
+</body>
+""", url='https://user.com/'),
+        ]
+        self._test_verify(True, True, {
+            # stays y.z despite user's username. since Mastodon queries Webfinger
+            # for preferredUsername@fed.brid.gy
+            # https://github.com/snarfed/bridgy-fed/issues/77#issuecomment-949955109
+            'preferredUsername': 'user.com',
+        })
+
+    def test_homepage(self, _, __):
+        self.assertEqual('https://user.com/', self.user.homepage)
+
+    def test_is_homepage(self, _, __):
+        for url in 'user.com', '//user.com', 'http://user.com', 'https://user.com':
+            self.assertTrue(self.user.is_homepage(url), url)
+
+        for url in (None, '', 'user', 'com', 'com.user', 'ftp://user.com',
+                    'https://user', '://user.com'):
+            self.assertFalse(self.user.is_homepage(url), url)
+
 
 @mock.patch('requests.post')
 @mock.patch('requests.get')
-class WebmentionUtilTest(testutil.TestCase):
+class WebmentionProtocolTest(testutil.TestCase):
 
     def setUp(self):
         super().setUp()

tests/testutil.py

@@ -25,23 +25,17 @@ import requests
 # load all Flask handlers
 import app
 from flask_app import app, cache, init_globals
-import activitypub, common
+import activitypub
+import common
+import models
 from models import Object, PROTOCOLS, Target, User
 import protocol
-
+from webmention import Webmention
 
 logger = logging.getLogger(__name__)
 
-# used in TestCase.make_user() to reuse keys across Users since they're
-# expensive to generate
-requests.post(f'http://{ndb_client.host}/reset')
-with ndb_client.context():
-    global_user = User.get_or_create('user.com')
 
-
-Object.source_protocol = ndb.StringProperty(choices=PROTOCOLS + ('fake',))
-
-class FakeProtocol(protocol.Protocol):
+class Fake(User, protocol.Protocol):
     LABEL = 'fake'
 
     # maps string ids to dict AS1 objects. send adds objects here, fetch
@@ -56,14 +50,14 @@ class FakeProtocol(protocol.Protocol):
 
     @classmethod
     def send(cls, obj, url, log_data=True):
-        logger.info(f'FakeProtocol.send {url}')
+        logger.info(f'Fake.send {url}')
         cls.sent.append((obj, url))
         cls.objects[obj.key.id()] = obj
 
     @classmethod
     def fetch(cls, obj):
         id = obj.key.id()
-        logger.info(f'FakeProtocol.load {id}')
+        logger.info(f'Fake.load {id}')
         cls.fetched.append(id)
 
         if id in cls.objects:
@@ -74,11 +68,21 @@ class FakeProtocol(protocol.Protocol):
 
     @classmethod
     def serve(cls, obj):
-        logger.info(f'FakeProtocol.load {obj.key.id()}')
-        return (f'FakeProtocol object {obj.key.id()}',
+        logger.info(f'Fake.load {obj.key.id()}')
+        return (f'Fake object {obj.key.id()}',
                 {'Accept': 'fake/protocol'})
 
 
+# used in TestCase.make_user() to reuse keys across Users since they're
+# expensive to generate
+requests.post(f'http://{ndb_client.host}/reset')
+with ndb_client.context():
+    global_user = Fake.get_or_create('user.com')
+
+
+models.reset_protocol_properties()
+
+
 class TestCase(unittest.TestCase, testutil.Asserts):
     maxDiff = None
 
@@ -91,9 +95,9 @@ class TestCase(unittest.TestCase, testutil.Asserts):
         protocol.objects_cache.clear()
         common.webmention_discover.cache.clear()
 
-        FakeProtocol.objects = {}
-        FakeProtocol.sent = []
-        FakeProtocol.fetched = []
+        Fake.objects = {}
+        Fake.sent = []
+        Fake.fetched = []
 
         # make random test data deterministic
         arroba.util._clockid = 17
@@ -122,15 +126,16 @@ class TestCase(unittest.TestCase, testutil.Asserts):
         self.client.__exit__(None, None, None)
         super().tearDown()
 
+    # TODO(#512): switch default to Fake, start using that more
     @staticmethod
-    def make_user(domain, **kwargs):
+    def make_user(domain, cls=Webmention, **kwargs):
         """Reuse RSA key across Users because generating it is expensive."""
-        user = User(id=domain,
-                    mod=global_user.mod,
-                    public_exponent=global_user.public_exponent,
-                    private_exponent=global_user.private_exponent,
-                    p256_key=global_user.p256_key,
-                    **kwargs)
+        user = cls(id=domain,
+                   mod=global_user.mod,
+                   public_exponent=global_user.public_exponent,
+                   private_exponent=global_user.private_exponent,
+                   p256_key=global_user.p256_key,
+                   **kwargs)
         user.put()
         return user
 

ui.py

@@ -2,8 +2,9 @@
 
 Needed for serving /convert/ui/webmention/... requests.
 """
+from models import User
 from protocol import Protocol
 
 
-class UIProtocol(Protocol):
+class UIProtocol(User, Protocol):
     LABEL = 'ui'

webfinger.py

@@ -45,7 +45,7 @@ class Actor(flask_util.XrdOrJrd):
         if domain.split('.')[-1] in NON_TLDS:
             error(f"{domain} doesn't look like a domain", status=404)
 
-        g.user = User.get_by_id(domain)
+        g.user = Webmention.get_by_id(domain)
         if g.user:
             actor = g.user.to_as1() or {}
             homepage = g.user.homepage

webmention.py

@@ -1,7 +1,7 @@
 """Handles inbound webmentions."""
+import difflib
 import logging
-import urllib.parse
-from urllib.parse import urlencode, urlparse
+from urllib.parse import urlencode, urljoin, urlparse
 
 import feedparser
 from flask import g, redirect, request
@@ -14,16 +14,15 @@ from oauth_dropins.webutil.appengine_config import tasks_client
 from oauth_dropins.webutil.appengine_info import APP_ID
 from oauth_dropins.webutil.flask_util import error, flash
 from oauth_dropins.webutil.util import json_dumps, json_loads
-from oauth_dropins.webutil import webmention
+import oauth_dropins.webutil.webmention as webutil_webmention
 from requests import HTTPError, RequestException, URLRequired
-from werkzeug.exceptions import BadGateway, BadRequest, HTTPException
+from werkzeug.exceptions import BadGateway, BadRequest, HTTPException, NotFound
 
-from activitypub import ActivityPub
+import activitypub
 from flask_app import app
 import common
-from models import Follower, Object, Target, User
-import models
-from protocol import Protocol, protocols
+from models import Follower, Object, PROTOCOLS, Target, User
+from protocol import Protocol
 
 logger = logging.getLogger(__name__)
 
@@ -32,11 +31,85 @@ TASKS_LOCATION = 'us-central1'
 
 CHAR_AFTER_SPACE = chr(ord(' ') + 1)
 
+# https://github.com/snarfed/bridgy-fed/issues/314
+WWW_DOMAINS = frozenset((
+    'www.jvt.me',
+))
 
-class Webmention(Protocol):
-    """Webmention protocol implementation."""
+
+class Webmention(User, Protocol):
+    """Webmention user and protocol implementation.
+
+    The key name is the domain.
+    """
     LABEL = 'webmention'
 
+    @classmethod
+    def _get_kind(cls):
+        return 'MagicKey'
+
+    def verify(self):
+        """Fetches site a couple ways to check for redirects and h-card.
+
+
+        Returns: :class:`Webmention` that was verified. May be different than
+          self! eg if self's domain started with www and we switch to the root
+          domain.
+        """
+        domain = self.key.id()
+        logger.info(f'Verifying {domain}')
+
+        if domain.startswith('www.') and domain not in WWW_DOMAINS:
+            # if root domain redirects to www, use root domain instead
+            # https://github.com/snarfed/bridgy-fed/issues/314
+            root = domain.removeprefix("www.")
+            root_site = f'https://{root}/'
+            try:
+                resp = util.requests_get(root_site, gateway=False)
+                if resp.ok and self.is_homepage(resp.url):
+                    logger.info(f'{root_site} redirects to {resp.url} ; using {root} instead')
+                    root_user = Webmention.get_or_create(root)
+                    self.use_instead = root_user.key
+                    self.put()
+                    return root_user.verify()
+            except RequestException:
+                pass
+
+        # check webfinger redirect
+        path = f'/.well-known/webfinger?resource=acct:{domain}@{domain}'
+        self.has_redirects = False
+        self.redirects_error = None
+        try:
+            url = urljoin(self.homepage, path)
+            resp = util.requests_get(url, gateway=False)
+            domain_urls = ([f'https://{domain}/' for domain in common.DOMAINS] +
+                           [common.host_url()])
+            expected = [urljoin(url, path) for url in domain_urls]
+            if resp.ok:
+                if resp.url in expected:
+                    self.has_redirects = True
+                elif resp.url:
+                    diff = '\n'.join(difflib.Differ().compare([resp.url], [expected[0]]))
+                    self.redirects_error = f'Current vs expected:<pre>{diff}</pre>'
+            else:
+                lines = [url, f'  returned HTTP {resp.status_code}']
+                if resp.url != url:
+                    lines[1:1] = ['  redirected to:', resp.url]
+                self.redirects_error = '<pre>' + '\n'.join(lines) + '</pre>'
+        except RequestException:
+            pass
+
+        # check home page
+        try:
+            obj = Webmention.load(self.homepage, gateway=True)
+            self.actor_as2 = activitypub.postprocess_as2(as2.from_as1(obj.as1))
+            self.has_hcard = True
+        except (BadRequest, NotFound):
+            self.actor_as2 = None
+            self.has_hcard = False
+
+        return self
+
     @classmethod
     def send(cls, obj, url):
         """Sends a webmention to a given target URL.
@@ -48,7 +121,7 @@ class Webmention(Protocol):
 
         endpoint = common.webmention_discover(url).endpoint
         if endpoint:
-            webmention.send(endpoint, source_url, url)
+            webutil_webmention.send(endpoint, source_url, url)
             return True
 
     @classmethod
@@ -130,7 +203,7 @@ class Webmention(Protocol):
         """Serves an :class:`Object` as HTML."""
         obj_as1 = obj.as1
 
-        from_proto = protocols.get(obj.source_protocol)
+        from_proto = PROTOCOLS.get(obj.source_protocol)
         if from_proto:
             # fill in author/actor if available
             for field in 'author', 'actor':
@@ -167,7 +240,7 @@ def webmention_external():
         error(f'Bad URL {source}')
 
     domain = util.domain_from_link(source, minimize=False)
-    g.user = User.get_by_id(domain)
+    g.user = Webmention.get_by_id(domain)
     if not g.user:
         error(f'No user found for domain {domain}')
 
@@ -215,7 +288,7 @@ def webmention_task():
     domain = util.domain_from_link(source, minimize=False)
     logger.info(f'webmention from {domain}')
 
-    g.user = User.get_by_id(domain)
+    g.user = Webmention.get_by_id(domain)
     if not g.user:
         error(f'No user found for domain {domain}', status=304)
 
@@ -373,7 +446,7 @@ def webmention_task():
         obj.target_as2 = target_as2
 
         try:
-            last = ActivityPub.send(obj, inbox, log_data=log_data)
+            last = activitypub.ActivityPub.send(obj, inbox, log_data=log_data)
             obj.delivered.append(target)
             last_success = last
         except BaseException as e:
@@ -429,7 +502,7 @@ def _activitypub_targets(obj):
         # fetch target page as AS2 object
         try:
             # TODO: make this generic across protocols
-            target_stored = ActivityPub.load(target)
+            target_stored = activitypub.ActivityPub.load(target)
             target_obj = target_stored.as2 or as2.from_as1(target_stored.as1)
         except (HTTPError, BadGateway) as e:
             resp = getattr(e, 'requests_response', None)
@@ -455,7 +528,7 @@ def _activitypub_targets(obj):
         if not inbox_url:
             # fetch actor as AS object
             # TODO: make this generic across protocols
-            actor_obj = ActivityPub.load(actor)
+            actor_obj = activitypub.ActivityPub.load(actor)
             actor = actor_obj.as2 or as2.from_as1(actor_obj.as1)
             inbox_url = actor.get('inbox')
 
@@ -464,7 +537,7 @@ def _activitypub_targets(obj):
             logger.error('Target actor has no inbox')
             continue
 
-        inbox_url = urllib.parse.urljoin(target, inbox_url)
+        inbox_url = urljoin(target, inbox_url)
         inboxes_to_targets[inbox_url] = target_obj
 
     if not targets or verb == 'share':

xrpc_actor.py

@@ -10,6 +10,7 @@ from oauth_dropins.webutil import util
 
 from flask_app import xrpc_server
 from models import User
+from webmention import Webmention
 
 logger = logging.getLogger(__name__)
 
@@ -24,7 +25,7 @@ def getProfile(input, actor=None):
     if not actor or not re.match(util.DOMAIN_RE, actor):
         raise ValueError(f'{actor} is not a domain')
 
-    g.user = User.get_by_id(actor)
+    g.user = Webmention.get_by_id(actor)
     if not g.user:
         raise ValueError(f'User {actor} not found')
     elif not g.user.actor_as2:

xrpc_feed.py

@@ -10,6 +10,7 @@ from oauth_dropins.webutil import util
 
 from flask_app import xrpc_server
 from models import Object, PAGE_SIZE, User
+from webmention import Webmention
 
 logger = logging.getLogger(__name__)
 
@@ -22,7 +23,7 @@ def getAuthorFeed(input, author=None, limit=None, before=None):
     if not author or not re.match(util.DOMAIN_RE, author):
         raise ValueError(f'{author} is not a domain')
 
-    g.user = User.get_by_id(author)
+    g.user = Webmention.get_by_id(author)
     if not g.user:
         raise ValueError(f'User {author} not found')
     elif not g.user.actor_as2:

xrpc_graph.py

@@ -8,6 +8,7 @@ from oauth_dropins.webutil import util
 from flask_app import xrpc_server
 import common
 from models import Follower, User
+from webmention import Webmention
 
 logger = logging.getLogger(__name__)
 
@@ -25,7 +26,7 @@ def get_followers(query_prop, output_field, user=None, limit=50, before=None):
     # TODO: what is user?
     if not user or not re.match(util.DOMAIN_RE, user):
         raise ValueError(f'{user} is not a domain')
-    elif not User.get_by_id(user):
+    elif not Webmention.get_by_id(user):
         raise ValueError(f'Unknown user {user}')
 
     collection = 'followers' if output_field == 'followers' else 'following'