Web.fetch: switch back to raising instead of returning empty Object

raise new custom NoMicroformats exception

activitypub.py

@@ -546,7 +546,6 @@ def postprocess_as2_actor(actor, wrap=True):
 @flask_util.cached(cache, CACHE_TIME)
 def actor(protocol, domain):
     """Serves a user's AS2 actor from the datastore."""
-    # TODO(#512): fetch from web site if we don't already have a User
     tld = domain.split('.')[-1]
     if tld in TLD_BLOCKLIST:
         error('', status=404)
@@ -554,8 +553,12 @@ def actor(protocol, domain):
     cls = PROTOCOLS[protocol]
     g.user = cls.get_by_id(domain)
     if not g.user:
-        obj = cls.load(f'https://{domain}/', gateway=True)
+        try:
-        g.user = cls.get_or_create(id=domain, actor_as2=as2.from_as1(obj.as1))
+            obj = cls.load(f'https://{domain}/', gateway=True)
+            actor_as2 = as2.from_as1(obj.as1)
+        except web.NoMicroformats as e:
+            actor_as2 = {}
+        g.user = cls.get_or_create(id=domain, actor_as2=actor_as2)
 
     # TODO: unify with common.actor()
     actor = postprocess_as2(g.user.actor_as2 or {})

models.py

@@ -122,7 +122,7 @@ class User(StringIdModel, metaclass=ProtocolUserMeta):
         assert cls != User
         user = cls.get_by_id(id)
         if user:
-            # override direct if it's set
+            # override direct from False => True if set
             direct = kwargs.get('direct')
             if direct and not user.direct:
                 logger.info(f'Setting {user.key} direct={direct}')

tests/test_web.py

@@ -1657,8 +1657,8 @@ class WebProtocolTest(testutil.TestCase):
         mock_get.return_value = REPOST
 
         obj = Object(id='https://user.com/')
-        Web.fetch(obj)
+        with self.assertRaises(BadRequest):
-        self.assertIsNone(obj.mf2)
+            Web.fetch(obj)
 
     def test_fetch_user_homepage_non_representative_hcard(self, mock_get, __):
         mock_get.return_value = requests_response(
@@ -1666,8 +1666,8 @@ class WebProtocolTest(testutil.TestCase):
             content_type=CONTENT_TYPE_HTML)
 
         obj = Object(id='https://user.com/')
-        Web.fetch(obj)
+        with self.assertRaises(BadRequest):
-        self.assertIsNone(obj.mf2)
+            Web.fetch(obj)
 
     def test_fetch_user_homepage_fail(self, mock_get, __):
         mock_get.return_value = requests_response('', status=500)

web.py

@@ -38,6 +38,11 @@ WWW_DOMAINS = frozenset((
 ))
 
 
+class NoMicroformats(BadRequest):
+    """Raised by :meth:`Web.fetch` when a page has no microformats2."""
+    pass
+
+
 class Web(User, Protocol):
     """Web user and webmention protocol implementation.
 
@@ -132,11 +137,11 @@ class Web(User, Protocol):
             pass
 
         # check home page
-        obj = Web.load(self.web_url(), gateway=True)
+        try:
-        if obj.mf2:
+            obj = Web.load(self.web_url(), gateway=True)
             self.actor_as2 = activitypub.postprocess_as2(as2.from_as1(obj.as1))
             self.has_hcard = True
-        else:
+        except (BadRequest, NotFound, NoMicroformats):
             self.actor_as2 = None
             self.has_hcard = False
 
@@ -193,7 +198,9 @@ class Web(User, Protocol):
             entry = mf2util.representative_hcard(parsed, parsed['url'])
             logger.info(f'Representative h-card: {json_dumps(entry, indent=2)}')
             if not entry:
-                return obj
+                msg = f"Couldn't find a representative h-card (http://microformats.org/wiki/representative-hcard-parsing) on {parsed['url']}"
+                logging.info(msg)
+                raise NoMicroformats(msg)
         else:
             entry = mf2util.find_first_entry(parsed, ['h-entry'])
             if not entry: