add /responses/[DOMAIN] user pages and Response.domain property

common.py

@@ -189,13 +189,13 @@ def send_webmentions(activity_wrapped, proxy=None, **response_props):
     # send webmentions and store Responses
     errors = []  # stores (code, body) tuples
     for target in targets:
-        if (util.domain_from_link(target, minimize=False) ==
-            util.domain_from_link(source, minimize=False)):
+        domain = util.domain_from_link(target, minimize=False)
+        if (domain == util.domain_from_link(source, minimize=False)):
             logger.info(f'Skipping same-domain webmention from {source} to {target}')
             continue
 
         response = Response(source=source, target=target, direction='in',
-                            **response_props)
+                            domain=domain, **response_props)
         response.put()
         wm_source = (response.proxy_url()
                      if verb in ('follow', 'like', 'share') or proxy

index.yaml

@@ -11,3 +11,10 @@ indexes:
   - name: status
   - name: updated
     direction: desc
+
+- kind: Response
+  properties:
+  - name: domain
+  - name: status
+  - name: updated
+    direction: desc

logs.py

@@ -8,16 +8,21 @@ from oauth_dropins.webutil import flask_util, logs, util
 from oauth_dropins.webutil.flask_util import error
 
 from app import app, cache
+import common
 from models import Response
 
 
 @app.get('/responses')
-def responses():
+@app.get(f'/responses/<regex("{common.DOMAIN_RE}"):domain>')
+def responses(domain=None):
     """Renders recent Responses, with links to logs."""
     query = Response.query()\
         .filter(Response.status.IN(('new', 'complete', 'error')))\
         .order(-Response.updated)
 
+    if domain:
+        query = query.filter(Response.domain == domain)
+
     # if there's a paging param (responses_before or responses_after), update
     # query with it
     # TODO: unify this with Bridgy's user page

models.py

@@ -71,6 +71,7 @@ class Response(StringIdModel):
     PROTOCOLS = ('activitypub', 'ostatus')
     DIRECTIONS = ('out', 'in')
 
+    domain = ndb.StringProperty()
     status = ndb.StringProperty(choices=STATUSES, default='new')
     protocol = ndb.StringProperty(choices=PROTOCOLS)
     direction = ndb.StringProperty(choices=DIRECTIONS)

tests/test_activitypub.py

@@ -238,6 +238,7 @@ class ActivityPubTest(testutil.TestCase):
         )
 
         resp = Response.get_by_id('http://this/reply http://orig/post')
+        self.assertEqual('orig', resp.domain)
         self.assertEqual('in', resp.direction)
         self.assertEqual('activitypub', resp.protocol)
         self.assertEqual('complete', resp.status)
@@ -286,6 +287,7 @@ class ActivityPubTest(testutil.TestCase):
             )
 
             resp = Response.get_by_id('http://this/mention http://target/')
+            self.assertEqual('target', resp.domain)
             self.assertEqual('in', resp.direction)
             self.assertEqual('activitypub', resp.protocol)
             self.assertEqual('complete', resp.status)
@@ -318,6 +320,7 @@ class ActivityPubTest(testutil.TestCase):
         }, kwargs['data'])
 
         resp = Response.get_by_id('http://this/like__ok http://orig/post')
+        self.assertEqual('orig', resp.domain)
         self.assertEqual('in', resp.direction)
         self.assertEqual('activitypub', resp.protocol)
         self.assertEqual('complete', resp.status)
@@ -356,6 +359,7 @@ class ActivityPubTest(testutil.TestCase):
         }, kwargs['data'])
 
         resp = Response.get_by_id('https://mastodon.social/6d1a https://www.realize.be/')
+        self.assertEqual('www.realize.be', resp.domain)
         self.assertEqual('in', resp.direction)
         self.assertEqual('activitypub', resp.protocol)
         self.assertEqual('complete', resp.status)
@@ -440,6 +444,7 @@ class ActivityPubTest(testutil.TestCase):
         self.assertEqual(200, got.status_code)
 
         resp = Response.get_by_id('http://this/like__ok http://orig/post')
+        self.assertEqual('orig', resp.domain)
         self.assertEqual('in', resp.direction)
         self.assertEqual('activitypub', resp.protocol)
         self.assertEqual('ignored', resp.status)

tests/test_salmon.py

@@ -91,6 +91,7 @@ class SalmonTest(testutil.TestCase):
 
         # check stored response
         resp = Response.get_by_id('https://my/reply http://orig/post')
+        self.assertEqual('orig', resp.domain)
         self.assertEqual('in', resp.direction)
         self.assertEqual('ostatus', resp.protocol)
         self.assertEqual('complete', resp.status)
@@ -125,6 +126,7 @@ class SalmonTest(testutil.TestCase):
 
         # check stored response
         resp = Response.get_by_id('https://my/like http://orig/post')
+        self.assertEqual('orig', resp.domain)
         self.assertEqual('in', resp.direction)
         self.assertEqual('ostatus', resp.protocol)
         self.assertEqual('complete', resp.status)

tests/test_webmention.py

@@ -394,6 +394,7 @@ class WebmentionTest(testutil.TestCase):
         self.assertEqual(self.key.private_pem(), rsa_key.exportKey())
 
         resp = Response.get_by_id('http://a/reply http://orig/as2')
+        self.assertEqual('a', resp.domain)
         self.assertEqual('out', resp.direction)
         self.assertEqual('activitypub', resp.protocol)
         self.assertEqual('complete', resp.status)
@@ -496,6 +497,7 @@ class WebmentionTest(testutil.TestCase):
         self.assertEqual(self.key.private_pem(), rsa_key.exportKey())
 
         resp = Response.get_by_id('http://a/repost http://orig/as2')
+        self.assertEqual('a', resp.domain)
         self.assertEqual('out', resp.direction)
         self.assertEqual('activitypub', resp.protocol)
         self.assertEqual('complete', resp.status)
@@ -591,13 +593,13 @@ class WebmentionTest(testutil.TestCase):
         mock_get.side_effect = [self.create, self.actor]
         mock_post.return_value = requests_response('abc xyz')
 
-        Response(id='http://orig/post https://skipped/inbox', status='complete',
-                 source_mf2=json_dumps(self.create_mf2)).put()
+        Response(id='http://orig/post https://skipped/inbox', domain='orig',
+                 status='complete', source_mf2=json_dumps(self.create_mf2)).put()
 
         different_create_mf2 = copy.deepcopy(self.create_mf2)
         different_create_mf2['items'][0]['properties']['content'][0]['value'] += ' different'
-        Response(id='http://orig/post https://updated/inbox', status='complete',
-                 direction='out', protocol='activitypub',
+        Response(id='http://orig/post https://updated/inbox', domain='orig',
+                 status='complete', direction='out', protocol='activitypub',
                  source_mf2=json_dumps(different_create_mf2)).put()
 
         Follower.get_or_create('orig', 'https://mastodon/aaa')
@@ -652,18 +654,20 @@ class WebmentionTest(testutil.TestCase):
         self.assertEqual(len(inboxes), len(mock_post.call_args_list))
 
         for call, inbox in zip(mock_post.call_args_list, inboxes):
-            self.assertEqual((inbox,), call[0])
-            self.assertEqual(
-                self.update_as2 if inbox == 'https://updated/inbox' else self.create_as2,
-                json_loads(call[1]['data']))
+            with self.subTest(call=call, inbox=inbox):
+                self.assertEqual((inbox,), call[0])
+                self.assertEqual(
+                    self.update_as2 if inbox == 'https://updated/inbox' else self.create_as2,
+                    json_loads(call[1]['data']))
 
-            resp = Response.get_by_id('http://orig/post %s' % inbox)
-            self.assertEqual('out', resp.direction, inbox)
-            self.assertEqual('activitypub', resp.protocol, inbox)
-            self.assertEqual('complete', resp.status, inbox)
-            self.assertEqual((different_create_mf2 if inbox == 'https://updated/inbox'
-                              else self.create_mf2),
-                             json_loads(resp.source_mf2), inbox)
+                resp = Response.get_by_id('http://orig/post %s' % inbox)
+                self.assertEqual('orig', resp.domain)
+                self.assertEqual('out', resp.direction, inbox)
+                self.assertEqual('activitypub', resp.protocol, inbox)
+                self.assertEqual('complete', resp.status, inbox)
+                self.assertEqual((different_create_mf2 if inbox == 'https://updated/inbox'
+                                  else self.create_mf2),
+                                 json_loads(resp.source_mf2), inbox)
 
     def test_activitypub_create_with_image(self, mock_get, mock_post):
         create_html = self.create_html.replace(
@@ -718,6 +722,7 @@ class WebmentionTest(testutil.TestCase):
         self.assertEqual(self.key.private_pem(), rsa_key.exportKey())
 
         resp = Response.get_by_id('http://a/follow http://followee/')
+        self.assertEqual('a', resp.domain)
         self.assertEqual('out', resp.direction)
         self.assertEqual('activitypub', resp.protocol)
         self.assertEqual('complete', resp.status)
@@ -754,6 +759,7 @@ class WebmentionTest(testutil.TestCase):
         self.assertEqual(self.key.private_pem(), rsa_key.exportKey())
 
         resp = Response.get_by_id('http://a/follow http://followee/')
+        self.assertEqual('a', resp.domain)
         self.assertEqual('out', resp.direction)
         self.assertEqual('activitypub', resp.protocol)
         self.assertEqual('error', resp.status)
@@ -798,6 +804,7 @@ class WebmentionTest(testutil.TestCase):
             entry.content[0]['value'])
 
         resp = Response.get_by_id('http://a/reply http://orig/post')
+        self.assertEqual('a', resp.domain)
         self.assertEqual('out', resp.direction)
         self.assertEqual('ostatus', resp.protocol)
         self.assertEqual('complete', resp.status)
@@ -831,6 +838,7 @@ class WebmentionTest(testutil.TestCase):
         self.assertEqual('http://orig/post', entry['activity_object'])
 
         resp = Response.get_by_id('http://a/like http://orig/post')
+        self.assertEqual('a', resp.domain)
         self.assertEqual('out', resp.direction)
         self.assertEqual('ostatus', resp.protocol)
         self.assertEqual('complete', resp.status)
@@ -882,6 +890,7 @@ class WebmentionTest(testutil.TestCase):
                       got.get_data(as_text=True))
 
         resp = Response.get_by_id('http://a/reply http://orig/url')
+        self.assertEqual('a', resp.domain)
         self.assertEqual('out', resp.direction)
         self.assertEqual('ostatus', resp.protocol)
         self.assertEqual('error', resp.status)

webmention.py

@@ -172,8 +172,9 @@ class Webmention(View):
                                     actor.get('publicInbox')or
                                     actor.get('inbox'))
             return [(Response.get_or_create(
-                        source=self.source_url, target=inbox, direction='out',
-                        protocol='activitypub', source_mf2=json_dumps(self.source_mf2)),
+                        source=self.source_url, target=inbox, domain=self.source_domain,
+                        direction='out', protocol='activitypub',
+                        source_mf2=json_dumps(self.source_mf2)),
                      inbox)
                     for inbox in sorted(inboxes) if inbox]
 
@@ -193,8 +194,9 @@ class Webmention(View):
           target_url = self.target_resp.url or target
 
           resp = Response.get_or_create(
-              source=self.source_url, target=target_url, direction='out',
-              protocol='activitypub', source_mf2=json_dumps(self.source_mf2))
+              source=self.source_url, target=target_url, domain=self.source_domain,
+              direction='out', protocol='activitypub',
+              source_mf2=json_dumps(self.source_mf2))
 
           # find target's inbox
           target_obj = self.target_resp.json()
@@ -258,7 +260,8 @@ class Webmention(View):
         finally:
             if status:
                 Response(source=self.source_url, target=target, status=status,
-                         direction='out', protocol = 'ostatus',
+                         domain=self.source_domain, direction='out',
+                         protocol = 'ostatus',
                          source_mf2=json_dumps(self.source_mf2)).put()
 
     def _try_salmon(self, target):