tighten common.unwrap so it doesn't remove protocol bot user URLs

...like https://bsky.brid.gy/ . this hopefully fixes following bot users in eg AP to enable protocols.

common.py

@@ -199,10 +199,11 @@ def unwrap(val, field=None):
         return [unwrap(v) for v in val]
 
     elif isinstance(val, str):
-        unwrapped = SUBDOMAIN_BASE_URL_RE.sub('', val)
+        if match := SUBDOMAIN_BASE_URL_RE.match(val):
-        if field in ID_FIELDS and re.fullmatch(DOMAIN_RE, unwrapped):
+            unwrapped = match.group('path')
-            unwrapped = f'https://{unwrapped}/'
+            if field in ID_FIELDS and re.fullmatch(DOMAIN_RE, unwrapped):
-        return unwrapped
+                return f'https://{unwrapped}/'
+            return unwrapped
 
     return val
 

eefake.brid.gy.as2.json

@@ -4,5 +4,5 @@
   "url": "https://eefake.brid.gy/",
   "preferredUsername": "eefake.brid.gy",
   "summary": "Only for unit tests",
-  "name": "ExplicitEnableFake"
+  "name": "ExplicitEnableFake protocol class in testutil"
 }

fa.brid.gy.as2.json

@@ -0,0 +1,8 @@
+{
+  "type": "Application",
+  "id": "https://fake.brid.gy/fake.brid.gy",
+  "url": "https://fake.brid.gy/",
+  "preferredUsername": "fake.brid.gy",
+  "summary": "Only for unit tests",
+  "name": "Fake protocol class in testutil"
+}

models.py

@@ -119,7 +119,7 @@ def reset_protocol_properties():
 
     abbrevs = f'({"|".join(PROTOCOLS.keys())}|fed)'
     common.SUBDOMAIN_BASE_URL_RE = re.compile(
-        rf'^https?://({abbrevs}\.brid\.gy|localhost(:8080)?)/(convert/|r/)?({abbrevs}/)?')
+        rf'^https?://({abbrevs}\.brid\.gy|localhost(:8080)?)/(convert/|r/)?({abbrevs}/)?(?P<path>.+)')
 
 
 class User(StringIdModel, metaclass=ProtocolUserMeta):

other.brid.gy.as2.json

@@ -0,0 +1,8 @@
+{
+  "type": "Application",
+  "id": "https://other.brid.gy/other.brid.gy",
+  "url": "https://other.brid.gy/",
+  "preferredUsername": "other.brid.gy",
+  "summary": "Only for unit tests",
+  "name": "OtherFake protocol class in testutil"
+}

tests/test_common.py

@@ -62,9 +62,12 @@ class CommonTest(TestCase):
 
     def test_unwrap_protocol_subdomain(self):
         for input, expected in [
-                ('https://fa.brid.gy/', ''),
                 ('https://fa.brid.gy/ap/fake:foo', 'fake:foo'),
                 ('https://bsky.brid.gy/convert/ap/did:plc:123', 'did:plc:123'),
+                # preserve protocol bot user ids
+                ('https://fed.brid.gy/', 'https://fed.brid.gy/'),
+                ('https://fa.brid.gy/', 'https://fa.brid.gy/'),
+                ('fa.brid.gy', 'fa.brid.gy'),
         ]:
             self.assertEqual(expected, common.unwrap(input))
 

tests/test_follow.py

@@ -16,6 +16,7 @@ from .testutil import Fake, TestCase
 
 from activitypub import ActivityPub
 from common import unwrap
+import ids
 from models import Follower, Object
 from web import Web
 
@@ -367,7 +368,10 @@ class FollowTest(TestCase):
 
         if not expected_follow_as1:
             expected_follow_as1 = as2.to_as1(unwrap(expected_follow))
+            expected_follow_as1['actor'] = ids.translate_user_id(
+                id=expected_follow_as1['actor'], from_=Web, to=Web)
         del expected_follow_as1['to']
+
         self.assert_object(follow_id,
                            users=[self.user.key],
                            notify=[followee],
@@ -414,7 +418,7 @@ class FollowTest(TestCase):
         expected_follow_as1 = as2.to_as1({
             **FOLLOW_URL,
             'id': id,
-            'actor': 'https://www.alice.com/',
+            'actor': 'www.alice.com',
         })
         del expected_follow_as1['to']
         followee = ActivityPub(id='https://ba.r/id').key
@@ -604,6 +608,9 @@ class UnfollowTest(TestCase):
         follower = Follower.query().get()
         self.assertEqual('inactive', follower.status)
 
+        expected_undo_as1 = as2.to_as1(unwrap(expected_undo))
+        expected_undo_as1['actor'] = ids.translate_user_id(
+            id=expected_undo_as1['actor'], from_=Web, to=Web)
         self.assert_object(
             'https://alice.com/#unfollow-2022-01-02T03:04:05-https://ba.r/id',
             users=[self.user.key],
@@ -611,7 +618,7 @@ class UnfollowTest(TestCase):
             status='complete',
             source_protocol='ui',
             labels=['user', 'activity'],
-            our_as1=unwrap(as2.to_as1(expected_undo)),
+            our_as1=expected_undo_as1,
             delivered=['http://ba.r/inbox'],
             delivered_protocol='activitypub')
 
@@ -667,6 +674,10 @@ class UnfollowTest(TestCase):
         follower = Follower.query().get()
         self.assertEqual('inactive', follower.status)
 
+        expected_undo_as1 = as2.to_as1(unwrap(expected_undo))
+        expected_undo_as1['actor'] = ids.translate_user_id(
+            id=expected_undo_as1['actor'], from_=Web, to=Web)
+
         self.assert_object(
             'https://www.alice.com/#unfollow-2022-01-02T03:04:05-https://ba.r/id',
             users=[user.key],
@@ -674,7 +685,7 @@ class UnfollowTest(TestCase):
             status='complete',
             source_protocol='ui',
             labels=['user', 'activity'],
-            our_as1=unwrap(as2.to_as1(expected_undo)),
+            our_as1=expected_undo_as1,
             delivered=['http://ba.r/inbox'],
             delivered_protocol='activitypub')
 

tests/test_integrations.py

@@ -4,19 +4,21 @@ from unittest.mock import patch
 
 from arroba.datastore_storage import DatastoreStorage
 from arroba.repo import Repo
-from flask import g
+from dns.resolver import NXDOMAIN
+from granary import as2
+from granary.tests.test_bluesky import ACTOR_PROFILE_BSKY, POST_BSKY
+from oauth_dropins.webutil.flask_util import NoContent
 from oauth_dropins.webutil.testutil import requests_response
 
 from activitypub import ActivityPub
 import app
 from atproto import ATProto
-from dns.resolver import NXDOMAIN
-from granary.tests.test_bluesky import ACTOR_PROFILE_BSKY, POST_BSKY
 import hub
 from models import Object, Target
 from web import Web
 
 from .testutil import ATPROTO_KEY, TestCase
+from .test_activitypub import ACTOR
 from . import test_atproto
 from . import test_web
 
@@ -355,3 +357,48 @@ class IntegrationTests(TestCase):
                                **POST_BSKY,
                                'cid': 'sydd',
                            })
+
+
+    @patch('requests.post', return_value=requests_response('OK'))  # create DID
+    @patch('requests.get')
+    def test_activitypub_follow_bsky_bot_user_enables_protocol(
+            self, mock_get, mock_post):
+        """AP follow of @bsky.brid.gy@bsky.brid.gy bridges the account into BLuesky.
+
+        ActivityPub user @alice@inst , https://inst/alice
+        ATProto bot user bsky.brid.gy (did:plc:bsky)
+        Follow is https://inst/follow
+        """
+        mock_get.return_value = self.as2_resp({
+            'type': 'Person',
+            'id': 'https://inst/alice',
+            'name': 'Mrs. ☕ Alice',
+            'preferredUsername': 'alice',
+            'inbox': 'http://inst/inbox',
+        })
+        bot_user = self.make_user(id='bsky.brid.gy', cls=Web, ap_subdomain='bsky')
+
+        # deliver follow
+        resp = self.post('/bsky.brid.gy/inbox', json={
+            'type': 'Follow',
+            'id': 'http://inst/follow',
+            'actor': 'https://inst/alice',
+            'object': 'https://bsky.brid.gy/bsky.brid.gy',
+        })
+        self.assertEqual(204, resp.status_code)
+
+        # check results
+        user = ActivityPub.get_by_id('https://inst/alice')
+        self.assertTrue(ActivityPub.is_enabled_to(ATProto, user=user))
+
+        self.assertEqual(1, len(user.copies))
+        self.assertEqual('atproto', user.copies[0].protocol)
+        did = user.copies[0].uri
+
+        storage = DatastoreStorage()
+        repo = storage.load_repo('alice.inst.ap.brid.gy')
+        self.assertEqual(did, repo.did)
+
+        records = repo.get_contents()
+        self.assertEqual(['app.bsky.actor.profile'], list(records.keys()))
+        self.assertEqual(['self'], list(records['app.bsky.actor.profile'].keys()))

tests/test_protocol.py

@@ -1851,10 +1851,9 @@ class ProtocolReceiveTest(TestCase):
         self.assertEqual(['fake'], user.enabled_protocols)
         self.assertEqual(['eefake:user'], Fake.created_for)
         self.assertTrue(ExplicitEnableFake.is_enabled_to(Fake, user))
-        self.assertEqual([
+        self.assertEqual([('fa.brid.gy/followers#accept-eefake:follow',
-            ('https://fa.brid.gy//followers#accept-eefake:follow',
+                           'eefake:user:target')],
-             'eefake:user:target'),
+                         ExplicitEnableFake.sent)
-        ], ExplicitEnableFake.sent)
 
         # another follow should be a noop
         follow['id'] += '2'

tests/test_web.py

@@ -208,7 +208,7 @@ REPLY = requests_response(REPLY_HTML, url='https://user.com/reply')
 REPLY_MF2 = util.parse_mf2(REPLY_HTML)['items'][0]
 REPLY_AS1 = microformats2.json_to_object(REPLY_MF2)
 REPLY_AS1['id'] = 'https://user.com/reply'
-REPLY_AS1['author']['id'] = 'https://user.com/'
+REPLY_AS1['author']['id'] = 'user.com'
 CREATE_REPLY_AS1 = {
     'objectType': 'activity',
     'verb': 'post',
@@ -351,13 +351,16 @@ NOTE_HTML = """\
 NOTE = requests_response(NOTE_HTML, url='https://user.com/post')
 NOTE_MF2 = util.parse_mf2(NOTE_HTML)['items'][0]
 NOTE_AS1 = microformats2.json_to_object(NOTE_MF2)
-NOTE_AS1.update({
+NOTE_AS1['id'] = 'https://user.com/post'
-    'author': {
+NOTE_AS1['author']['id'] = 'user.com'
-        **NOTE_AS1['author'],
+CREATE_AS1 = {
-        'id': 'https://user.com/',
+    'objectType': 'activity',
-    },
+    'verb': 'post',
-    'id': 'https://user.com/post',
+    'id': 'https://user.com/post#bridgy-fed-create',
-})
+    'actor': ACTOR_AS1_UNWRAPPED,
+    'object': copy.deepcopy(NOTE_AS1),
+    'published': '2022-01-02T03:04:05+00:00',
+}
 NOTE_AS2 = {
     'type': 'Note',
     'id': 'http://localhost/r/https://user.com/post',
@@ -368,14 +371,6 @@ NOTE_AS2 = {
     'contentMap': {'en': 'hello i am a post'},
     'to': [as2.PUBLIC_AUDIENCE],
 }
-CREATE_AS1 = {
-    'objectType': 'activity',
-    'verb': 'post',
-    'id': 'https://user.com/post#bridgy-fed-create',
-    'actor': ACTOR_AS1_UNWRAPPED,
-    'object': NOTE_AS1,
-    'published': '2022-01-02T03:04:05+00:00',
-}
 CREATE_AS2 = {
     '@context': 'https://www.w3.org/ns/activitystreams',
     'type': 'Create',
@@ -481,11 +476,11 @@ class WebTest(TestCase):
             'acct:user.com',
             'acct:@user.com@user.com',
             'acc:me@user.com',
-            'ap.brid.gy',
             'localhost',
         ):
-            with self.assertRaises(AssertionError):
+            with self.subTest(id=bad):
-                Web(id=bad).put()
+                with self.assertRaises(AssertionError):
+                    Web(id=bad).put()
 
     def test_get_or_create_lower_cases_domain(self, mock_get, mock_post):
         mock_get.return_value = requests_response('')
@@ -1150,10 +1145,12 @@ class WebTest(TestCase):
         self.assertEqual(202, got.status_code)
 
         inboxes = ['https://inbox/', 'https://public/inbox', 'https://shared/inbox']
+        expected_create_as1 = copy.deepcopy(CREATE_AS1)
+        expected_create_as1['object']['author']['id'] = 'https://user.com/'
         self.assert_object('https://user.com/post#bridgy-fed-create',
                            users=[self.user.key],
                            source_protocol='web',
-                           our_as1=CREATE_AS1,
+                           our_as1=expected_create_as1,
                            type='post',
                            labels=['activity', 'user'],
                            delivered=inboxes,