diff --git a/activitypub.py b/activitypub.py index 92c3bff..e3eb759 100644 --- a/activitypub.py +++ b/activitypub.py @@ -65,7 +65,7 @@ def actor(domain): # TODO: unify with common.actor() actor = { - **common.postprocess_as2(json_loads(user.actor_as2), user=user), + **common.postprocess_as2(user.actor_as2, user=user), 'id': host_url(domain), # This has to be the domain for Mastodon etc interop! It seems like it # should be the custom username from the acct: u-url in their h-card, @@ -126,10 +126,8 @@ def inbox(domain=None): return msg, 200 activity_unwrapped = redirect_unwrap(activity) - activity_obj = Object( - id=id, - as2=json_dumps(activity_unwrapped), - source_protocol='activitypub') + activity_obj = Object(id=id, as2=activity_unwrapped, + source_protocol='activitypub') activity_obj.put() if type == 'Accept': # eg in response to a Follow @@ -161,7 +159,7 @@ def inbox(domain=None): elif digest.removeprefix('SHA-256=') != expected: logger.warning('Invalid Digest header, required for HTTP Signature') else: - key_actor = json_loads(common.get_object(keyId, user=user).as2) + key_actor = common.get_object(keyId, user=user).as2 key = key_actor.get("publicKey", {}).get('publicKeyPem') logger.info(f'Verifying signature for {request.path} with key {key}') try: @@ -190,10 +188,7 @@ def inbox(domain=None): error("Couldn't find obj_id of object to update") obj = Object.get_by_id(obj_id) or Object(id=obj_id) - obj.populate( - as2=json_dumps(obj_as2), - source_protocol='activitypub', - ) + obj.populate(as2=obj_as2, source_protocol='activitypub') obj.put() activity_obj.status = 'complete' @@ -226,20 +221,20 @@ def inbox(domain=None): # fetch actor if necessary so we have name, profile photo, etc if actor and isinstance(actor, str): actor = activity['actor'] = activity_unwrapped['actor'] = \ - json_loads(common.get_object(actor, user=user).as2) + common.get_object(actor, user=user).as2 # fetch object if necessary so we can render it in feeds inner_obj = activity_unwrapped.get('object') if type in FETCH_OBJECT_TYPES and isinstance(inner_obj, str): obj = Object.get_by_id(inner_obj) or common.get_object(inner_obj, user=user) obj_as2 = activity['object'] = activity_unwrapped['object'] = \ - json_loads(obj.as2) if obj.as2 else as2.from_as1(json_loads(obj.as1)) + obj.as2 if obj.as2 else as2.from_as1(obj.as1) if type == 'Follow': resp = accept_follow(activity, activity_unwrapped, user) # send webmentions to each target - activity_obj.as2 = json_dumps(activity_unwrapped) + activity_obj.as2 = activity_unwrapped common.send_webmentions(as2.to_as1(activity), activity_obj, proxy=True) # deliver original posts and reposts to followers @@ -305,7 +300,7 @@ def accept_follow(follow, follow_unwrapped, user): # store Follower follower = Follower.get_or_create(dest=user.key.id(), src=follower_id, - last_follow=json_dumps(follow)) + last_follow=follow) follower.status = 'active' follower.put() diff --git a/common.py b/common.py index e44ba59..c98990e 100644 --- a/common.py +++ b/common.py @@ -158,8 +158,7 @@ def get_object(id, user=None): logging.warning(f'Wiping out mf2 property: {obj.mf2}') obj.mf2 = None - obj.populate(as2=json_dumps(obj_as2), - source_protocol='activitypub') + obj.populate(as2=obj_as2, source_protocol='activitypub') obj.put() return obj diff --git a/follow.py b/follow.py index 329a001..9be321b 100644 --- a/follow.py +++ b/follow.py @@ -156,7 +156,7 @@ class FollowCallback(indieauth.Callback): flash(f"Couldn't find ActivityPub profile link for {addr}") return redirect(f'/user/{domain}/following') - followee = json_loads(common.get_object(as2_url, user=user).as2) + followee = common.get_object(as2_url, user=user).as2 id = followee.get('id') inbox = followee.get('inbox') if not id or not inbox: @@ -175,11 +175,10 @@ class FollowCallback(indieauth.Callback): } common.signed_post(inbox, user=user, data=follow_as2) - follow_json = json_dumps(follow_as2, sort_keys=True) Follower.get_or_create(dest=id, src=domain, status='active', - last_follow=follow_json) + last_follow=follow_as2) Object(id=follow_id, domains=[domain], labels=['user', 'activity'], - source_protocol='ui', status='complete', as2=follow_json, + source_protocol='ui', status='complete', as2=follow_as2, ).put() link = common.pretty_link(util.get_url(followee) or id, text=addr) @@ -229,13 +228,12 @@ class UnfollowCallback(indieauth.Callback): error(f'Bad state {state}') followee_id = follower.dest - last_follow = json_loads(follower.last_follow) - followee = last_follow['object'] + followee = follower.last_follow['object'] if isinstance(followee, str): # fetch as AS2 to get full followee with inbox followee_id = followee - followee = json_loads(common.get_object(followee_id, user=user).as2) + followee = common.get_object(followee_id, user=user).as2 inbox = followee.get('inbox') if not inbox: @@ -249,15 +247,14 @@ class UnfollowCallback(indieauth.Callback): 'type': 'Undo', 'id': unfollow_id, 'actor': common.host_url(domain), - 'object': last_follow, + 'object': follower.last_follow, } common.signed_post(inbox, user=user, data=unfollow_as2) follower.status = 'inactive' follower.put() Object(id=unfollow_id, domains=[domain], labels=['user', 'activity'], - source_protocol='ui', status='complete', - as2=json_dumps(unfollow_as2, sort_keys=True), + source_protocol='ui', status='complete', as2=unfollow_as2, ).put() link = common.pretty_link(util.get_url(followee) or followee_id) diff --git a/models.py b/models.py index 34de3ef..fbc21da 100644 --- a/models.py +++ b/models.py @@ -14,7 +14,7 @@ from flask import request from google.cloud import ndb from granary import as1, as2, bluesky, microformats2 from oauth_dropins.webutil.appengine_info import DEBUG -from oauth_dropins.webutil.models import StringIdModel +from oauth_dropins.webutil.models import JsonProperty, StringIdModel from oauth_dropins.webutil import util from oauth_dropins.webutil.util import json_dumps, json_loads @@ -64,7 +64,7 @@ class User(StringIdModel): has_redirects = ndb.BooleanProperty() redirects_error = ndb.TextProperty() has_hcard = ndb.BooleanProperty() - actor_as2 = ndb.TextProperty() + actor_as2 = JsonProperty() use_instead = ndb.KeyProperty() created = ndb.DateTimeProperty(auto_now_add=True) @@ -130,7 +130,7 @@ class User(StringIdModel): def to_as1(self): """Returns this user as an AS1 actor dict, if possible.""" if self.actor_as2: - return as2.to_as1(json_loads(self.actor_as2)) + return as2.to_as1(self.actor_as2) def username(self): """Returns the user's preferred username. @@ -143,9 +143,8 @@ class User(StringIdModel): domain = self.key.id() if self.actor_as2: - actor = json_loads(self.actor_as2) for url in [u.get('value') if isinstance(u, dict) else u - for u in util.get_list(actor, 'url')]: + for u in util.get_list(self.actor_as2, 'url')]: if url and url.startswith('acct:'): urluser, urldomain = util.parse_acct_uri(url) if urldomain == domain: @@ -177,7 +176,7 @@ class User(StringIdModel): def user_page_link(self): """Returns a pretty user page link with the user's name and profile picture.""" domain = self.key.id() - actor = util.json_loads(self.actor_as2) if self.actor_as2 else {} + actor = self.actor_as2 or {} name = (actor.get('name') or # prettify if domain, noop if username util.domain_from_link(self.username())) @@ -236,8 +235,7 @@ class User(StringIdModel): # check home page try: - _, _, actor_as2 = common.actor(self) - self.actor_as2 = json_dumps(actor_as2) + _, _, self.actor_as2 = common.actor(self) self.has_hcard = True except (BadRequest, NotFound): self.actor_as2 = None @@ -280,22 +278,22 @@ class Object(StringIdModel): source_protocol = ndb.StringProperty(choices=PROTOCOLS) labels = ndb.StringProperty(repeated=True, choices=LABELS) - # these are all JSON. They're TextProperty, and not JsonProperty, so that - # their plain text is visible in the App Engine admin console. (JsonProperty - # uses a blob.) - as2 = ndb.TextProperty() # only one of the rest will be populated... - bsky = ndb.TextProperty() # Bluesky / AT Protocol - mf2 = ndb.TextProperty() # HTML microformats2 + # TODO: switch back to ndb.JsonProperty if/when they fix it for the web console + # https://github.com/googleapis/python-ndb/issues/874 + as2 = JsonProperty() # only one of the rest will be populated... + bsky = JsonProperty() # Bluesky / AT Protocol + mf2 = JsonProperty() # HTML microformats2 @ndb.ComputedProperty def as1(self): - assert bool(self.as2) ^ bool(self.bsky) ^ bool(self.mf2), \ - f'{bool(self.as2)} {bool(self.bsky)} {bool(self.mf2)}' - return (as2.to_as1(common.redirect_unwrap(json_loads(self.as2))) if self.as2 - else bluesky.to_as1(json_loads(self.bsky)) if self.bsky - else microformats2.json_to_object(json_loads(self.mf2)) - if self.mf2 - else None) + assert (self.as2 is not None) ^ (self.bsky is not None) ^ (self.mf2 is not None), \ + f'{self.as2} {self.bsky} {self.mf2}' + if self.as2 is not None: + return as2.to_as1(common.redirect_unwrap(self.as2)) + elif self.bsky is not None: + return bluesky.to_as1(self.bsky) + elif self.mf2 is not None: + return microformats2.json_to_object(self.mf2) @ndb.ComputedProperty def type(self): # AS1 objectType, or verb if it's an activity @@ -371,7 +369,7 @@ class Follower(StringIdModel): dest = ndb.StringProperty() # Most recent AP (AS2) JSON Follow activity. If inbound, must have a # composite actor object with an inbox, publicInbox, or sharedInbox. - last_follow = ndb.TextProperty() + last_follow = JsonProperty() status = ndb.StringProperty(choices=STATUSES, default='active') created = ndb.DateTimeProperty(auto_now_add=True) @@ -403,7 +401,4 @@ class Follower(StringIdModel): def to_as2(self): """Returns this follower as an AS2 actor dict, if possible.""" if self.last_follow: - last_follow = json_loads(self.last_follow) - person = last_follow.get('actor' if util.is_web(self.src) else 'object') - if person: - return person + return self.last_follow.get('actor' if util.is_web(self.src) else 'object') diff --git a/pages.py b/pages.py index 4a2908e..d999983 100644 --- a/pages.py +++ b/pages.py @@ -209,7 +209,7 @@ def fetch_objects(query, user): if isinstance(inner_obj, str): inner_obj = Object.get_by_id(inner_obj) if inner_obj: - inner_obj = json_loads(inner_obj.as1) + inner_obj = inner_obj.as1 content = (inner_obj.get('content') or inner_obj.get('displayName') diff --git a/render.py b/render.py index 26ef056..2665698 100644 --- a/render.py +++ b/render.py @@ -9,7 +9,6 @@ from granary import as2, atom, microformats2 from oauth_dropins.webutil import flask_util from oauth_dropins.webutil.flask_util import error from oauth_dropins.webutil import util -from oauth_dropins.webutil.util import json_loads from app import app, cache import common diff --git a/tests/test_activitypub.py b/tests/test_activitypub.py index ac05797..ba27aab 100644 --- a/tests/test_activitypub.py +++ b/tests/test_activitypub.py @@ -202,8 +202,7 @@ class ActivityPubTest(testutil.TestCase): def setUp(self): super().setUp() - self.user = User.get_or_create('foo.com', has_hcard=True, - actor_as2=json_dumps(ACTOR)) + self.user = User.get_or_create('foo.com', has_hcard=True, actor_as2=ACTOR) def test_actor(self, *_): got = self.client.get('/foo.com') @@ -369,7 +368,7 @@ class ActivityPubTest(testutil.TestCase): 'url': 'https://foo.com/orig', } with app.test_request_context('/'): - Object(id=orig_url, as2=json_dumps(note)).put() + Object(id=orig_url, as2=note).put() repost = { **REPOST_FULL, @@ -559,7 +558,7 @@ class ActivityPubTest(testutil.TestCase): object_ids=[FOLLOW['object']]) follower = Follower.query().get() - self.assertEqual(FOLLOW_WRAPPED_WITH_ACTOR, json_loads(follower.last_follow)) + self.assertEqual(FOLLOW_WRAPPED_WITH_ACTOR, follower.last_follow) def test_inbox_follow_accept_with_object(self, *mocks): wrapped_user = { @@ -583,7 +582,7 @@ class ActivityPubTest(testutil.TestCase): follower = Follower.query().get() follow['actor'] = ACTOR - self.assertEqual(follow, json_loads(follower.last_follow)) + self.assertEqual(follow, follower.last_follow) follow.update({ 'object': unwrapped_user, @@ -652,7 +651,7 @@ class ActivityPubTest(testutil.TestCase): # check that the Follower doesn't have www follower = Follower.get_by_id(f'foo.com {ACTOR["id"]}') self.assertEqual('active', follower.status) - self.assertEqual(FOLLOW_WRAPPED_WITH_ACTOR, json_loads(follower.last_follow)) + self.assertEqual(FOLLOW_WRAPPED_WITH_ACTOR, follower.last_follow) def test_inbox_undo_follow(self, mock_head, mock_get, mock_post): mock_head.return_value = requests_response(url='https://foo.com/') @@ -850,7 +849,7 @@ class ActivityPubTest(testutil.TestCase): self.assertEqual('active', other.key.get().status) def test_delete_note(self, _, mock_get, ___): - obj = Object(id='http://an/obj', as2='{}') + obj = Object(id='http://an/obj', as2={}) obj.put() mock_get.side_effect = [ @@ -872,7 +871,7 @@ class ActivityPubTest(testutil.TestCase): self.assert_entities_equal(obj, common.get_object.cache['http://an/obj']) def test_update_note(self, *mocks): - Object(id='https://a/note', as2='{}').put() + Object(id='https://a/note', as2={}).put() self._test_update(*mocks) def test_update_unknown(self, *mocks): @@ -928,7 +927,7 @@ class ActivityPubTest(testutil.TestCase): object_ids=[LIKE['object']]) def test_inbox_id_already_seen(self, *mocks): - obj_key = Object(id=FOLLOW_WRAPPED['id'], as2='{}').put() + obj_key = Object(id=FOLLOW_WRAPPED['id'], as2={}).put() got = self.client.post('/foo.com/inbox', json=FOLLOW_WRAPPED) self.assertEqual(200, got.status_code) @@ -964,10 +963,10 @@ class ActivityPubTest(testutil.TestCase): def store_followers(self): Follower.get_or_create('foo.com', 'https://bar.com', - last_follow=json_dumps(FOLLOW_WITH_ACTOR)) + last_follow=FOLLOW_WITH_ACTOR) Follower.get_or_create('http://other/actor', 'foo.com') Follower.get_or_create('foo.com', 'https://baz.com', - last_follow=json_dumps(FOLLOW_WITH_ACTOR)) + last_follow=FOLLOW_WITH_ACTOR) Follower.get_or_create('foo.com', 'baj.com', status='inactive') def test_followers_collection(self, *args): @@ -1032,10 +1031,10 @@ class ActivityPubTest(testutil.TestCase): def store_following(self): Follower.get_or_create('https://bar.com', 'foo.com', - last_follow=json_dumps(FOLLOW_WITH_OBJECT)) + last_follow=FOLLOW_WITH_OBJECT) Follower.get_or_create('foo.com', 'http://other/actor') Follower.get_or_create('https://baz.com', 'foo.com', - last_follow=json_dumps(FOLLOW_WITH_OBJECT)) + last_follow=FOLLOW_WITH_OBJECT) Follower.get_or_create('baj.com', 'foo.com', status='inactive') def test_following_collection(self, *args): diff --git a/tests/test_common.py b/tests/test_common.py index f5d83b6..b42d8f5 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -4,7 +4,6 @@ from unittest import mock from granary import as2 from oauth_dropins.webutil import appengine_config, util -from oauth_dropins.webutil.util import json_dumps, json_loads from oauth_dropins.webutil.testutil import requests_response import requests from werkzeug.exceptions import BadGateway @@ -245,7 +244,7 @@ class CommonTest(testutil.TestCase): id = 'http://the/id' got = common.get_object(id) self.assert_equals(id, got.key.id()) - self.assert_equals(AS2_OBJ, json_loads(got.as2)) + self.assert_equals(AS2_OBJ, got.as2) mock_get.assert_has_calls([self.as2_req(id)]) # second time is in cache @@ -253,13 +252,13 @@ class CommonTest(testutil.TestCase): mock_get.reset_mock() got = common.get_object(id) self.assert_equals(id, got.key.id()) - self.assert_equals(AS2_OBJ, json_loads(got.as2)) + self.assert_equals(AS2_OBJ, got.as2) mock_get.assert_not_called() @mock.patch('requests.get') def test_get_object_datastore(self, mock_get): id = 'http://the/id' - stored = Object(id=id, as2=json_dumps(AS2_OBJ)) + stored = Object(id=id, as2=AS2_OBJ) stored.put() common.get_object.cache.clear() @@ -276,7 +275,7 @@ class CommonTest(testutil.TestCase): @mock.patch('requests.get') def test_get_object_strips_fragment(self, mock_get): - stored = Object(id='http://the/id', as2=json_dumps(AS2_OBJ)) + stored = Object(id='http://the/id', as2=AS2_OBJ) stored.put() common.get_object.cache.clear() @@ -288,7 +287,7 @@ class CommonTest(testutil.TestCase): def test_get_object_datastore_no_as2(self, mock_get): """If the stored Object has no as2, we should fall back to HTTP.""" id = 'http://the/id' - stored = Object(id=id, mf2='{}', status='in progress') + stored = Object(id=id, as2={}, status='in progress') stored.put() common.get_object.cache.clear() @@ -296,7 +295,7 @@ class CommonTest(testutil.TestCase): mock_get.assert_has_calls([self.as2_req(id)]) self.assert_equals(id, got.key.id()) - self.assert_equals(AS2_OBJ, json_loads(got.as2)) + self.assert_equals(AS2_OBJ, got.as2) mock_get.assert_has_calls([self.as2_req(id)]) self.assert_object(id, as2=AS2_OBJ, as1=AS2_OBJ, diff --git a/tests/test_follow.py b/tests/test_follow.py index 883ed25..7dd72cc 100644 --- a/tests/test_follow.py +++ b/tests/test_follow.py @@ -198,8 +198,7 @@ class FollowTest(testutil.TestCase): followers = Follower.query().fetch() self.assert_entities_equal( - Follower(id='https://bar/id alice.com', - last_follow=json_dumps(expected_follow, sort_keys=True), + Follower(id='https://bar/id alice.com', last_follow=expected_follow, src='alice.com', dest='https://bar/id', status='active'), followers, ignore=['created', 'updated']) @@ -249,8 +248,7 @@ class FollowTest(testutil.TestCase): } followers = Follower.query().fetch() self.assert_entities_equal( - Follower(id='https://bar/id www.alice.com', - last_follow=json_dumps(expected_follow, sort_keys=True), + Follower(id='https://bar/id www.alice.com', last_follow=expected_follow, src='www.alice.com', dest='https://bar/id', status='active'), followers, ignore=['created', 'updated']) @@ -301,7 +299,7 @@ class UnfollowTest(testutil.TestCase): super().setUp() self.user = User.get_or_create('alice.com') self.follower = Follower( - id='https://bar/id alice.com', last_follow=json_dumps(FOLLOW_ADDRESS), + id='https://bar/id alice.com', last_follow=FOLLOW_ADDRESS, src='alice.com', dest='https://bar/id', status='active', ).put() self.state = util.encode_oauth_state({ @@ -335,10 +333,10 @@ class UnfollowTest(testutil.TestCase): def test_callback_last_follow_object_str(self, mock_get, mock_post): follower = self.follower.get() - follower.last_follow = json_dumps({ + follower.last_follow = { **FOLLOW_ADDRESS, 'object': FOLLOWEE['id'], - }) + } follower.put() mock_get.side_effect = ( @@ -391,7 +389,7 @@ class UnfollowTest(testutil.TestCase): self.user.put() self.follower = Follower( - id='https://bar/id www.alice.com', last_follow=json_dumps(FOLLOW_ADDRESS), + id='https://bar/id www.alice.com', last_follow=FOLLOW_ADDRESS, src='www.alice.com', dest='https://bar/id', status='active', ).put() diff --git a/tests/test_models.py b/tests/test_models.py index 4fe261d..288f0e0 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -5,7 +5,6 @@ from unittest import mock from flask import get_flashed_messages from granary import as2 from oauth_dropins.webutil.testutil import requests_response -from oauth_dropins.webutil.util import json_dumps, json_loads from app import app import common @@ -58,13 +57,13 @@ class UserTest(testutil.TestCase): def test_address(self): self.assertEqual('@y.z@y.z', self.user.address()) - self.user.actor_as2 = '{"type": "Person"}' + self.user.actor_as2 = {'type': 'Person'} self.assertEqual('@y.z@y.z', self.user.address()) - self.user.actor_as2 = '{"url": "http://foo"}' + self.user.actor_as2 = {'url': 'http://foo'} self.assertEqual('@y.z@y.z', self.user.address()) - self.user.actor_as2 = '{"url": ["http://foo", "acct:bar@foo", "acct:baz@y.z"]}' + self.user.actor_as2 = {'url': ['http://foo', 'acct:bar@foo', 'acct:baz@y.z']} self.assertEqual('@baz@y.z', self.user.address()) def _test_verify(self, redirects, hcard, actor, redirects_error=None): @@ -79,7 +78,7 @@ class UserTest(testutil.TestCase): if actor is None: self.assertIsNone(self.user.actor_as2) else: - got = {k: v for k, v in json_loads(self.user.actor_as2).items() + got = {k: v for k, v in self.user.actor_as2.items() if k in actor} self.assert_equals(actor, got) self.assert_equals(redirects_error, self.user.redirects_error) @@ -250,7 +249,7 @@ class ObjectTest(testutil.TestCase): def test_proxy_url(self): with app.test_request_context('/'): - obj = Object(id='abc', as2='{}') + obj = Object(id='abc', as2={}) self.assertEqual('http://localhost/render?id=abc', obj.proxy_url()) def test_actor_link(self): @@ -272,24 +271,24 @@ class ObjectTest(testutil.TestCase): }}), ): with app.test_request_context('/'): - obj = Object(id='x', as2=json_dumps(as2)) + obj = Object(id='x', as2=as2) self.assertEqual(expected, obj.actor_link()) def test_actor_link_user(self): - user = User(id='foo.com', actor_as2='{"name": "Alice"}') + user = User(id='foo.com', actor_as2={"name": "Alice"}) obj = Object(id='x', source_protocol='ui', domains=['foo.com']) self.assertEqual( ' Alice', obj.actor_link(user)) def test_put_updates_get_object_cache(self): - obj = Object(id='x', as2='{}') + obj = Object(id='x', as2={}) obj.put() key = common.get_object.cache_key('x') self.assert_entities_equal(obj, common.get_object.cache[key]) def test_put_fragment_id_doesnt_update_get_object_cache(self): - obj = Object(id='x#y', as2='{}') + obj = Object(id='x#y', as2={}) obj.put() self.assertNotIn(common.get_object.cache_key('x#y'), common.get_object.cache) @@ -301,9 +300,9 @@ class FollowerTest(testutil.TestCase): def setUp(self): super().setUp() self.inbound = Follower(dest='foo.com', src='http://bar/@baz', - last_follow=json_dumps({'actor': ACTOR})) + last_follow={'actor': ACTOR}) self.outbound = Follower(dest='http://bar/@baz', src='foo.com', - last_follow=json_dumps({'object': ACTOR})) + last_follow={'object': ACTOR}) def test_to_as1(self): self.assertEqual({}, Follower().to_as1()) diff --git a/tests/test_pages.py b/tests/test_pages.py index b47b587..d54e9ef 100644 --- a/tests/test_pages.py +++ b/tests/test_pages.py @@ -14,7 +14,6 @@ from granary.tests.test_as1 import ( ) from oauth_dropins.webutil import util from oauth_dropins.webutil.testutil import requests_response -from oauth_dropins.webutil.util import json_dumps, json_loads import common from models import Object, Follower, User @@ -70,9 +69,8 @@ class PagesTest(testutil.TestCase): user = User.get_by_id('orig') self.assertTrue(user.has_hcard) - actor_as2 = json_loads(user.actor_as2) - self.assertEqual('Person', actor_as2['type']) - self.assertEqual('http://localhost/orig', actor_as2['id']) + self.assertEqual('Person', user.actor_as2['type']) + self.assertEqual('http://localhost/orig', user.actor_as2['id']) def test_check_web_site_bad_url(self): got = self.client.post('/web-site', data={'url': '!!!'}) @@ -97,7 +95,7 @@ class PagesTest(testutil.TestCase): User.get_or_create('bar.com') Follower.get_or_create('bar.com', 'https://no/stored/follow') Follower.get_or_create('bar.com', 'https://masto/user', - last_follow=json_dumps(FOLLOW_WITH_ACTOR)) + last_follow=FOLLOW_WITH_ACTOR) got = self.client.get('/user/bar.com/followers') self.assert_equals(200, got.status_code) @@ -118,7 +116,7 @@ class PagesTest(testutil.TestCase): def test_following(self): Follower.get_or_create('https://no/stored/follow', 'bar.com') Follower.get_or_create('https://masto/user', 'bar.com', - last_follow=json_dumps(FOLLOW_WITH_OBJECT)) + last_follow=FOLLOW_WITH_OBJECT) User.get_or_create('bar.com') got = self.client.get('/user/bar.com/following') self.assert_equals(200, got.status_code) diff --git a/tests/test_redirect.py b/tests/test_redirect.py index d251ca8..a6127b8 100644 --- a/tests/test_redirect.py +++ b/tests/test_redirect.py @@ -3,7 +3,6 @@ import copy from granary import as2 -from oauth_dropins.webutil.util import json_dumps, json_loads from app import app, cache from common import redirect_unwrap @@ -73,8 +72,7 @@ class RedirectTest(testutil.TestCase): def _test_as2(self, content_type): with app.test_request_context('/'): - self.obj = Object(id='https://foo.com/bar', - as2=json_dumps(REPOST_AS2)).put() + self.obj = Object(id='https://foo.com/bar', as2=REPOST_AS2).put() resp = self.client.get('/r/https://foo.com/bar', headers={'Accept': content_type}) @@ -88,7 +86,7 @@ class RedirectTest(testutil.TestCase): def test_as2_deleted(self): with app.test_request_context('/'): - Object(id='https://foo.com/bar', as2='{}', deleted=True).put() + Object(id='https://foo.com/bar', as2={}, deleted=True).put() resp = self.client.get('/r/https://foo.com/bar', headers={'Accept': as2.CONTENT_TYPE}) diff --git a/tests/test_render.py b/tests/test_render.py index 1bc5b6c..112881d 100644 --- a/tests/test_render.py +++ b/tests/test_render.py @@ -5,7 +5,6 @@ from unittest import skip from granary import as2 from granary.tests.test_as1 import COMMENT, DELETE_OF_ID, UPDATE -from oauth_dropins.webutil.util import json_dumps from app import app import common @@ -47,7 +46,7 @@ class RenderTest(testutil.TestCase): def test_render(self): with app.test_request_context('/'): - Object(id='abc', as2=json_dumps(as2.from_as1(COMMENT))).put() + Object(id='abc', as2=as2.from_as1(COMMENT)).put() resp = self.client.get('/render?id=abc') self.assertEqual(200, resp.status_code) self.assert_multiline_equals(EXPECTED_HTML, resp.get_data(as_text=True), ignore_blanks=True) @@ -56,7 +55,7 @@ class RenderTest(testutil.TestCase): comment = copy.deepcopy(COMMENT) del comment['url'] with app.test_request_context('/'): - Object(id='abc', as2=json_dumps(as2.from_as1(comment))).put() + Object(id='abc', as2=as2.from_as1(comment)).put() resp = self.client.get('/render?id=abc') self.assertEqual(200, resp.status_code) @@ -70,7 +69,7 @@ class RenderTest(testutil.TestCase): def test_render_update_redirect(self): with app.test_request_context('/'): # UPDATE's object field is a full object - Object(id='abc', as2=json_dumps(as2.from_as1(UPDATE))).put() + Object(id='abc', as2=as2.from_as1(UPDATE)).put() resp = self.client.get('/render?id=abc') self.assertEqual(301, resp.status_code) @@ -81,7 +80,7 @@ class RenderTest(testutil.TestCase): def test_render_delete_redirect(self): with app.test_request_context('/'): # DELETE_OF_ID's object field is a bare string id - Object(id='abc', as1=json_dumps(as2.from_as1(DELETE_OF_ID))).put() + Object(id='abc', as1=as2.from_as1(DELETE_OF_ID)).put() resp = self.client.get('/render?id=abc') self.assertEqual(301, resp.status_code) diff --git a/tests/test_webfinger.py b/tests/test_webfinger.py index 47a1a05..9c66d54 100644 --- a/tests/test_webfinger.py +++ b/tests/test_webfinger.py @@ -3,8 +3,6 @@ import html import urllib.parse -from oauth_dropins.webutil.util import json_dumps, json_loads - import common from models import User from . import testutil @@ -47,7 +45,7 @@ class WebfingerTest(testutil.TestCase): 'icon': {'type': 'Image', 'url': 'https://foo.com/me.jpg'}, } self.user = User.get_or_create('foo.com', has_hcard=True, - actor_as2=json_dumps(self.actor_as2)) + actor_as2=self.actor_as2) self.user.put() self.expected_webfinger = { 'subject': 'acct:foo.com@foo.com', @@ -153,14 +151,14 @@ class WebfingerTest(testutil.TestCase): self.assertEqual(self.expected_webfinger, got.json) def test_webfinger_custom_username(self): - self.user.actor_as2 = json_dumps({ + self.user.actor_as2 = { **self.actor_as2, 'url': [ 'https://foo.com/about-me', 'acct:notthisuser@boop.org', 'acct:customuser@foo.com', ], - }) + } self.user.put() self.expected_webfinger.update({ diff --git a/tests/test_webmention.py b/tests/test_webmention.py index 810ffaa..d7a1dc5 100644 --- a/tests/test_webmention.py +++ b/tests/test_webmention.py @@ -500,7 +500,7 @@ class WebmentionTest(testutil.TestCase): }], } with app.test_request_context('/'): - Object(id='http://a/reply', status='complete', mf2=json_dumps(mf2)).put() + Object(id='http://a/reply', status='complete', mf2=mf2).put() mock_get.side_effect = self.activitypub_gets mock_post.return_value = requests_response('abc xyz') @@ -518,7 +518,7 @@ class WebmentionTest(testutil.TestCase): def test_redo_repost_isnt_update(self, mock_get, mock_post): """Like and Announce shouldn't use Update, they should just resend as is.""" with app.test_request_context('/'): - Object(id='http://a/repost', mf2='{}', status='complete').put() + Object(id='http://a/repost', mf2={}, status='complete').put() mock_get.side_effect = [self.repost, self.orig_as2, self.actor] mock_post.return_value = requests_response('abc xyz') @@ -536,8 +536,7 @@ class WebmentionTest(testutil.TestCase): def test_skip_update_if_content_unchanged(self, mock_get, mock_post): """https://github.com/snarfed/bridgy-fed/issues/78""" with app.test_request_context('/'): - Object(id='http://a/reply', status='complete', - mf2=json_dumps(self.reply_mf2['items'][0]), + Object(id='http://a/reply', status='complete', mf2=self.reply_mf2['items'][0], delivered=[Target(uri='https://foo.com/inbox', protocol='activitypub')] ).put() mock_get.side_effect = self.activitypub_gets @@ -725,30 +724,30 @@ class WebmentionTest(testutil.TestCase): def make_followers(): Follower.get_or_create('orig', 'https://mastodon/aaa') Follower.get_or_create('orig', 'https://mastodon/bbb', - last_follow=json_dumps({'actor': { + last_follow={'actor': { 'publicInbox': 'https://public/inbox', 'inbox': 'https://unused', - }})) + }}) Follower.get_or_create('orig', 'https://mastodon/ccc', - last_follow=json_dumps({'actor': { + last_follow={'actor': { 'endpoints': { 'sharedInbox': 'https://shared/inbox', }, - }})) + }}) Follower.get_or_create('orig', 'https://mastodon/ddd', - last_follow=json_dumps({'actor': { + last_follow={'actor': { 'inbox': 'https://inbox', - }})) + }}) Follower.get_or_create('orig', 'https://mastodon/ggg', status='inactive', - last_follow=json_dumps({'actor': { + last_follow={'actor': { 'inbox': 'https://unused/2', - }})) + }}) Follower.get_or_create('orig', 'https://mastodon/hhh', - last_follow=json_dumps({'actor': { + last_follow={'actor': { # dupe of eee; should be de-duped 'inbox': 'https://inbox', - }})) + }}) def test_create_post_run_task_new(self, mock_get, mock_post): mock_get.side_effect = [self.create, self.actor] @@ -784,7 +783,7 @@ class WebmentionTest(testutil.TestCase): with app.test_request_context('/'): Object(id='https://orig/post', domains=['orig'], status='in progress', - mf2=json_dumps(self.create_mf2['items'][0]), + mf2=self.create_mf2['items'][0], delivered=[Target(uri='https://skipped/inbox', protocol='activitypub')], undelivered=[Target(uri='https://shared/inbox', protocol='activitypub')], failed=[Target(uri='https://public/inbox', protocol='activitypub')], @@ -793,9 +792,9 @@ class WebmentionTest(testutil.TestCase): self.make_followers() # already sent, should be skipped Follower.get_or_create('orig', 'https://mastodon/eee', - last_follow=json_dumps({'actor': { + last_follow={'actor': { 'inbox': 'https://skipped/inbox', - }})) + }}) got = self.client.post('/_ah/queue/webmention', data={ 'source': 'https://orig/post', @@ -827,7 +826,7 @@ class WebmentionTest(testutil.TestCase): with app.test_request_context('/'): Object(id='https://orig/post', domains=['orig'], status='in progress', - mf2=json_dumps({**self.create_mf2, 'content': 'different'}), + mf2={**self.create_mf2, 'content': 'different'}, delivered=[Target(uri='https://delivered/inbox', protocol='activitypub')], undelivered=[Target(uri='https://shared/inbox', protocol='activitypub')], failed=[Target(uri='https://public/inbox', protocol='activitypub')], @@ -872,7 +871,7 @@ class WebmentionTest(testutil.TestCase): Follower.get_or_create( 'orig', 'https://mastodon/aaa', - last_follow=json_dumps({'actor': {'inbox': 'https://inbox'}})) + last_follow={'actor': {'inbox': 'https://inbox'}}) got = self.client.post('/_ah/queue/webmention', data={ 'source': 'https://orig/post', @@ -930,10 +929,10 @@ class WebmentionTest(testutil.TestCase): self.assertEqual('https://foo.com/about-me a', followers[0].key.id()) self.assertEqual('a', followers[0].src) self.assertEqual('https://foo.com/about-me', followers[0].dest) - self.assertEqual(self.follow_as2, json_loads(followers[0].last_follow)) + self.assertEqual(self.follow_as2, followers[0].last_follow) def test_follow_no_actor(self, mock_get, mock_post): - self.user_orig.actor_as2 = json_dumps(self.follow_as2['actor']) + self.user_orig.actor_as2 = self.follow_as2['actor'] self.user_orig.put() html = self.follow_html.replace( @@ -1086,15 +1085,15 @@ class WebmentionTest(testutil.TestCase): mock_get.side_effect = [self.author] mock_post.return_value = requests_response('abc xyz') Follower.get_or_create('orig', 'https://mastodon/ccc', - last_follow=json_dumps({'actor': { + last_follow={'actor': { 'endpoints': { 'sharedInbox': 'https://shared/inbox', }, - }})) + }}) Follower.get_or_create('orig', 'https://mastodon/ddd', - last_follow=json_dumps({'actor': { + last_follow={'actor': { 'inbox': 'https://inbox', - }})) + }}) got = self.client.post('/_ah/queue/webmention', data={ 'source': 'https://orig/', @@ -1120,7 +1119,7 @@ class WebmentionTest(testutil.TestCase): 'to': ['https://www.w3.org/ns/activitystreams#Public'], }) - got_as2 = json_loads(Object.get_by_id(id).as2) + got_as2 = Object.get_by_id(id).as2 self.assert_object(id, domains=['orig'], source_protocol='webmention', diff --git a/tests/test_xrpc_actor.py b/tests/test_xrpc_actor.py index 060b5ba..3af425b 100644 --- a/tests/test_xrpc_actor.py +++ b/tests/test_xrpc_actor.py @@ -1,7 +1,6 @@ """Unit tests for actor.py.""" from oauth_dropins.webutil import util from oauth_dropins.webutil.testutil import requests_response -from oauth_dropins.webutil.util import json_dumps, json_loads import requests from . import testutil @@ -17,8 +16,7 @@ class XrpcActorTest(testutil.TestCase): 'summary': "I'm a person", 'image': [{'type': 'Image', 'url': 'http://foo.com/header.png'}], } - User.get_or_create('foo.com', has_hcard=True, - actor_as2=json_dumps(actor)).put() + User.get_or_create('foo.com', has_hcard=True, actor_as2=actor).put() resp = self.client.get('/xrpc/app.bsky.actor.getProfile', query_string={'actor': 'foo.com'}) diff --git a/tests/test_xrpc_feed.py b/tests/test_xrpc_feed.py index 0f40f17..d0cb2b3 100644 --- a/tests/test_xrpc_feed.py +++ b/tests/test_xrpc_feed.py @@ -14,7 +14,6 @@ from granary.tests.test_bluesky import ( ) from oauth_dropins.webutil import util from oauth_dropins.webutil.testutil import requests_response -from oauth_dropins.webutil.util import json_dumps, json_loads import requests from werkzeug.exceptions import BadGateway @@ -81,17 +80,16 @@ class XrpcFeedTest(testutil.TestCase): def setUp(self): super().setUp() - User.get_or_create('foo.com', has_hcard=True, - actor_as2=json_dumps(ACTOR)).put() + User.get_or_create('foo.com', has_hcard=True, actor_as2=ACTOR).put() def test_getAuthorFeed(self): - post_as2 = json_dumps(as2.from_as1(POST_AS)) + post_as2 = as2.from_as1(POST_AS) with app.test_request_context('/'): Object(id='a', domains=['foo.com'], labels=['user'], as2=post_as2).put() Object(id='b', domains=['foo.com'], labels=['user'], - as2=json_dumps(as2.from_as1(REPLY_AS))).put() + as2=as2.from_as1(REPLY_AS)).put() Object(id='c', domains=['foo.com'], labels=['user'], - as2=json_dumps(as2.from_as1(REPOST_AS))).put() + as2=as2.from_as1(REPOST_AS)).put() # not outbound from user Object(id='d', domains=['foo.com'], labels=['feed'], as2=post_as2).put() # deleted @@ -130,7 +128,7 @@ class XrpcFeedTest(testutil.TestCase): def test_getPostThread(self): with app.test_request_context('/'): Object(id='http://a/post', domains=['foo.com'], labels=['user'], - as2=json_dumps(as2.from_as1(POST_THREAD_AS))).put() + as2=as2.from_as1(POST_THREAD_AS)).put() resp = self.client.get('/xrpc/app.bsky.feed.getPostThread', query_string={'uri': 'http://a/post'}) @@ -148,15 +146,15 @@ class XrpcFeedTest(testutil.TestCase): def test_getRepostedBy(self): with app.test_request_context('/'): - Object(id='repost/1', domains=['foo.com'], as2=json_dumps(as2.from_as1({ + Object(id='repost/1', domains=['foo.com'], as2=as2.from_as1({ **REPOST_AS, 'object': 'http://a/post', - }))).put() - Object(id='repost/2', domains=['foo.com'], as2=json_dumps(as2.from_as1({ + })).put() + Object(id='repost/2', domains=['foo.com'], as2=as2.from_as1({ **REPOST_AS, 'object': 'http://a/post', 'actor': as2.to_as1(ACTOR), - }))).put() + })).put() got = self.client.get('/xrpc/app.bsky.feed.getRepostedBy', query_string={'uri': 'http://a/post'}) diff --git a/tests/test_xrpc_graph.py b/tests/test_xrpc_graph.py index dc21c25..d32ea59 100644 --- a/tests/test_xrpc_graph.py +++ b/tests/test_xrpc_graph.py @@ -1,7 +1,6 @@ """Unit tests for graph.py.""" from granary import bluesky from oauth_dropins.webutil.testutil import requests_response -from oauth_dropins.webutil.util import json_dumps, json_loads import requests from .test_activitypub import ACTOR, FOLLOW, FOLLOW_WITH_ACTOR, FOLLOW_WITH_OBJECT @@ -77,11 +76,11 @@ class XrpcGraphTest(testutil.TestCase): Follower.get_or_create('foo.com', 'https://no/stored/follow') Follower.get_or_create('foo.com', 'https://masto/user', - last_follow=json_dumps(FOLLOW_WITH_ACTOR)) + last_follow=FOLLOW_WITH_ACTOR) Follower.get_or_create('foo.com', 'http://other', - last_follow=json_dumps(other_follow)) + last_follow=other_follow) Follower.get_or_create('nope.com', 'http://nope', - last_follow=json_dumps(other_follow)) + last_follow=other_follow) resp = self.client.get('/xrpc/app.bsky.graph.getFollowers', query_string={'user': 'foo.com'}) @@ -122,11 +121,11 @@ class XrpcGraphTest(testutil.TestCase): Follower.get_or_create('https://no/stored/follow', 'foo.com') Follower.get_or_create('https://masto/user', 'foo.com', - last_follow=json_dumps(FOLLOW_WITH_OBJECT)) + last_follow=FOLLOW_WITH_OBJECT) Follower.get_or_create( 'http://other', 'foo.com', - last_follow=json_dumps(other_follow)) + last_follow=other_follow) Follower.get_or_create('http://nope', 'nope.com', - last_follow=json_dumps(other_follow)) + last_follow=other_follow) resp = self.client.get('/xrpc/app.bsky.graph.getFollows', query_string={'user': 'foo.com'}) diff --git a/tests/testutil.py b/tests/testutil.py index f8f2a3f..056d138 100644 --- a/tests/testutil.py +++ b/tests/testutil.py @@ -14,7 +14,6 @@ from granary.tests.test_as1 import ( from oauth_dropins.webutil import testutil, util from oauth_dropins.webutil.appengine_config import ndb_client from oauth_dropins.webutil.testutil import requests_response -from oauth_dropins.webutil.util import json_dumps, json_loads import requests from app import app, cache @@ -52,19 +51,19 @@ class TestCase(unittest.TestCase, testutil.Asserts): with app.test_request_context('/'): # post Object(id='a', domains=['foo.com'], labels=['feed', 'notification'], - as2=json_dumps(as2.from_as1(NOTE))).put() + as2=as2.from_as1(NOTE)).put() # different domain Object(id='b', domains=['bar.org'], labels=['feed', 'notification'], - as2=json_dumps(as2.from_as1(MENTION))).put() + as2=as2.from_as1(MENTION)).put() # reply Object(id='d', domains=['foo.com'], labels=['feed', 'notification'], - as2=json_dumps(as2.from_as1(COMMENT))).put() + as2=as2.from_as1(COMMENT)).put() # not feed/notif Object(id='e', domains=['foo.com'], - as2=json_dumps(as2.from_as1(NOTE))).put() + as2=as2.from_as1(NOTE)).put() # deleted Object(id='f', domains=['foo.com'], labels=['feed', 'notification', 'user'], - as2=json_dumps(as2.from_as1(NOTE)), deleted=True).put() + as2=as2.from_as1(NOTE), deleted=True).put() def req(self, url, **kwargs): """Returns a mock requests call.""" @@ -112,14 +111,6 @@ class TestCase(unittest.TestCase, testutil.Asserts): if mf2 and 'items' in mf2: props['mf2'] = mf2['items'][0] - # sort keys in JSON properties - for prop in 'as2', 'bsky', 'mf2': - if prop in props: - props[prop] = json_dumps(props[prop], sort_keys=True) - got_val = getattr(got, prop, None) - if got_val: - setattr(got, prop, json_dumps(json_loads(got_val), sort_keys=True)) - for computed in 'type', 'object_ids': val = props.pop(computed, None) if val is not None: diff --git a/webmention.py b/webmention.py index 71c109b..707f4db 100644 --- a/webmention.py +++ b/webmention.py @@ -171,9 +171,9 @@ class Webmention(View): labels=['user'], ) if self.source_as2: - obj.as2 = json_dumps(common.redirect_unwrap(self.source_as2)) + obj.as2 = common.redirect_unwrap(self.source_as2) else: - obj.mf2 = json_dumps(self.source_mf2) + obj.mf2 = self.source_mf2 if self.source_as1.get('objectType') == 'activity': obj.labels.append('activity') @@ -214,7 +214,7 @@ class Webmention(View): dest = ((target_as2.get('id') or util.get_first(target_as2, 'url')) if target_as2 else util.get_url(self.source_as1, 'object')) Follower.get_or_create(dest=dest, src=self.source_domain, - last_follow=json_dumps(self.source_as2)) + last_follow=self.source_as2) try: last = common.signed_post(inbox, user=self.user, data=self.source_as2, @@ -290,7 +290,7 @@ class Webmention(View): Follower.key > Key('Follower', self.source_domain + ' '), Follower.key < Key('Follower', self.source_domain + chr(ord(' ') + 1))): if follower.status != 'inactive' and follower.last_follow: - actor = json_loads(follower.last_follow).get('actor') + actor = follower.last_follow.get('actor') if actor and isinstance(actor, dict): inboxes.add(actor.get('endpoints', {}).get('sharedInbox') or actor.get('publicInbox') or @@ -306,8 +306,7 @@ class Webmention(View): for target in targets: # fetch target page as AS2 object try: - target_obj = json_loads( - common.get_object(target, user=self.user).as2) + target_obj = common.get_object(target, user=self.user).as2 except (requests.HTTPError, BadGateway) as e: resp = getattr(e, 'requests_response', None) if resp and resp.ok: @@ -330,7 +329,7 @@ class Webmention(View): if not inbox_url: # fetch actor as AS object - actor = json_loads(common.get_object(actor, user=self.user).as2) + actor = common.get_object(actor, user=self.user).as2 inbox_url = actor.get('inbox') if not inbox_url: