From 95e46c5ebb738fb0ce46f1c6fd9e02324a256cb2 Mon Sep 17 00:00:00 2001 From: Ryan Barrett Date: Fri, 8 Mar 2024 13:24:28 -0800 Subject: [PATCH] Revert "cache outbound HTTP request responses, locally to each inbound request" This reverts commit 30debfc8faf730190bd51a3aef49df6c6bfbd50a. seemed promising, but broke in production. Saw a lot of `IncompleteRead`s on both GETs and POSTs. Rolled back for now. ``` ('Connection broken: IncompleteRead(9172 bytes read, -4586 more expected)', IncompleteRead(9172 bytes read, -4586 more expected)) ... File "oauth_dropins/webutil/util.py", line 1673, in call resp = getattr((session or requests), fn)(url, *args, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "requests_cache/session.py", line 102, in get return self.request('GET', url, params=params, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "requests_cache/session.py", line 158, in request return super().request(method, url, *args, headers=headers, **kwargs) # type: ignore ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "requests/sessions.py", line 589, in request resp = self.send(prep, **send_kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "requests_cache/session.py", line 205, in send response = self._send_and_cache(request, actions, cached_response, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "requests_cache/session.py", line 233, in _send_and_cache self.cache.save_response(response, actions.cache_key, actions.expires) File "requests_cache/backends/base.py", line 89, in save_response cached_response = CachedResponse.from_response(response, expires=expires) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "requests_cache/models/response.py", line 102, in from_response obj.raw = CachedHTTPResponse.from_response(response) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "requests_cache/models/raw_response.py", line 69, in from_response _ = response.content # This property reads, decodes, and stores response content ^^^^^^^^^^^^^^^^ File "requests/models.py", line 899, in content self._content = b"".join(self.iter_content(CONTENT_CHUNK_SIZE)) or b"" ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "requests/models.py", line 818, in generate raise ChunkedEncodingError(e) ``` --- activitypub.py | 8 +- atproto.py | 8 +- common.py | 33 +------- requirements.txt | 1 - tests/test_activitypub.py | 60 +++++++++------ tests/test_atproto.py | 80 +++++++++---------- tests/test_convert.py | 12 +-- tests/test_follow.py | 154 +++++++++++++++++++++---------------- tests/test_integrations.py | 2 +- tests/test_models.py | 9 +-- tests/test_pages.py | 2 +- tests/test_protocol.py | 14 ++-- tests/test_redirect.py | 6 +- tests/test_web.py | 8 +- tests/test_webfinger.py | 10 +-- web.py | 14 ++-- webfinger.py | 2 +- 17 files changed, 207 insertions(+), 216 deletions(-) diff --git a/activitypub.py b/activitypub.py index edd2252..256821a 100644 --- a/activitypub.py +++ b/activitypub.py @@ -479,19 +479,19 @@ class ActivityPub(User, Protocol): def signed_get(url, from_user=None, **kwargs): - return signed_request(common.requests_get, url, from_user=from_user, **kwargs) + return signed_request(util.requests_get, url, from_user=from_user, **kwargs) def signed_post(url, from_user, **kwargs): assert from_user - return signed_request(common.requests_post, url, from_user=from_user, **kwargs) + return signed_request(util.requests_post, url, from_user=from_user, **kwargs) def signed_request(fn, url, data=None, headers=None, from_user=None, **kwargs): """Wraps ``requests.*`` and adds HTTP Signature. Args: - fn (callable): :func:`common.requests_get` or :func:`common.requests_post` + fn (callable): :func:`util.requests_get` or :func:`util.requests_post` url (str): data (dict): optional AS2 object from_user (models.User): user to sign request as; optional. If not @@ -544,7 +544,7 @@ def signed_request(fn, url, data=None, headers=None, from_user=None, **kwargs): logger.info(f'Got {resp.status_code} headers: {resp.headers}') # handle GET redirects manually so that we generate a new HTTP signature - if resp.is_redirect and fn == common.requests_get: + if resp.is_redirect and fn == util.requests_get: new_url = urljoin(url, resp.headers['Location']) return signed_request(fn, new_url, data=data, headers=headers, **kwargs) diff --git a/atproto.py b/atproto.py index d431903..a572b51 100644 --- a/atproto.py +++ b/atproto.py @@ -109,7 +109,7 @@ class ATProto(User, Protocol): if user: return user.key.id() - return did.resolve_handle(handle, get_fn=common.requests_get) + return did.resolve_handle(handle, get_fn=util.requests_get) def profile_id(self): return f'at://{self.key.id()}/app.bsky.actor.profile/self' @@ -201,7 +201,7 @@ class ATProto(User, Protocol): logger.info(f'Creating new did:plc for {user.key}') did_plc = did.create_plc(user.handle_as('atproto'), pds_url=cls.PDS_URL, - post_fn=common.requests_post) + post_fn=util.requests_post) Object.get_or_create(did_plc.did, raw=did_plc.doc) # TODO: move this to ATProto.get_or_create? @@ -355,7 +355,7 @@ class ATProto(User, Protocol): # did:plc, did:web if id.startswith('did:'): try: - obj.raw = did.resolve(id, get_fn=common.requests_get) + obj.raw = did.resolve(id, get_fn=util.requests_get) return True except (ValueError, requests.RequestException) as e: util.interpret_http_exception(e) @@ -421,7 +421,7 @@ class ATProto(User, Protocol): for url in util.get_urls(o, 'image'): if url not in blobs: blob = AtpRemoteBlob.get_or_create( - url=url, get_fn=common.requests_get) + url=url, get_fn=util.requests_get) blobs[url] = blob.as_object() ret = bluesky.from_as1(cls.translate_ids(obj.as1), blobs=blobs) diff --git a/common.py b/common.py index edd05cf..12cfeee 100644 --- a/common.py +++ b/common.py @@ -17,7 +17,6 @@ from oauth_dropins.webutil.appengine_config import tasks_client from oauth_dropins.webutil import appengine_info from oauth_dropins.webutil.appengine_info import DEBUG from oauth_dropins.webutil import flask_util -from requests_cache import CachedSession logger = logging.getLogger(__name__) @@ -244,7 +243,7 @@ def webmention_endpoint_cache_key(url): lock=threading.Lock()) def webmention_discover(url, **kwargs): """Thin caching wrapper around :func:`oauth_dropins.webutil.webmention.discover`.""" - return webmention.discover(url, session=requests_session(), **kwargs) + return webmention.discover(url, **kwargs) def add(seq, val): @@ -325,33 +324,3 @@ def create_task(queue, delay=None, **params): msg = f'Added {queue} task {task.name} : {params}' logger.info(msg) return msg, 202 - - -def requests_session(): - if not hasattr(g, 'requests_session'): - g.session = CachedSession( - # more or less everything - allowable_codes=(200, 204, 205, 206, - 300, 301, 303, 304, 308, - 400, 401, 402, 403, 404, 405, 406, 408, 410, 411, - 412, 413, 414, 415, 417, 422, 428, 429, 431, 451, - 500, 501, 502, 503, 504, 511), - backend='memory', # BaseCache class - match_headers=('Accept',), - ) - - return g.session - - -def requests_get(*args, **kwargs): - return util.requests_get(*args, session=requests_session(), **kwargs) - -def requests_post(*args, **kwargs): - return util.requests_post(*args, session=requests_session(), **kwargs) - -def requests_head(*args, **kwargs): - return util.requests_head(*args, session=requests_session(), **kwargs) - - -def fetch_mf2(*args, **kwargs): - return util.fetch_mf2(*args, session=requests_session(), **kwargs) diff --git a/requirements.txt b/requirements.txt index 9e0395b..eadb48a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -88,7 +88,6 @@ pytz==2024.1 PyYAML==6.0.1 redis==5.0.3 requests==2.31.0 -requests-cache==1.2.0 requests-oauthlib==1.3.1 rsa==4.9 sgmllib3k==1.0.0 diff --git a/tests/test_activitypub.py b/tests/test_activitypub.py index e5633ec..33a145f 100644 --- a/tests/test_activitypub.py +++ b/tests/test_activitypub.py @@ -299,9 +299,9 @@ AS2 = requests_response(AS2_OBJ, headers={ NOT_ACCEPTABLE = requests_response(status=406) -@patch('requests_cache.CachedSession.post') -@patch('requests_cache.CachedSession.get') -@patch('requests_cache.CachedSession.head') +@patch('requests.post') +@patch('requests.get') +@patch('requests.head') class ActivityPubTest(TestCase): def setUp(self): @@ -599,6 +599,7 @@ class ActivityPubTest(TestCase): ) def _test_inbox_reply(self, reply, mock_head, mock_get, mock_post): + mock_head.return_value = requests_response(url='https://user.com/post') mock_get.side_effect = ( (list(mock_get.side_effect) if mock_get.side_effect else [self.as2_resp(ACTOR)]) @@ -686,6 +687,7 @@ class ActivityPubTest(TestCase): baj = self.make_user('fake:baj', cls=Fake, obj_id='fake:baj') Follower.get_or_create(to=swentel, from_=baj, status='inactive') + mock_head.return_value = requests_response(url='http://target') mock_get.return_value = self.as2_resp(ACTOR) # source actor mock_post.return_value = requests_response() @@ -723,6 +725,7 @@ class ActivityPubTest(TestCase): delivered_protocol='fake') def test_repost_of_indieweb(self, mock_head, mock_get, mock_post): + mock_head.return_value = requests_response(url='https://user.com/orig') mock_get.return_value = WEBMENTION_DISCOVERY mock_post.return_value = requests_response() # webmention @@ -772,6 +775,7 @@ class ActivityPubTest(TestCase): baj = self.make_user('fake:baj', cls=Fake, obj_id='fake:baj') Follower.get_or_create(to=to, from_=baj, status='inactive') + mock_head.return_value = requests_response(url='http://target') mock_get.return_value = self.as2_resp(NOTE_OBJECT) got = self.post('/ap/sharedInbox', json=REPOST) @@ -830,6 +834,7 @@ class ActivityPubTest(TestCase): Follower.get_or_create(to=self.make_user(ACTOR['id'], cls=ActivityPub), from_=self.user) + mock_head.return_value = requests_response(url='http://target') mock_get.return_value = self.as2_resp(ACTOR) # source actor not_public = copy.deepcopy(NOTE) @@ -854,6 +859,7 @@ class ActivityPubTest(TestCase): self.assertIsNone(Object.get_by_id('http://inst/foo')) def test_inbox_like(self, mock_head, mock_get, mock_post): + mock_head.return_value = requests_response(url='https://user.com/post') mock_get.side_effect = [ # source actor self.as2_resp(LIKE_WITH_ACTOR['actor']), @@ -991,6 +997,7 @@ class ActivityPubTest(TestCase): self.user.direct = False self.user.put() + mock_head.return_value = requests_response(url='https://user.com/') mock_get.side_effect = [ self.as2_resp(ACTOR), # source actor WEBMENTION_DISCOVERY, @@ -1036,6 +1043,7 @@ class ActivityPubTest(TestCase): def test_inbox_follow_use_instead_strip_www(self, mock_head, mock_get, mock_post): self.make_user('www.user.com', cls=Web, use_instead=self.user.key) + mock_head.return_value = requests_response(url='https://www.user.com/') mock_get.side_effect = [ # source actor self.as2_resp(ACTOR), @@ -1067,6 +1075,7 @@ class ActivityPubTest(TestCase): follower.follow.get().as2['url']) def test_inbox_follow_web_brid_gy_subdomain(self, mock_head, mock_get, mock_post): + mock_head.return_value = requests_response(url='https://user.com/') mock_get.side_effect = [ # source actor self.as2_resp(ACTOR), @@ -1124,6 +1133,7 @@ class ActivityPubTest(TestCase): from_=self.make_user(ACTOR['id'], cls=ActivityPub, obj_as2=ACTOR), status='inactive') + mock_head.return_value = requests_response(url='https://user.com/') mock_get.side_effect = [ test_web.ACTOR_HTML_RESP, WEBMENTION_DISCOVERY, @@ -1137,6 +1147,7 @@ class ActivityPubTest(TestCase): self.assertEqual('active', follower.key.get().status) def test_inbox_undo_follow_doesnt_exist(self, mock_head, mock_get, mock_post): + mock_head.return_value = requests_response(url='https://user.com/') mock_get.side_effect = [ self.as2_resp(ACTOR), test_web.ACTOR_HTML_RESP, @@ -1148,6 +1159,7 @@ class ActivityPubTest(TestCase): self.assertEqual(202, got.status_code) def test_inbox_undo_follow_inactive(self, mock_head, mock_get, mock_post): + mock_head.return_value = requests_response(url='https://user.com/') mock_get.side_effect = [ self.as2_resp(ACTOR), test_web.ACTOR_HTML_RESP, @@ -1164,6 +1176,7 @@ class ActivityPubTest(TestCase): self.assertEqual('inactive', follower.key.get().status) def test_inbox_undo_follow_composite_object(self, mock_head, mock_get, mock_post): + mock_head.return_value = requests_response(url='https://user.com/') mock_get.side_effect = [ self.as2_resp(ACTOR), test_web.ACTOR_HTML_RESP, @@ -1807,7 +1820,7 @@ class ActivityPubUtilsTest(TestCase): self.assertEqual('http://inst.com/@user', ActivityPub.handle_to_id('@user@inst.com')) - @patch('requests_cache.CachedSession.get') + @patch('requests.get') def test_handle_to_id_fetch(self, mock_get): mock_get.return_value = requests_response(test_webfinger.WEBFINGER) self.assertEqual('http://localhost/user.com', @@ -1816,7 +1829,7 @@ class ActivityPubUtilsTest(TestCase): mock_get, 'https://inst.com/.well-known/webfinger?resource=acct:user@inst.com') - @patch('requests_cache.CachedSession.get', return_value=requests_response({})) + @patch('requests.get', return_value=requests_response({})) def test_handle_to_id_not_found(self, mock_get): self.assertIsNone(ActivityPub.handle_to_id('@user@inst.com')) self.assert_req( @@ -1977,7 +1990,7 @@ class ActivityPubUtilsTest(TestCase): self.assertEqual(['https://masto.foo/@other'], postprocess_as2(obj)['cc']) - @patch('requests_cache.CachedSession.get') + @patch('requests.get') def test_signed_get_redirects_manually_with_new_sig_headers(self, mock_get): mock_get.side_effect = [ requests_response(status=302, redirected_url='http://second', @@ -1990,7 +2003,7 @@ class ActivityPubUtilsTest(TestCase): second = mock_get.call_args_list[1][1] self.assertNotEqual(first['headers'], second['headers']) - @patch('requests_cache.CachedSession.get') + @patch('requests.get') def test_signed_get_redirects_to_relative_url(self, mock_get): mock_get.side_effect = [ # redirected URL is relative, we have to resolve it @@ -2011,7 +2024,7 @@ class ActivityPubUtilsTest(TestCase): first['auth'].header_signer.sign(first['headers'], method='GET', path='/'), second['auth'].header_signer.sign(second['headers'], method='GET', path='/')) - @patch('requests_cache.CachedSession.post', return_value=requests_response(status=200)) + @patch('requests.post', return_value=requests_response(status=200)) def test_signed_post_from_user_is_activitypub_use_instance_actor(self, mock_post): activitypub.signed_post('https://url', from_user=ActivityPub(id='http://fed')) @@ -2021,7 +2034,7 @@ class ActivityPubUtilsTest(TestCase): rsa_key = kwargs['auth'].header_signer._rsa._key self.assertEqual(instance_actor().private_pem(), rsa_key.exportKey()) - @patch('requests_cache.CachedSession.post') + @patch('requests.post') def test_signed_post_ignores_redirect(self, mock_post): mock_post.side_effect = [ requests_response(status=302, redirected_url='http://second', @@ -2032,7 +2045,7 @@ class ActivityPubUtilsTest(TestCase): mock_post.assert_called_once() self.assertEqual(302, resp.status_code) - @patch('requests_cache.CachedSession.get') + @patch('requests.get') def test_fetch_direct(self, mock_get): mock_get.return_value = AS2 obj = Object(id='http://orig') @@ -2043,7 +2056,7 @@ class ActivityPubUtilsTest(TestCase): self.as2_req('http://orig'), )) - @patch('requests_cache.CachedSession.get') + @patch('requests.get') def test_fetch_direct_ld_content_type(self, mock_get): mock_get.return_value = requests_response(AS2_OBJ, headers={ 'Content-Type': 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"', @@ -2056,7 +2069,7 @@ class ActivityPubUtilsTest(TestCase): self.as2_req('http://orig'), )) - @patch('requests_cache.CachedSession.get') + @patch('requests.get') def test_fetch_via_html(self, mock_get): mock_get.side_effect = [HTML_WITH_AS2, AS2] obj = Object(id='http://orig') @@ -2068,7 +2081,7 @@ class ActivityPubUtilsTest(TestCase): self.as2_req('http://as2', headers=as2.CONNEG_HEADERS), )) - @patch('requests_cache.CachedSession.get') + @patch('requests.get') def test_fetch_only_html(self, mock_get): mock_get.return_value = HTML @@ -2076,7 +2089,7 @@ class ActivityPubUtilsTest(TestCase): self.assertFalse(ActivityPub.fetch(obj)) self.assertIsNone(obj.as1) - @patch('requests_cache.CachedSession.get') + @patch('requests.get') def test_fetch_not_acceptable(self, mock_get): mock_get.return_value = NOT_ACCEPTABLE @@ -2084,13 +2097,13 @@ class ActivityPubUtilsTest(TestCase): self.assertFalse(ActivityPub.fetch(obj)) self.assertIsNone(obj.as1) - @patch('requests_cache.CachedSession.get') + @patch('requests.get') def test_fetch_ssl_error(self, mock_get): mock_get.side_effect = requests.exceptions.SSLError with self.assertRaises(BadGateway): ActivityPub.fetch(Object(id='http://orig')) - @patch('requests_cache.CachedSession.get') + @patch('requests.get') def test_fetch_no_content(self, mock_get): mock_get.return_value = self.as2_resp('') @@ -2099,7 +2112,7 @@ class ActivityPubUtilsTest(TestCase): mock_get.assert_has_calls([self.as2_req('http://the/id')]) - @patch('requests_cache.CachedSession.get') + @patch('requests.get') def test_fetch_not_json(self, mock_get): mock_get.return_value = self.as2_resp('XYZ not JSON') @@ -2145,9 +2158,8 @@ class ActivityPubUtilsTest(TestCase): def test_convert_actor_as2(self): self.assert_equals(ACTOR, ActivityPub.convert(Object(as2=ACTOR))) - @patch('requests_cache.CachedSession.get') - def test_convert_actor_as1_from_user(self, mock_get): - mock_get.return_value = requests_response(test_web.ACTOR_HTML) + @patch('requests.get', return_value=requests_response(test_web.ACTOR_HTML)) + def test_convert_actor_as1_from_user(self, _): obj = Object(our_as1={ 'objectType': 'person', 'id': 'https://user.com/', @@ -2305,7 +2317,7 @@ class ActivityPubUtilsTest(TestCase): obj.as2['actor']['url'] = [obj.as2['actor'].pop('id')] self.assertEqual('http://mas.to/inbox', ActivityPub.target_for(obj)) - @patch('requests_cache.CachedSession.get') + @patch('requests.get') def test_target_for_object_fetch(self, mock_get): mock_get.return_value = self.as2_resp(ACTOR) @@ -2316,7 +2328,7 @@ class ActivityPubUtilsTest(TestCase): self.assertEqual('http://mas.to/inbox', ActivityPub.target_for(obj)) mock_get.assert_has_calls([self.as2_req('http://the/author')]) - @patch('requests_cache.CachedSession.get') + @patch('requests.get') def test_target_for_author_is_object_id(self, mock_get): mock_get.return_value = HTML @@ -2326,13 +2338,13 @@ class ActivityPubUtilsTest(TestCase): # test is that we short circuit out instead of infinite recursion self.assertIsNone(ActivityPub.target_for(obj)) - @patch('requests_cache.CachedSession.post') + @patch('requests.post') def test_send_blocklisted(self, mock_post): self.assertFalse(ActivityPub.send(Object(as2=NOTE), 'https://fed.brid.gy/ap/sharedInbox')) mock_post.assert_not_called() - @patch('requests_cache.CachedSession.post') + @patch('requests.post') def test_send_convert_ids(self, mock_post): mock_post.return_value = requests_response() diff --git a/tests/test_atproto.py b/tests/test_atproto.py index bd5e4c0..a0d668f 100644 --- a/tests/test_atproto.py +++ b/tests/test_atproto.py @@ -71,7 +71,7 @@ class ATProtoTest(TestCase): return user - @patch('requests_cache.CachedSession.get', return_value=requests_response(DID_DOC)) + @patch('requests.get', return_value=requests_response(DID_DOC)) def test_put_validates_id(self, mock_get): for bad in ( '', @@ -94,7 +94,7 @@ class ATProtoTest(TestCase): self.store_object(id='did:plc:user', raw=DID_DOC) self.assertEqual('han.dull', ATProto(id='did:plc:user').handle) - @patch('requests_cache.CachedSession.get', return_value=requests_response(DID_DOC)) + @patch('requests.get', return_value=requests_response(DID_DOC)) def test_get_or_create(self, _): user = self.make_user('did:plc:user', cls=ATProto) self.assertEqual('han.dull', user.key.get().handle) @@ -130,8 +130,7 @@ class ATProtoTest(TestCase): @patch('dns.resolver.resolve', side_effect=dns.resolver.NXDOMAIN()) # resolving handle, HTTPS method, not found - @patch('requests_cache.CachedSession.get', - return_value=requests_response('', status=404)) + @patch('requests.get', return_value=requests_response('', status=404)) def test_handle_to_id_not_found(self, *_): self.assertIsNone(ATProto.handle_to_id('han.dull')) @@ -148,7 +147,7 @@ class ATProtoTest(TestCase): got = ATProto.pds_for(Object(id='at://did:plc:user/co.ll/123')) self.assertEqual('https://some.pds', got) - @patch('requests_cache.CachedSession.get', return_value=requests_response(DID_DOC)) + @patch('requests.get', return_value=requests_response(DID_DOC)) def test_pds_for_fetch_did(self, mock_get): got = ATProto.pds_for(Object(id='at://did:plc:user/co.ll/123')) self.assertEqual('https://some.pds', got) @@ -180,7 +179,7 @@ class ATProtoTest(TestCase): self.assertEqual('https://some.pds', got) @patch('dns.resolver.resolve', side_effect=dns.resolver.NXDOMAIN()) - @patch('requests_cache.CachedSession.get', side_effect=[ + @patch('requests.get', side_effect=[ # resolving handle, HTTPS method requests_response('did:plc:user', content_type='text/plain'), # fetching DID doc @@ -219,8 +218,7 @@ class ATProtoTest(TestCase): Object(id='at://foo'))) self.assertIsNone(ATProto.target_for(Object(id='fake:post'))) - @patch('requests_cache.CachedSession.get', - return_value=requests_response({'foo': 'bar'})) + @patch('requests.get', return_value=requests_response({'foo': 'bar'})) def test_fetch_did_plc(self, mock_get): obj = Object(id='did:plc:123') self.assertTrue(ATProto.fetch(obj)) @@ -230,8 +228,7 @@ class ATProtoTest(TestCase): self.req('https://plc.local/did:plc:123'), )) - @patch('requests_cache.CachedSession.get', - return_value=requests_response({'foo': 'bar'})) + @patch('requests.get', return_value=requests_response({'foo': 'bar'})) def test_fetch_did_web(self, mock_get): obj = Object(id='did:web:user.com') self.assertTrue(ATProto.fetch(obj)) @@ -241,8 +238,7 @@ class ATProtoTest(TestCase): self.req('https://user.com/.well-known/did.json'), )) - @patch('requests_cache.CachedSession.get', - return_value=requests_response('not json')) + @patch('requests.get', return_value=requests_response('not json')) def test_fetch_did_plc_not_json(self, mock_get): obj = Object(id='did:web:user.com') self.assertFalse(ATProto.fetch(obj)) @@ -271,15 +267,17 @@ class ATProtoTest(TestCase): ) @patch('dns.resolver.resolve', side_effect=dns.resolver.NXDOMAIN()) - # resolve handle, HTTPS method - @patch('requests_cache.CachedSession.get', - return_value= requests_response('did:plc:abc', content_type='text/plain')) - # AppView getRecord - @patch('requests.get', return_value= requests_response({ - 'cid': 'bafy...', - 'value': {'foo': 'bar'}, - })) - def test_fetch_bsky_app_url(self, mock_get, _, __): + @patch('requests.get', side_effect=[ + # resolving handle, HTTPS method + + requests_response('did:plc:abc', content_type='text/plain'), + # AppView getRecord + requests_response({ + 'cid': 'bafy...', + 'value': {'foo': 'bar'}, + }), + ]) + def test_fetch_bsky_app_url(self, mock_get, _): obj = Object(id='https://bsky.app/profile/han.dull/post/789') self.assertTrue(ATProto.fetch(obj)) @@ -371,7 +369,7 @@ class ATProtoTest(TestCase): 'image': [{'url': 'http://my/pic'}], }))) - @patch('requests_cache.CachedSession.get', return_value=requests_response( + @patch('requests.get', return_value=requests_response( 'blob contents', content_type='image/png')) def test_convert_fetch_blobs_true(self, mock_get): cid = CID.decode('bafkreicqpqncshdd27sgztqgzocd3zhhqnnsv6slvzhs5uz6f57cq6lmtq') @@ -418,8 +416,7 @@ class ATProtoTest(TestCase): with self.assertRaises(BadRequest): ATProto.convert(obj) - @patch('requests_cache.CachedSession.get', - return_value=requests_response('', status=404)) + @patch('requests.get', return_value=requests_response('', status=404)) def test_web_url(self, mock_get): user = self.make_user('did:plc:user', cls=ATProto) self.assertEqual('https://bsky.app/profile/did:plc:user', user.web_url()) @@ -427,8 +424,7 @@ class ATProtoTest(TestCase): self.store_object(id='did:plc:user', raw=DID_DOC) self.assertEqual('https://bsky.app/profile/han.dull', user.web_url()) - @patch('requests_cache.CachedSession.get', - return_value=requests_response('', status=404)) + @patch('requests.get', return_value=requests_response('', status=404)) def test_handle_or_id(self, mock_get): user = self.make_user('did:plc:user', cls=ATProto) self.assertIsNone(user.handle) @@ -438,8 +434,7 @@ class ATProtoTest(TestCase): self.assertEqual('han.dull', user.handle) self.assertEqual('han.dull', user.handle_or_id()) - @patch('requests_cache.CachedSession.get', - return_value=requests_response('', status=404)) + @patch('requests.get', return_value=requests_response('', status=404)) def test_handle_as(self, mock_get): user = self.make_user('did:plc:user', cls=ATProto) @@ -450,7 +445,7 @@ class ATProtoTest(TestCase): self.store_object(id='did:plc:user', raw=DID_DOC) self.assertEqual('@han.dull@atproto.brid.gy', user.handle_as('activitypub')) - @patch('requests_cache.CachedSession.get', return_value=requests_response(DID_DOC)) + @patch('requests.get', return_value=requests_response(DID_DOC)) def test_profile_id(self, mock_get): self.assertEqual('at://did:plc:user/app.bsky.actor.profile/self', self.make_user('did:plc:user', cls=ATProto).profile_id()) @@ -458,7 +453,7 @@ class ATProtoTest(TestCase): @patch('atproto.DEBUG', new=False) @patch('google.cloud.dns.client.ManagedZone', autospec=True) @patch.object(tasks_client, 'create_task', return_value=Task(name='my task')) - @patch('requests_cache.CachedSession.post', + @patch('requests.post', return_value=requests_response('OK')) # create DID on PLC def test_create_for(self, mock_post, mock_create_task, mock_zone): mock_zone.return_value = zone = MagicMock() @@ -507,7 +502,7 @@ class ATProtoTest(TestCase): @patch('google.cloud.dns.client.ManagedZone', autospec=True) @patch.object(tasks_client, 'create_task', return_value=Task(name='my task')) - @patch('requests_cache.CachedSession.post', + @patch('requests.post', return_value=requests_response('OK')) # create DID on PLC def test_send_new_repo(self, mock_post, mock_create_task, _): user = self.make_user(id='fake:user', cls=Fake) @@ -569,11 +564,11 @@ class ATProtoTest(TestCase): self.assert_task(mock_create_task, 'atproto-commit', '/queue/atproto-commit') - @patch('requests_cache.CachedSession.get', return_value=requests_response( + @patch('requests.get', return_value=requests_response( 'blob contents', content_type='image/png')) # image blob fetch @patch('google.cloud.dns.client.ManagedZone', autospec=True) @patch.object(tasks_client, 'create_task', return_value=Task(name='my task')) - @patch('requests_cache.CachedSession.post', + @patch('requests.post', return_value=requests_response('OK')) # create DID on PLC def test_send_new_repo_includes_user_profile(self, mock_post, mock_create_task, _, __): @@ -854,8 +849,7 @@ class ATProtoTest(TestCase): @patch.object(tasks_client, 'create_task', return_value=Task(name='my task')) @patch('requests.get') - @patch('requests_cache.CachedSession.get', return_value=requests_response(DID_DOC)) - def test_poll_notifications(self, _, mock_get, mock_create_task): + def test_poll_notifications(self, mock_get, mock_create_task): user_a = self.make_user(id='fake:user-a', cls=Fake, copies=[Target(uri='did:plc:a', protocol='atproto')]) user_b = self.make_user(id='fake:user-b', cls=Fake, @@ -919,6 +913,7 @@ class ATProtoTest(TestCase): 'reason': 'reply', }], }), + requests_response(DID_DOC), requests_response({ 'cursor': '...', 'notifications': [{ @@ -947,8 +942,8 @@ class ATProtoTest(TestCase): assert mock_get.call_args_list[0].kwargs['headers'].pop('Authorization') self.assertEqual(expected_list_notifs, mock_get.call_args_list[0]) - assert mock_get.call_args_list[1].kwargs['headers'].pop('Authorization') - self.assertEqual(expected_list_notifs, mock_get.call_args_list[1]) + assert mock_get.call_args_list[2].kwargs['headers'].pop('Authorization') + self.assertEqual(expected_list_notifs, mock_get.call_args_list[2]) like_obj = Object.get_by_id('at://did:plc:d/app.bsky.feed.like/123') self.assertEqual(like, like_obj.bsky) @@ -967,11 +962,7 @@ class ATProtoTest(TestCase): @patch.object(tasks_client, 'create_task', return_value=Task(name='my task')) @patch('requests.get') - @patch('requests_cache.CachedSession.get', return_value=requests_response({ - **DID_DOC, - 'id': 'did:plc:alice.com', - })) - def test_poll_posts(self, _, mock_get, mock_create_task): + def test_poll_posts(self, mock_get, mock_create_task): user_a = self.make_user(id='fake:user-a', cls=Fake, copies=[Target(uri='did:plc:a', protocol='atproto')]) user_b = self.make_user(id='fake:user-b', cls=Fake, @@ -1007,6 +998,10 @@ class ATProtoTest(TestCase): 'post': post_view, }], }), + requests_response({ + **DID_DOC, + 'id': 'did:plc:alice.com', + }), requests_response({ 'cursor': '...', 'feed': [], @@ -1042,6 +1037,7 @@ class ATProtoTest(TestCase): }) self.assertEqual([ get_timeline, + self.req('https://alice.com/.well-known/did.json'), get_timeline, get_timeline, ], mock_get.call_args_list) diff --git a/tests/test_convert.py b/tests/test_convert.py index 322254f..9f1fde9 100644 --- a/tests/test_convert.py +++ b/tests/test_convert.py @@ -133,7 +133,7 @@ class ConvertTest(testutil.TestCase): base_url='https://ap.brid.gy/') self.assertEqual(404, resp.status_code) - @patch('requests_cache.CachedSession.get') + @patch('requests.get') def test_activitypub_to_web_fetch(self, mock_get): mock_get.return_value = self.as2_resp(as2.from_as1(COMMENT)) url = 'https://user.com/bar?baz=baj&biff' @@ -147,7 +147,7 @@ class ConvertTest(testutil.TestCase): mock_get.assert_has_calls((self.as2_req(url),)) - @patch('requests_cache.CachedSession.get') + @patch('requests.get') def test_activitypub_to_web_fetch_fails(self, mock_get): mock_get.side_effect = [requests_response('', status=405)] @@ -269,7 +269,7 @@ A ☕ reply # self.assertEqual(f'https://ap.brid.gy/convert/web/https:/foo%3Fbar%23baz', # resp.headers['Location']) - @patch('requests_cache.CachedSession.get') + @patch('requests.get') def test_web_to_activitypub_object(self, mock_get): mock_get.return_value = requests_response(HTML_NO_ID) @@ -282,7 +282,7 @@ A ☕ reply self.assertEqual(200, resp.status_code) self.assert_equals(COMMENT_AS2, resp.json, ignore=['to']) - @patch('requests_cache.CachedSession.get') + @patch('requests.get') def test_web_to_activitypub_fetch(self, mock_get): mock_get.return_value = requests_response(HTML) # protocol inference url = 'https://user.com/bar?baz=baj&biff' @@ -294,7 +294,7 @@ A ☕ reply self.assertEqual(200, resp.status_code) self.assert_equals(COMMENT_AS2, resp.json, ignore=['to']) - @patch('requests_cache.CachedSession.get') + @patch('requests.get') def test_web_to_activitypub_no_user(self, mock_get): hcard = requests_response(""" @@ -320,7 +320,7 @@ A ☕ reply 'attributedTo': 'https://web.brid.gy/nope.com', }, resp.json, ignore=['to']) - @patch('requests_cache.CachedSession.get') + @patch('requests.get') def test_web_to_activitypub_url_decode(self, mock_get): """https://github.com/snarfed/bridgy-fed/issues/581""" mock_get.return_value = requests_response(HTML_NO_ID) diff --git a/tests/test_follow.py b/tests/test_follow.py index eeac105..d98f425 100644 --- a/tests/test_follow.py +++ b/tests/test_follow.py @@ -60,7 +60,7 @@ UNDO_FOLLOW = { del UNDO_FOLLOW['object']['id'] -@patch('requests_cache.CachedSession.get') +@patch('requests.get') class RemoteFollowTest(TestCase): def setUp(self): @@ -133,8 +133,8 @@ class RemoteFollowTest(TestCase): self.assertEqual('/web/user.com', got.headers['Location']) -@patch('requests_cache.CachedSession.post') -@patch('requests_cache.CachedSession.get') +@patch('requests.post') +@patch('requests.get') class FollowTest(TestCase): def setUp(self): @@ -146,9 +146,9 @@ class FollowTest(TestCase): 'state': '@foo@ba.r', } - # IndieAuth endpoint discovery - @patch('requests.get', return_value=requests_response('')) - def test_start(self, _, __, ___): + def test_start(self, mock_get, _): + mock_get.return_value = requests_response('') # IndieAuth endpoint discovery + resp = self.client.post('/follow/start', data={ 'me': 'https://alice.com', 'address': '@foo@ba.r', @@ -157,16 +157,17 @@ class FollowTest(TestCase): self.assertTrue(resp.headers['Location'].startswith(indieauth.INDIEAUTH_URL), resp.headers['Location']) - # indieauth endpoint discovery, token validation - @patch('requests.get', return_value= requests_response('')) - @patch('requests.post', return_value=requests_response('me=https://alice.com')) - def test_callback_address(self, _, __, mock_get, mock_post): + def test_callback_address(self, mock_get, mock_post): mock_get.side_effect = ( + requests_response(''), # indieauth https://alice.com fetch for user json WEBFINGER, self.as2_resp(FOLLOWEE), self.as2_resp(FOLLOWEE), ) - mock_post.return_value = requests_response('OK') # AP Follow to inbox + mock_post.side_effect = ( + requests_response('me=https://alice.com'), + requests_response('OK'), # AP Follow to inbox + ) state = util.encode_oauth_state(self.state) resp = self.client.get(f'/follow/callback?code=my_code&state={state}', @@ -176,14 +177,14 @@ class FollowTest(TestCase): self.req('https://ba.r/.well-known/webfinger?resource=acct:foo@ba.r'), )) - @patch('requests.get', return_value=requests_response('')) - @patch('requests.post', return_value=requests_response('me=https://alice.com')) - def test_callback_url(self, _, __, mock_get, mock_post): + def test_callback_url(self, mock_get, mock_post): mock_get.side_effect = ( + requests_response(''), # indieauth https://alice.com fetch for user json self.as2_resp(FOLLOWEE), self.as2_resp(FOLLOWEE), ) mock_post.side_effect = ( + requests_response('me=https://alice.com'), requests_response('OK'), # AP Follow to inbox ) @@ -193,13 +194,15 @@ class FollowTest(TestCase): base_url='https://fed.brid.gy/') self.check('https://ba.r/actor', resp, FOLLOW_URL, mock_get, mock_post) - @patch('requests.get', return_value=requests_response('')) - @patch('requests.post', return_value=requests_response('me=https://alice.com')) - def test_callback_stored_followee_with_our_as1(self, _, __, mock_get, mock_post): + def test_callback_stored_followee_with_our_as1(self, mock_get, mock_post): self.store_object(id='https://ba.r/id', our_as1=as2.to_as1(FOLLOWEE), source_protocol='activitypub') + mock_get.side_effect = ( + requests_response(''), # indieauth https://alice.com fetch for user json + ) mock_post.side_effect = ( + requests_response('me=https://alice.com'), requests_response('OK'), # AP Follow to inbox ) @@ -216,9 +219,7 @@ class FollowTest(TestCase): self.check('https://ba.r/id', resp, follow_with_profile_link, mock_get, mock_post, fetched_followee=False) - @patch('requests.get', return_value= requests_response('')) - @patch('requests.post', return_value=requests_response('me=https://alice.com')) - def test_callback_user_with_custom_username(self, _, __, mock_get, mock_post): + def test_callback_user_with_custom_username(self, mock_get, mock_post): self.user.obj.clear() self.user.obj.as2 = { 'type': 'Person', @@ -227,10 +228,14 @@ class FollowTest(TestCase): self.user.obj.put() mock_get.side_effect = ( + requests_response(''), # indieauth https://alice.com fetch for user json self.as2_resp(FOLLOWEE), self.as2_resp(FOLLOWEE), ) - mock_post.return_value = requests_response('OK') # AP Follow to inbox + mock_post.side_effect = ( + requests_response('me=https://alice.com'), + requests_response('OK'), # AP Follow to inbox + ) self.state['state'] = 'https://ba.r/actor' state = util.encode_oauth_state(self.state) @@ -247,9 +252,7 @@ class FollowTest(TestCase): }, }) - @patch('requests.get', return_value= requests_response('')) - @patch('requests.post', return_value=requests_response('me=https://alice.com')) - def test_callback_composite_url_field(self, _, __, mock_get, mock_post): + def test_callback_composite_url_field(self, mock_get, mock_post): """https://console.cloud.google.com/errors/detail/CKmLytj-nPv9RQ;time=P30D?project=bridgy-federated""" followee = { **FOLLOWEE, @@ -262,10 +265,14 @@ class FollowTest(TestCase): }], } mock_get.side_effect = ( + requests_response(''), # indieauth https://alice.com fetch for user json self.as2_resp(followee), self.as2_resp(followee), ) - mock_post.return_value = requests_response('OK') # AP Follow to inbox + mock_post.side_effect = ( + requests_response('me=https://alice.com'), + requests_response('OK'), # AP Follow to inbox + ) self.state['state'] = 'https://ba.r/actor' state = util.encode_oauth_state(self.state) @@ -274,10 +281,10 @@ class FollowTest(TestCase): self.check('https://ba.r/actor', resp, FOLLOW_URL, mock_get, mock_post) - @patch('requests.get', return_value= requests_response('')) - @patch('requests.post', return_value=requests_response('me=https://alice.com')) - def test_callback_bridged_account_error(self, _, __, mock_get, mock_post): + def test_callback_bridged_account_error(self, mock_get, mock_post): + mock_post.return_value = requests_response('me=https://alice.com') mock_get.side_effect = [ + requests_response(''), # indieauth https://alice.com fetch for user json requests_response({ # webfinger 'subject': 'acct:bob.com@web.brid.gy', 'aliases': ['https://bob.com/'], @@ -299,10 +306,10 @@ class FollowTest(TestCase): ["@bob.com@web.brid.gy is a bridged account. Try following them on the web!"], get_flashed_messages()) - @patch('requests.get', return_value= requests_response('')) - @patch('requests.post', return_value=requests_response('me=https://alice.com')) - def test_callback_upgraded_bridged_account_error(self, _, __, mock_get, mock_post): + def test_callback_upgraded_bridged_account_error(self, mock_get, mock_post): + mock_post.return_value = requests_response('me=https://alice.com') mock_get.side_effect = [ + requests_response(''), # indieauth https://alice.com fetch for user json requests_response({ # webfinger 'subject': 'acct:bob.com@bob.com', 'aliases': ['https://bob.com/'], @@ -373,26 +380,26 @@ class FollowTest(TestCase): self.assertEqual('https://alice.com', session['indieauthed-me']) - @patch('requests.post', return_value=requests_response('me=https://alice.com')) - def test_callback_missing_user(self, _, mock_get, mock_post): + def test_callback_missing_user(self, mock_get, mock_post): self.user.key.delete() + mock_post.return_value = requests_response('me=https://alice.com') state = util.encode_oauth_state(self.state) resp = self.client.get(f'/follow/callback?code=my_code&state={state}') self.assertEqual(400, resp.status_code) - @patch('requests.get', return_value= requests_response('')) - @patch('requests.post', return_value=requests_response('me=https://alice.com')) - def test_callback_user_use_instead(self, _, __, mock_get, mock_post): + def test_callback_user_use_instead(self, mock_get, mock_post): user = self.make_user('www.alice.com', cls=Web, obj_id='https://www.alice.com/') self.user.use_instead = user.key self.user.put() mock_get.side_effect = ( + requests_response(''), self.as2_resp(FOLLOWEE), self.as2_resp(FOLLOWEE), ) mock_post.side_effect = ( + requests_response('me=https://alice.com'), requests_response('OK'), # AP Follow to inbox ) @@ -428,9 +435,7 @@ class FollowTest(TestCase): followers, ignore=['created', 'updated']) - @patch('requests.get', return_value= requests_response('')) - @patch('requests.post', return_value=requests_response('me=https://alice.com')) - def test_callback_url_composite_url(self, _, __, mock_get, mock_post): + def test_callback_url_composite_url(self, mock_get, mock_post): followee = { **FOLLOWEE, 'attachments': [{ @@ -440,10 +445,12 @@ class FollowTest(TestCase): }], } mock_get.side_effect = ( + requests_response(''), self.as2_resp(followee), self.as2_resp(followee), ) mock_post.side_effect = ( + requests_response('me=https://alice.com'), requests_response('OK'), # AP Follow to inbox ) @@ -476,8 +483,11 @@ class FollowTest(TestCase): }, base_url='https://fed.brid.gy/') self.check('https://ba.r/actor', resp, FOLLOW_URL, mock_get, mock_post) - @patch('requests.get', return_value= requests_response('')) - def test_indieauthed_session_wrong_me(self, _, mock_get, mock_post): + def test_indieauthed_session_wrong_me(self, mock_get, mock_post): + mock_get.side_effect = ( + requests_response(''), # IndieAuth endpoint discovery + ) + with self.client.session_transaction(base_url='https://fed.brid.gy/') \ as ctx_session: ctx_session['indieauthed-me'] = 'https://eve.com' @@ -490,8 +500,9 @@ class FollowTest(TestCase): self.assertTrue(resp.headers['Location'].startswith(indieauth.INDIEAUTH_URL), resp.headers['Location']) - @patch('requests.get', side_effect=requests.ConnectionError('foo')) - def test_start_homepage_fetch_fails(self, _, mock_get, mock_post): + def test_start_homepage_fetch_fails(self, mock_get, mock_post): + mock_get.side_effect = requests.ConnectionError('foo') + resp = self.client.post('/follow/start', data={ 'me': 'https://alice.com', 'address': 'https://ba.r/actor', @@ -503,8 +514,8 @@ class FollowTest(TestCase): get_flashed_messages()) -@patch('requests_cache.CachedSession.post') -@patch('requests_cache.CachedSession.get') +@patch('requests.post') +@patch('requests.get') class UnfollowTest(TestCase): def setUp(self): @@ -523,9 +534,9 @@ class UnfollowTest(TestCase): 'state': self.follower.key.id(), }) - # IndieAuth endpoint discovery - @patch('requests.get', return_value= requests_response('')) - def test_start(self, _, mock_get, mock_post): + def test_start(self, mock_get, _): + mock_get.return_value = requests_response('') # IndieAuth endpoint discovery + resp = self.client.post('/unfollow/start', data={ 'me': 'https://alice.com', 'key': self.follower.key.id(), @@ -534,11 +545,11 @@ class UnfollowTest(TestCase): self.assertTrue(resp.headers['Location'].startswith(indieauth.INDIEAUTH_URL), resp.headers['Location']) - # indieauth endpoint discovery, token validation - @patch('requests.get', return_value= requests_response('')) - @patch('requests.post', return_value=requests_response('me=https://alice.com')) - def test_callback(self, _, __, mock_get, mock_post): + def test_callback(self, mock_get, mock_post): + # oauth-dropins indieauth https://alice.com fetch for user json + mock_get.return_value = requests_response('') mock_post.side_effect = ( + requests_response('me=https://alice.com'), requests_response('OK'), # AP Undo Follow to inbox ) @@ -546,9 +557,7 @@ class UnfollowTest(TestCase): base_url='https://fed.brid.gy/') self.check(resp, UNDO_FOLLOW, mock_get, mock_post) - @patch('requests.get', return_value= requests_response('')) - @patch('requests.post', return_value=requests_response('me=https://alice.com')) - def test_callback_last_follow_object_str(self, _, __, mock_get, mock_post): + def test_callback_last_follow_object_str(self, mock_get, mock_post): to = self.follower.to.get() to.obj = None to.put() @@ -558,10 +567,13 @@ class UnfollowTest(TestCase): obj.put() mock_get.side_effect = ( + # oauth-dropins indieauth https://alice.com fetch for user json + requests_response(''), # actor fetch to discover inbox self.as2_resp(FOLLOWEE), ) mock_post.side_effect = ( + requests_response('me=https://alice.com'), requests_response('OK'), # AP Undo Follow to inbox ) @@ -607,9 +619,7 @@ class UnfollowTest(TestCase): self.assertEqual('https://alice.com', session['indieauthed-me']) - @patch('requests.get', return_value= requests_response('')) - @patch('requests.post', return_value=requests_response('me=https://alice.com')) - def test_callback_user_use_instead(self, _, __, mock_get, mock_post): + def test_callback_user_use_instead(self, mock_get, mock_post): user = self.make_user('www.alice.com', cls=Web) self.user.use_instead = user.key self.user.put() @@ -620,7 +630,9 @@ class UnfollowTest(TestCase): follow=Object(id=FOLLOW_ADDRESS['id'], as2=FOLLOW_ADDRESS).put(), status='active') + mock_get.return_value = requests_response('') mock_post.side_effect = ( + requests_response('me=https://alice.com'), requests_response('OK'), # AP Undo Follow to inbox ) @@ -647,7 +659,7 @@ class UnfollowTest(TestCase): } del expected_undo['object']['id'] - inbox_args, inbox_kwargs = mock_post.call_args + inbox_args, inbox_kwargs = mock_post.call_args_list[1] self.assertEqual(('http://ba.r/inbox',), inbox_args) self.assert_equals({ **expected_undo, @@ -668,9 +680,7 @@ class UnfollowTest(TestCase): delivered=['http://ba.r/inbox'], delivered_protocol='activitypub') - @patch('requests.get', return_value= requests_response('')) - @patch('requests.post', return_value=requests_response('me=https://alice.com')) - def test_callback_composite_url(self, _, __, mock_get, mock_post): + def test_callback_composite_url(self, mock_get, mock_post): follower = self.follower.to.get().obj follower.our_as1 = { **as2.to_as1(FOLLOWEE), @@ -681,7 +691,10 @@ class UnfollowTest(TestCase): } follower.put() + # oauth-dropins indieauth https://alice.com fetch for user json + mock_get.return_value = requests_response('') mock_post.side_effect = ( + requests_response('me=https://alice.com'), requests_response('OK'), # AP Undo Follow to inbox ) @@ -691,8 +704,9 @@ class UnfollowTest(TestCase): get_flashed_messages()) self.check(resp, UNDO_FOLLOW, mock_get, mock_post) - @patch('requests.get', return_value= requests_response('')) - def test_indieauthed_session(self, _, mock_get, mock_post): + def test_indieauthed_session(self, mock_get, mock_post): + # oauth-dropins indieauth https://alice.com fetch for user json + mock_get.return_value = requests_response('') mock_post.side_effect = ( requests_response('OK'), # AP Undo Follow to inbox ) @@ -707,8 +721,11 @@ class UnfollowTest(TestCase): }, base_url='https://fed.brid.gy/') self.check(resp, UNDO_FOLLOW, mock_get, mock_post) - @patch('requests.get', return_value= requests_response('')) - def test_indieauthed_session_wrong_me(self, _, mock_get, mock_post): + def test_indieauthed_session_wrong_me(self, mock_get, mock_post): + mock_get.side_effect = ( + requests_response(''), # IndieAuth endpoint discovery + ) + with self.client.session_transaction(base_url='https://fed.brid.gy/') \ as ctx_session: ctx_session['indieauthed-me'] = 'https://eve.com' @@ -721,8 +738,9 @@ class UnfollowTest(TestCase): self.assertTrue(resp.headers['Location'].startswith(indieauth.INDIEAUTH_URL), resp.headers['Location']) - @patch('requests.get', side_effect=requests.ConnectionError('foo')) - def test_start_homepage_fetch_fails(self, _, mock_get, mock_post): + def test_start_homepage_fetch_fails(self, mock_get, mock_post): + mock_get.side_effect = requests.ConnectionError('foo') + resp = self.client.post('/unfollow/start', data={ 'me': 'https://alice.com', 'key': self.follower.key.id(), diff --git a/tests/test_integrations.py b/tests/test_integrations.py index 75f9516..e9510af 100644 --- a/tests/test_integrations.py +++ b/tests/test_integrations.py @@ -24,7 +24,7 @@ DID_DOC = { class IntegrationTests(TestCase): - @patch('requests_cache.CachedSession.post') + @patch('requests.post') @patch('requests.get') @patch('common.ENABLED_BRIDGES', new=[('activitypub', 'atproto')]) def test_atproto_notify_reply_to_activitypub(self, mock_get, mock_post): diff --git a/tests/test_models.py b/tests/test_models.py index 0c117d3..42bd579 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -67,7 +67,7 @@ class UserTest(TestCase): self.assert_entities_equal(same, user, ignore=['updated']) @patch.object(tasks_client, 'create_task', return_value=Task(name='my task')) - @patch('requests_cache.CachedSession.post', + @patch('requests.post', return_value=requests_response('OK')) # create DID on PLC def test_get_or_create_propagate(self, mock_post, mock_create_task): common.RUN_TASKS_INLINE = False @@ -108,8 +108,8 @@ class UserTest(TestCase): mock_create_task.assert_called() @patch.object(tasks_client, 'create_task') - @patch('requests_cache.CachedSession.post') - @patch('requests_cache.CachedSession.get') + @patch('requests.post') + @patch('requests.get') def test_get_or_create_propagate_not_enabled(self, mock_get, mock_post, mock_create_task): mock_get.return_value = self.as2_resp(ACTOR) @@ -622,8 +622,7 @@ class ObjectTest(TestCase): } self.assert_equals('good', Object(our_as1=obj, atom='trigger').as1['id']) - @patch('requests_cache.CachedSession.get', - return_value=requests_response(DID_DOC)) + @patch('requests.get', return_value=requests_response(DID_DOC)) def test_as1_from_bsky(self, mock_get): like_bsky = { '$type': 'app.bsky.feed.like', diff --git a/tests/test_pages.py b/tests/test_pages.py index 9b4e069..ff06000 100644 --- a/tests/test_pages.py +++ b/tests/test_pages.py @@ -400,7 +400,7 @@ class PagesTest(TestCase): self.assertIn('Dr. Eve', got.text) @patch.object(tasks_client, 'create_task', return_value=Task(name='my task')) - @patch('requests_cache.CachedSession.post', + @patch('requests.post', return_value=requests_response('OK')) # create DID on PLC def test_bridge_user(self, mock_post, mock_create_task): common.RUN_TASKS_INLINE = False diff --git a/tests/test_protocol.py b/tests/test_protocol.py index 62bbb62..3f0696c 100644 --- a/tests/test_protocol.py +++ b/tests/test_protocol.py @@ -112,26 +112,26 @@ class ProtocolTest(TestCase): self.store_object(id='http://ba.d/obj') self.assertIsNone(Protocol.for_id('http://ba.d/obj')) - @patch('requests_cache.CachedSession.get') + @patch('requests.get') def test_for_id_activitypub_fetch(self, mock_get): mock_get.return_value = self.as2_resp(ACTOR) self.assertEqual(ActivityPub, Protocol.for_id('http://a.p/actor')) self.assertIn(self.as2_req('http://a.p/actor'), mock_get.mock_calls) - @patch('requests_cache.CachedSession.get') + @patch('requests.get') def test_for_id_activitypub_fetch_fails(self, mock_get): mock_get.return_value = requests_response('', status=403) self.assertIsNone(Protocol.for_id('http://a.p/actor')) self.assertIn(self.as2_req('http://a.p/actor'), mock_get.mock_calls) mock_get.assert_called_once() - @patch('requests_cache.CachedSession.get') + @patch('requests.get') def test_for_id_web_fetch(self, mock_get): mock_get.return_value = ACTOR_HTML_RESP self.assertEqual(Web, Protocol.for_id('http://web.site/')) self.assertIn(self.req('http://web.site/'), mock_get.mock_calls) - @patch('requests_cache.CachedSession.get') + @patch('requests.get') def test_for_id_web_fetch_no_mf2(self, mock_get): mock_get.return_value = requests_response('') self.assertIsNone(Protocol.for_id('http://web.site/')) @@ -270,7 +270,7 @@ class ProtocolTest(TestCase): self.assertFalse(loaded.new) self.assertEqual(['foo'], Fake.fetched) - @patch('requests_cache.CachedSession.get', return_value=ACTOR_HTML_RESP) + @patch('requests.get', return_value=ACTOR_HTML_RESP) def test_load_remote_true_clear_our_as1(self, _): self.store_object(id='https://fo.o', our_as1={'should': 'disappear'}, source_protocol='web') @@ -1558,8 +1558,8 @@ class ProtocolReceiveTest(TestCase): self.assertEqual(1, len(followers)) self.assertEqual(self.alice.key, followers[0].to) - @patch('requests_cache.CachedSession.post') - @patch('requests_cache.CachedSession.get') + @patch('requests.post') + @patch('requests.get') def test_skip_web_same_domain(self, mock_get, mock_post): Web.fetchable = { 'http://x.com/alice': {}, diff --git a/tests/test_redirect.py b/tests/test_redirect.py index be42357..3eefaec 100644 --- a/tests/test_redirect.py +++ b/tests/test_redirect.py @@ -94,7 +94,7 @@ class RedirectTest(testutil.TestCase): self.assert_user(Web, 'user.com', direct=False) - @patch('requests_cache.CachedSession.get') + @patch('requests.get') def test_as2_fetch_post(self, mock_get): mock_get.side_effect = [ requests_response(REPOST_HTML), @@ -107,7 +107,7 @@ class RedirectTest(testutil.TestCase): self.assert_equals(REPOST_AS2, resp.json) self.assertEqual('Accept', resp.headers['Vary']) - @patch('requests_cache.CachedSession.get') + @patch('requests.get') def test_as2_fetch_post_no_backlink(self, mock_get): mock_get.side_effect = [ requests_response( @@ -121,7 +121,7 @@ class RedirectTest(testutil.TestCase): self.assert_equals(REPOST_AS2, resp.json) self.assertEqual('Accept', resp.headers['Vary']) - @patch('requests_cache.CachedSession.get') + @patch('requests.get') def test_as2_no_user_fetch_homepage(self, mock_get): mock_get.return_value = requests_response(ACTOR_HTML) self.user.key.delete() diff --git a/tests/test_web.py b/tests/test_web.py index 42dd5c6..9ad15b5 100644 --- a/tests/test_web.py +++ b/tests/test_web.py @@ -410,8 +410,8 @@ ACTIVITYPUB_GETS = [ ] -@patch('requests_cache.CachedSession.post') -@patch('requests_cache.CachedSession.get') +@patch('requests.post') +@patch('requests.get') class WebTest(TestCase): def setUp(self): super().setUp() @@ -2477,8 +2477,8 @@ http://this/404s "Couldn't connect to https://orig.co/: ")) -@patch('requests_cache.CachedSession.post') -@patch('requests_cache.CachedSession.get') +@patch('requests.post') +@patch('requests.get') class WebUtilTest(TestCase): def setUp(self): diff --git a/tests/test_webfinger.py b/tests/test_webfinger.py index b20c0fa..bcf78a6 100644 --- a/tests/test_webfinger.py +++ b/tests/test_webfinger.py @@ -321,7 +321,7 @@ class WebfingerTest(TestCase): finally: PROTOCOLS.pop('nohandle') - @patch('requests_cache.CachedSession.get') + @patch('requests.get') def test_create_user(self, mock_get): self.user.key.delete() self.user.obj_key.delete() @@ -348,22 +348,20 @@ class WebfingerTest(TestCase): got = self.client.get('/.well-known/webfinger?resource=acct%3A%40localhost') self.assertEqual(400, got.status_code, got.get_data(as_text=True)) - @patch('requests_cache.CachedSession.get', return_value=requests_response( + @patch('requests.get', return_value=requests_response( WEBFINGER, content_type='application/jrd+json')) def test_fetch(self, mock_get): self.assertEqual(WEBFINGER, fetch('@foo@bar')) self.assert_req(mock_get, 'https://bar/.well-known/webfinger?resource=acct:foo@bar') - @patch('requests_cache.CachedSession.get', - return_value=requests_response(WEBFINGER)) + @patch('requests.get', return_value=requests_response(WEBFINGER)) def test_fetch_actor_url(self, mock_get): self.assertEqual('http://localhost/user.com', fetch_actor_url('@foo@bar')) self.assert_req(mock_get, 'https://bar/.well-known/webfinger?resource=acct:foo@bar') - @patch('requests_cache.CachedSession.get', - return_value=requests_response({'links': []})) + @patch('requests.get', return_value=requests_response({'links': []})) def test_fetch_actor_url_not_found(self, mock_get): self.assertIsNone(fetch_actor_url('@foo@bar')) self.assert_req(mock_get, diff --git a/web.py b/web.py index e5f1dc1..2022f1a 100644 --- a/web.py +++ b/web.py @@ -243,7 +243,7 @@ class Web(User, Protocol): root = domain.removeprefix("www.") root_site = f'https://{root}/' try: - resp = common.requests_get(root_site, gateway=False) + resp = util.requests_get(root_site, gateway=False) if resp.ok and self.is_web_url(resp.url): logger.info(f'{root_site} serves ok ; using {root} instead') root_user = Web.get_or_create(root) @@ -261,7 +261,7 @@ class Web(User, Protocol): self.redirects_error = None try: url = urljoin(self.web_url(), path) - resp = common.requests_get(url, gateway=False) + resp = util.requests_get(url, gateway=False) domain_urls = ([f'https://{domain}/' for domain in common.DOMAINS] + [common.host_url()]) expected = [urljoin(url, path) for url in domain_urls] @@ -398,7 +398,7 @@ class Web(User, Protocol): if not endpoint: return False - webmention.send(endpoint, source_url, url, session=common.requests_session()) + webmention.send(endpoint, source_url, url) return True @classmethod @@ -435,8 +435,8 @@ class Web(User, Protocol): else None) try: - parsed = common.fetch_mf2(url, gateway=gateway, metaformats_hcard=True, - require_backlink=require_backlink) + parsed = util.fetch_mf2(url, gateway=gateway, metaformats_hcard=True, + require_backlink=require_backlink) except ValueError as e: error(str(e)) @@ -492,7 +492,7 @@ class Web(User, Protocol): if not isinstance(author, dict): logger.info(f'Fetching full authorship for author {author}') author = mf2util.find_author({'items': [entry]}, hentry=entry, - fetch_mf2_func=common.fetch_mf2) + fetch_mf2_func=util.fetch_mf2) logger.info(f'Got: {author}') if author: props['author'] = util.trim_nulls([{ @@ -671,7 +671,7 @@ def poll_feed_task(): headers['If-None-Match'] = user.feed_etag if user.feed_last_modified: headers['If-Modified-Since'] = user.feed_last_modified - resp = common.requests_get(url, headers=headers, gateway=True) + resp = util.requests_get(url, headers=headers, gateway=True) content_type = resp.headers.get('Content-Type') or '' type = FEED_TYPES.get(content_type.split(';')[0]) diff --git a/webfinger.py b/webfinger.py index 07af6b0..994fbf4 100644 --- a/webfinger.py +++ b/webfinger.py @@ -225,7 +225,7 @@ def fetch(addr): return None try: - resp = common.requests_get( + resp = util.requests_get( f'https://{addr_domain}/.well-known/webfinger?resource={resource}') except BaseException as e: if util.is_connection_failure(e):