From 312b3760fca0f91ecdcd32ad6ce586b06169acf9 Mon Sep 17 00:00:00 2001 From: Marnanel Thurman Date: Sun, 7 Feb 2021 20:30:03 +0000 Subject: [PATCH 01/19] Started to put test_timelines in order. Each test sets up the statuses as it needs. --- kepi/trilby_api/tests/test_timelines.py | 180 +++++++++++++----------- 1 file changed, 100 insertions(+), 80 deletions(-) diff --git a/kepi/trilby_api/tests/test_timelines.py b/kepi/trilby_api/tests/test_timelines.py index a680293..33bb59d 100644 --- a/kepi/trilby_api/tests/test_timelines.py +++ b/kepi/trilby_api/tests/test_timelines.py @@ -17,115 +17,135 @@ from unittest import skip # Tests for timelines. API docs are here: # https://docs.joinmastodon.org/methods/statuses/ -TIMELINE_DATA = [ - # Visibility is: - # A=public: visible to anyone, and in public timelines - # U=unlisted: visible to anyone, but not in public timelines - # X=private: visible to followers and anyone tagged - # D=direct: visible only to those who are tagged - - # We haven't yet implemented: - # - (user) tags - # - hashtags - # - user lists - # - following users but hiding reblogs - # and when we do, these tests will need updating. - # - # All statuses are posted by alice. - # - # id visibility visible in - ( 'A', 'A', - ['public', 'follower', 'stranger', 'home', ], ), - ( 'B', 'U', - ['follower', 'stranger', 'home', ], ), - ( 'C', 'X', - ['follower', 'home',], ), - ( 'D', 'D', - ['home', ], ), - - ] - class TestTimelines(TrilbyTestCase): - def _set_up(self): + def add_status(self, source, visibility, content): + status = Status( + account = source, + content = content, + visibility = visibility, + ) + status.save() - self._alice = create_local_person("alice") + logger.info("Created status: %s", status) - for (id, visibility, visible_in) in TIMELINE_DATA: - status = Status( - account = self._alice, - content = id, - visibility = visibility, - ) - status.save() - - def _check_timelines(self, - situation, + def timeline_contents(self, path, as_user): - expected = [] - for (id, visibility, visible_in) in TIMELINE_DATA: - if situation in visible_in: - expected.append(f'

{id}

') - expected = sorted(expected) - details = sorted([x['content'] \ for x in self.get( path = path, as_user = as_user, )]) - self.assertListEqual( - expected, - details, - msg = f"Visibility in '{situation}' mismatch: "+\ - f"expected {expected}, but got {details}.", - ) + result = '' + for detail in details: - def test_public(self): - self._set_up() + if detail.startswith('

') and detail.endswith('

'): + detail = detail[3:-4] - self._check_timelines( - situation = 'public', - path = '/api/v1/timelines/public', - as_user = None, + result += detail + + logger.info("Timeline contents for %s as %s...", + path, + as_user) + logger.info(" ...are %s", + result) + + return result + + def test_public_as_anon(self): + + alice = create_local_person("alice") + + self.add_status(source=alice, content='A', visibility='A') + self.add_status(source=alice, content='B', visibility='U') + self.add_status(source=alice, content='C', visibility='X') + self.add_status(source=alice, content='D', visibility='D') + + self.assertEqual( + self.timeline_contents( + path = '/api/v1/timelines/public', + as_user = None, + ), + 'A', ) - def test_follower(self): - self._set_up() - self._george = create_local_person("george") + def test_public_as_user(self): + + alice = create_local_person("alice") + + self.add_status(source=alice, content='A', visibility='A') + self.add_status(source=alice, content='B', visibility='U') + self.add_status(source=alice, content='C', visibility='X') + self.add_status(source=alice, content='D', visibility='D') + + self.assertEqual( + self.timeline_contents( + path = '/api/v1/timelines/public', + as_user = alice, + ), + 'A', + ) + + def test_public_as_follower(self): + + alice = create_local_person("alice") + george = create_local_person("george") follow = Follow( - follower = self._george, - following = self._alice, + follower = george, + following = alice, offer = None, ) follow.save() - self._check_timelines( - situation = 'public', - path = '/api/v1/timelines/public', - as_user = self._george, + self.add_status(source=alice, content='A', visibility='A') + self.add_status(source=alice, content='B', visibility='U') + self.add_status(source=alice, content='C', visibility='X') + self.add_status(source=alice, content='D', visibility='D') + + self.assertEqual( + self.timeline_contents( + path = '/api/v1/timelines/public', + as_user = george, + ), + 'AC', ) - def test_stranger(self): - self._set_up() - self._henry = create_local_person("henry") + def test_public_as_stranger(self): - self._check_timelines( - situation = 'public', - path = '/api/v1/timelines/public', - as_user = self._henry, + alice = create_local_person("alice") + henry = create_local_person("henry") + + self.add_status(source=alice, content='A', visibility='A') + self.add_status(source=alice, content='B', visibility='U') + self.add_status(source=alice, content='C', visibility='X') + self.add_status(source=alice, content='D', visibility='D') + + self.assertEqual( + self.timeline_contents( + path = '/api/v1/timelines/public', + as_user = henry, + ), + 'A', ) - def test_home(self): - self._set_up() + def test_home_as_user(self): + alice = create_local_person("alice") - self._check_timelines( - situation = 'home', - path = '/api/v1/timelines/home', - as_user = self._alice, + self.add_status(source=alice, content='A', visibility='A') + self.add_status(source=alice, content='B', visibility='U') + self.add_status(source=alice, content='C', visibility='X') + self.add_status(source=alice, content='D', visibility='D') + + self.assertEqual( + self.timeline_contents( + path = '/api/v1/timelines/home', + as_user = alice, + ), + 'ABCD', ) @skip("to be implemented later") From f40a6d862d586fc7b214c3cc15b96c6d25469f52 Mon Sep 17 00:00:00 2001 From: Marnanel Thurman Date: Tue, 9 Feb 2021 17:39:39 +0000 Subject: [PATCH 02/19] Many new timeline tests, per spec --- kepi/trilby_api/tests/test_timelines.py | 147 +++++++++++++++++++++++- 1 file changed, 146 insertions(+), 1 deletion(-) diff --git a/kepi/trilby_api/tests/test_timelines.py b/kepi/trilby_api/tests/test_timelines.py index 33bb59d..e3db31f 100644 --- a/kepi/trilby_api/tests/test_timelines.py +++ b/kepi/trilby_api/tests/test_timelines.py @@ -11,8 +11,10 @@ from rest_framework.test import APIClient, force_authenticate from kepi.trilby_api.views import * from kepi.trilby_api.tests import * from kepi.trilby_api.models import * +from kepi.bowler_pub.tests import create_remote_person from django.conf import settings from unittest import skip +import httpretty # Tests for timelines. API docs are here: # https://docs.joinmastodon.org/methods/statuses/ @@ -29,9 +31,12 @@ class TestTimelines(TrilbyTestCase): logger.info("Created status: %s", status) + return status + def timeline_contents(self, path, - as_user): + as_user = None, + ): details = sorted([x['content'] \ for x in self.get( @@ -132,6 +137,146 @@ class TestTimelines(TrilbyTestCase): 'A', ) + @httpretty.activate() + def test_public_local_and_remote(self): + + alice = create_local_person("alice") + peter = create_remote_person( + remote_url = "https://example.com/users/peter", + name = "peter", + auto_fetch = True, + ) + + self.add_status(source=alice, content='A', visibility='A') + self.add_status(source=peter, content='B', visibility='A') + self.add_status(source=alice, content='C', visibility='A') + self.add_status(source=peter, content='D', visibility='A') + + self.assertEqual( + self.timeline_contents( + path = '/api/v1/timelines/public', + ), + 'ABCD', + ) + + self.assertEqual( + self.timeline_contents( + path = '/api/v1/timelines/public?local=true', + ), + 'AC', + ) + + self.assertEqual( + self.timeline_contents( + path = '/api/v1/timelines/public?local=false', + ), + 'ABCD', + ) + + self.assertEqual( + self.timeline_contents( + path = '/api/v1/timelines/public?remote=true', + ), + 'BD', + ) + + self.assertEqual( + self.timeline_contents( + path = '/api/v1/timelines/public?remote=false', + ), + 'ABCD', + ) + + self.assertEqual( + self.timeline_contents( + path = '/api/v1/timelines/public?local=true&remote=true', + ), + '', + ) + + def test_public_only_media(self): + + # We don't support added media at present anyway, + # so turning this on will always get the empty set + + alice = create_local_person("alice") + + self.add_status(source=alice, content='A', visibility='A') + self.add_status(source=alice, content='B', visibility='A') + self.add_status(source=alice, content='C', visibility='A') + self.add_status(source=alice, content='D', visibility='A') + + self.assertEqual( + self.timeline_contents( + path = '/api/v1/timelines/public?only_media=true', + ), + '', + ) + + def test_public_max_since_and_min(self): + + alice = create_local_person("alice") + + self.add_status(source=alice, content='A', visibility='A') + self.add_status(source=alice, content='B', visibility='A') + status_c = self.add_status(source=alice, content='C', visibility='A') + self.add_status(source=alice, content='D', visibility='A') + + c_id = str(status_c.id) + + self.assertEqual( + self.timeline_contents( + path = '/api/v1/timelines/public?since='+c_id, + ), + 'D', + ) + + self.assertEqual( + self.timeline_contents( + path = '/api/v1/timelines/public?max_id='+c_id, + ), + 'ABC', + ) + + self.assertEqual( + self.timeline_contents( + path = '/api/v1/timelines/public?min_id='+c_id, + ), + 'CD', + ) + + def test_public_limit(self): + + alice = create_local_person("alice") + + alphabet = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' + + for i in range(len(alphabet)): + self.add_status( + source=alice, + content=alphabet[i], + visibility='A', + ) + + for i in range(len(alphabet)): + self.assertEqual( + self.timeline_contents( + path = '/api/v1/timelines/public?limit='+str(i), + ), + alphabet[:i], + ) + + # the default is specified as 20 + self.assertEqual( + self.timeline_contents( + path = '/api/v1/timelines/public', + ), + alphabet[:20], + message = 'default is 20', + ) + + ###### home timeline + def test_home_as_user(self): alice = create_local_person("alice") From 17519f62cbc0dbc0ab1ab9da329943bcf45329ac Mon Sep 17 00:00:00 2001 From: Marnanel Thurman Date: Tue, 9 Feb 2021 17:42:10 +0000 Subject: [PATCH 03/19] split public and home timeline tests to separate classes --- kepi/trilby_api/tests/test_timelines.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/kepi/trilby_api/tests/test_timelines.py b/kepi/trilby_api/tests/test_timelines.py index e3db31f..1e0317f 100644 --- a/kepi/trilby_api/tests/test_timelines.py +++ b/kepi/trilby_api/tests/test_timelines.py @@ -19,7 +19,7 @@ import httpretty # Tests for timelines. API docs are here: # https://docs.joinmastodon.org/methods/statuses/ -class TestTimelines(TrilbyTestCase): +class TimelineTestCase(TrilbyTestCase): def add_status(self, source, visibility, content): status = Status( @@ -60,7 +60,9 @@ class TestTimelines(TrilbyTestCase): return result - def test_public_as_anon(self): +class TestPublicTimeline(TimelineTestCase): + + def test_as_anon(self): alice = create_local_person("alice") @@ -77,7 +79,7 @@ class TestTimelines(TrilbyTestCase): 'A', ) - def test_public_as_user(self): + def test_as_user(self): alice = create_local_person("alice") @@ -94,7 +96,7 @@ class TestTimelines(TrilbyTestCase): 'A', ) - def test_public_as_follower(self): + def test_as_follower(self): alice = create_local_person("alice") george = create_local_person("george") @@ -119,7 +121,7 @@ class TestTimelines(TrilbyTestCase): 'AC', ) - def test_public_as_stranger(self): + def test_as_stranger(self): alice = create_local_person("alice") henry = create_local_person("henry") @@ -138,7 +140,7 @@ class TestTimelines(TrilbyTestCase): ) @httpretty.activate() - def test_public_local_and_remote(self): + def test_local_and_remote(self): alice = create_local_person("alice") peter = create_remote_person( @@ -194,7 +196,7 @@ class TestTimelines(TrilbyTestCase): '', ) - def test_public_only_media(self): + def test_only_media(self): # We don't support added media at present anyway, # so turning this on will always get the empty set @@ -213,7 +215,7 @@ class TestTimelines(TrilbyTestCase): '', ) - def test_public_max_since_and_min(self): + def test_max_since_and_min(self): alice = create_local_person("alice") @@ -245,7 +247,7 @@ class TestTimelines(TrilbyTestCase): 'CD', ) - def test_public_limit(self): + def test_limit(self): alice = create_local_person("alice") @@ -275,9 +277,9 @@ class TestTimelines(TrilbyTestCase): message = 'default is 20', ) - ###### home timeline +class TestHomeTimeline(TimelineTestCase): - def test_home_as_user(self): + def test_as_user(self): alice = create_local_person("alice") self.add_status(source=alice, content='A', visibility='A') From 3a3ce2fae25473434209932ddc8c0085c27741d1 Mon Sep 17 00:00:00 2001 From: Marnanel Thurman Date: Tue, 9 Feb 2021 18:06:04 +0000 Subject: [PATCH 04/19] Home timeline tests added --- kepi/trilby_api/tests/test_timelines.py | 179 +++++++++++++++++++++++- 1 file changed, 172 insertions(+), 7 deletions(-) diff --git a/kepi/trilby_api/tests/test_timelines.py b/kepi/trilby_api/tests/test_timelines.py index 1e0317f..9eb878e 100644 --- a/kepi/trilby_api/tests/test_timelines.py +++ b/kepi/trilby_api/tests/test_timelines.py @@ -279,22 +279,187 @@ class TestPublicTimeline(TimelineTestCase): class TestHomeTimeline(TimelineTestCase): - def test_as_user(self): - alice = create_local_person("alice") + def add_standard_statuses(self): + self.alice = create_local_person("alice") + self.bob = create_local_person("bob") + self.carol = create_local_person("carol") - self.add_status(source=alice, content='A', visibility='A') - self.add_status(source=alice, content='B', visibility='U') - self.add_status(source=alice, content='C', visibility='X') - self.add_status(source=alice, content='D', visibility='D') + self.add_status(source=self.bob, content='A', visibility='A') + self.add_status(source=self.carol, content='B', visibility='A') + self.add_status(source=self.carol, content='C', visibility='A') + self.add_status(source=self.bob, content='D', visibility='A') + + Follow( + follower=self.alice, + following=self.bob, + offer=None).save() + + def follow_carol(self): + Follow( + follower=self.alice, + following=self.carol, + offer=None).save() + + def test_not_anon(self): + found = self.get( + path = '/api/v1/timelines/home', + as_user = None, + expect_result = 401, + ) + + def test_simple(self): + + self.add_standard_statuses() self.assertEqual( self.timeline_contents( path = '/api/v1/timelines/home', - as_user = alice, + as_user = self.alice, + ), + 'AD', + ) + + self.follow_carol() + + self.assertEqual( + self.timeline_contents( + path = '/api/v1/timelines/home', + as_user = self.alice, ), 'ABCD', ) + def test_max_since_and_min(self): + + self.add_standard_statuses() + + c_id = '3' # FIXME hack + + self.assertEqual( + self.timeline_contents( + path = '/api/v1/timelines/home?since='+c_id, + as_user = self.alice, + ), + 'D', + ) + + self.assertEqual( + self.timeline_contents( + path = '/api/v1/timelines/home?max_id='+c_id, + as_user = self.alice, + ), + 'A', + ) + + self.assertEqual( + self.timeline_contents( + path = '/api/v1/timelines/home?min_id='+c_id, + as_user = self.alice, + ), + 'D', + ) + + self.follow_carol() + + self.assertEqual( + self.timeline_contents( + path = '/api/v1/timelines/home?since='+c_id, + as_user = self.alice, + ), + 'D', + ) + + self.assertEqual( + self.timeline_contents( + path = '/api/v1/timelines/home?max_id='+c_id, + as_user = self.alice, + ), + 'ABC', + ) + + self.assertEqual( + self.timeline_contents( + path = '/api/v1/timelines/home?min_id='+c_id, + as_user = self.alice, + ), + 'CD', + ) + + def test_limit(self): + + self.alice = create_local_person("alice") + self.bob = create_local_person("bob") + self.carol = create_local_person("carol") + + alphabet = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' + + for i in range(len(alphabet)): + self.add_status( + source=self.bob, + content=alphabet[i], + visibility='A', + ) + + self.add_status( + source=self.carol, + content=alphabet[i].lower(), + visibility='A', + ) + + for i in range(len(alphabet)): + self.assertEqual( + self.timeline_contents( + path = '/api/v1/timelines/home?limit='+str(i), + as_user = self.alice, + ), + alphabet[:i], + ) + + # the default is specified as 20 + self.assertEqual( + self.timeline_contents( + path = '/api/v1/timelines/home', + as_user = self.alice, + ), + alphabet[:20], + message = 'default is 20', + ) + + @httpretty.activate() + def test_local(self): + + self.add_standard_statuses() + + self.peter = create_remote_person( + remote_url = "https://example.com/users/peter", + name = "peter", + auto_fetch = True, + ) + self.add_status(source=self.peter, content='P', visibility='A') + self.add_status(source=self.peter, content='Q', visibility='A') + Follow( + follower = self.alice, + following = self.peter, + offer = None, + ).save() + + self.assertEqual( + self.timeline_contents( + path = '/api/v1/timelines/home', + as_user = self.alice, + ), + 'ABCDPQ', + ) + + self.assertEqual( + self.timeline_contents( + path = '/api/v1/timelines/home?local=true', + as_user = self.alice, + ), + 'ABCD', + ) + +class TestTimelinesNotImplemented(TimelineTestCase): @skip("to be implemented later") def test_hashtag(self): raise NotImplementedError() From 6ae898af0325c8f5464eb33dbad3b825776e5262 Mon Sep 17 00:00:00 2001 From: Marnanel Thurman Date: Sun, 14 Feb 2021 21:57:32 +0000 Subject: [PATCH 05/19] Inbox lookup for LocalPerson used the "follow" relationship backwards; now fixed. --- kepi/trilby_api/models/person.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kepi/trilby_api/models/person.py b/kepi/trilby_api/models/person.py index 26159e1..a48a15c 100644 --- a/kepi/trilby_api/models/person.py +++ b/kepi/trilby_api/models/person.py @@ -606,7 +606,7 @@ class LocalPerson(Person): all_your_friends_public_posts = trilby_models.Status.objects.filter( visibility = trilby_utils.VISIBILITY_PUBLIC, - account__rel_following__following = self, + account__rel_followers__follower = self, ) logger.debug("%s.inbox: all friends' public: %s", From a9d03dd280a9259686b741da23095b6703f9c056 Mon Sep 17 00:00:00 2001 From: Marnanel Thurman Date: Sun, 14 Feb 2021 21:59:47 +0000 Subject: [PATCH 06/19] Shorten Status.__str__ because it gets overwhelming in the logs --- kepi/trilby_api/models/status.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kepi/trilby_api/models/status.py b/kepi/trilby_api/models/status.py index e1d9e9b..826ae04 100644 --- a/kepi/trilby_api/models/status.py +++ b/kepi/trilby_api/models/status.py @@ -278,7 +278,7 @@ class Status(PolymorphicModel): trilby_signals.reblogged.send(sender=self) def __str__(self): - return '[Status %s: %s]' % ( + return '%s: %s' % ( self.id, self.content, ) From d4af44913ba6f76b73050621121f42bccf4f0a7b Mon Sep 17 00:00:00 2001 From: Marnanel Thurman Date: Sun, 14 Feb 2021 22:00:18 +0000 Subject: [PATCH 07/19] TrilbyTestCase uses the "data" and "extra" params the same way as Django's test client --- kepi/trilby_api/tests/__init__.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/kepi/trilby_api/tests/__init__.py b/kepi/trilby_api/tests/__init__.py index 8ab0908..d52774f 100644 --- a/kepi/trilby_api/tests/__init__.py +++ b/kepi/trilby_api/tests/__init__.py @@ -74,11 +74,11 @@ class TrilbyTestCase(TestCase): return result def request(self, verb, path, - data={}, + data = None, as_user=None, expect_result=200, parse_result=True, - *args, **kwargs, + **extra, ): c = APIClient() @@ -92,8 +92,7 @@ class TrilbyTestCase(TestCase): path=path, data=data, format='json', - *args, - **kwargs, + extra=extra, ) if expect_result is not None: From 8eb2b2468d66d532de247a678de987c0a8f6125e Mon Sep 17 00:00:00 2001 From: Marnanel Thurman Date: Sun, 14 Feb 2021 22:01:43 +0000 Subject: [PATCH 08/19] Timelines tests pass GET params in via "data" and not literally in the path. We start testing limits with limit=1 rather than limit=0, because it was confusing the defaults mechanism further up. I'm not sure limit=0 is useful anywhere at all. And some minor fixes. --- kepi/trilby_api/tests/test_timelines.py | 71 +++++++++++++++++-------- 1 file changed, 48 insertions(+), 23 deletions(-) diff --git a/kepi/trilby_api/tests/test_timelines.py b/kepi/trilby_api/tests/test_timelines.py index 9eb878e..a26db4e 100644 --- a/kepi/trilby_api/tests/test_timelines.py +++ b/kepi/trilby_api/tests/test_timelines.py @@ -35,12 +35,14 @@ class TimelineTestCase(TrilbyTestCase): def timeline_contents(self, path, + data = None, as_user = None, ): details = sorted([x['content'] \ for x in self.get( path = path, + data = data, as_user = as_user, )]) @@ -163,35 +165,40 @@ class TestPublicTimeline(TimelineTestCase): self.assertEqual( self.timeline_contents( - path = '/api/v1/timelines/public?local=true', + path = '/api/v1/timelines/public', + data = {'local': True}, ), 'AC', ) self.assertEqual( self.timeline_contents( - path = '/api/v1/timelines/public?local=false', + path = '/api/v1/timelines/public', + data = {'local': False}, ), 'ABCD', ) self.assertEqual( self.timeline_contents( - path = '/api/v1/timelines/public?remote=true', + path = '/api/v1/timelines/public', + data = {'remote': True}, ), 'BD', ) self.assertEqual( self.timeline_contents( - path = '/api/v1/timelines/public?remote=false', + path = '/api/v1/timelines/public', + data = {'remote': False}, ), 'ABCD', ) self.assertEqual( self.timeline_contents( - path = '/api/v1/timelines/public?local=true&remote=true', + path = '/api/v1/timelines/public', + data = {'local': True, 'remote': True}, ), '', ) @@ -210,7 +217,8 @@ class TestPublicTimeline(TimelineTestCase): self.assertEqual( self.timeline_contents( - path = '/api/v1/timelines/public?only_media=true', + path = '/api/v1/timelines/public', + data = {'only_media': True}, ), '', ) @@ -228,21 +236,24 @@ class TestPublicTimeline(TimelineTestCase): self.assertEqual( self.timeline_contents( - path = '/api/v1/timelines/public?since='+c_id, + path = '/api/v1/timelines/public', + data = {'since': status_c.id}, ), 'D', ) self.assertEqual( self.timeline_contents( - path = '/api/v1/timelines/public?max_id='+c_id, + path = '/api/v1/timelines/public', + data = {'max_id': status_c.id}, ), 'ABC', ) self.assertEqual( self.timeline_contents( - path = '/api/v1/timelines/public?min_id='+c_id, + path = '/api/v1/timelines/public', + data = {'min_id': status_c.id}, ), 'CD', ) @@ -260,10 +271,11 @@ class TestPublicTimeline(TimelineTestCase): visibility='A', ) - for i in range(len(alphabet)): + for i in range(1, len(alphabet)): self.assertEqual( self.timeline_contents( - path = '/api/v1/timelines/public?limit='+str(i), + path = '/api/v1/timelines/public', + data = {'limit': i}, ), alphabet[:i], ) @@ -274,7 +286,7 @@ class TestPublicTimeline(TimelineTestCase): path = '/api/v1/timelines/public', ), alphabet[:20], - message = 'default is 20', + msg = 'default is 20', ) class TestHomeTimeline(TimelineTestCase): @@ -307,7 +319,7 @@ class TestHomeTimeline(TimelineTestCase): expect_result = 401, ) - def test_simple(self): + def test_0_simple(self): self.add_standard_statuses() @@ -337,7 +349,8 @@ class TestHomeTimeline(TimelineTestCase): self.assertEqual( self.timeline_contents( - path = '/api/v1/timelines/home?since='+c_id, + path = '/api/v1/timelines/home', + data = {'since': c_id}, as_user = self.alice, ), 'D', @@ -345,7 +358,8 @@ class TestHomeTimeline(TimelineTestCase): self.assertEqual( self.timeline_contents( - path = '/api/v1/timelines/home?max_id='+c_id, + path = '/api/v1/timelines/home', + data = {'max_id': c_id}, as_user = self.alice, ), 'A', @@ -353,7 +367,8 @@ class TestHomeTimeline(TimelineTestCase): self.assertEqual( self.timeline_contents( - path = '/api/v1/timelines/home?min_id='+c_id, + path = '/api/v1/timelines/home', + data = {'min_id': c_id}, as_user = self.alice, ), 'D', @@ -363,7 +378,8 @@ class TestHomeTimeline(TimelineTestCase): self.assertEqual( self.timeline_contents( - path = '/api/v1/timelines/home?since='+c_id, + path = '/api/v1/timelines/home', + data = {'since': c_id}, as_user = self.alice, ), 'D', @@ -371,7 +387,8 @@ class TestHomeTimeline(TimelineTestCase): self.assertEqual( self.timeline_contents( - path = '/api/v1/timelines/home?max_id='+c_id, + path = '/api/v1/timelines/home', + data = {'max_id': c_id}, as_user = self.alice, ), 'ABC', @@ -379,7 +396,8 @@ class TestHomeTimeline(TimelineTestCase): self.assertEqual( self.timeline_contents( - path = '/api/v1/timelines/home?min_id='+c_id, + path = '/api/v1/timelines/home', + data = {'min_id': c_id}, as_user = self.alice, ), 'CD', @@ -391,6 +409,11 @@ class TestHomeTimeline(TimelineTestCase): self.bob = create_local_person("bob") self.carol = create_local_person("carol") + Follow( + follower=self.alice, + following=self.bob, + offer=None).save() + alphabet = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' for i in range(len(alphabet)): @@ -406,10 +429,11 @@ class TestHomeTimeline(TimelineTestCase): visibility='A', ) - for i in range(len(alphabet)): + for i in range(1, len(alphabet)): self.assertEqual( self.timeline_contents( - path = '/api/v1/timelines/home?limit='+str(i), + path = '/api/v1/timelines/home', + data = {'limit': i}, as_user = self.alice, ), alphabet[:i], @@ -422,7 +446,7 @@ class TestHomeTimeline(TimelineTestCase): as_user = self.alice, ), alphabet[:20], - message = 'default is 20', + msg = 'default is 20', ) @httpretty.activate() @@ -453,7 +477,8 @@ class TestHomeTimeline(TimelineTestCase): self.assertEqual( self.timeline_contents( - path = '/api/v1/timelines/home?local=true', + path = '/api/v1/timelines/home', + data = {'local': True}, as_user = self.alice, ), 'ABCD', From e77ce337c465b21bc809130db67acf88fc4dc195 Mon Sep 17 00:00:00 2001 From: Marnanel Thurman Date: Sun, 14 Feb 2021 22:03:37 +0000 Subject: [PATCH 09/19] Heroic attempt to work with django_rest_framework instead of fighting against it. Specifically, filter_queryset() does the filtering, and we don't attempt to provide our own get() on a ListAPIView. --- kepi/trilby_api/views/timelines.py | 78 ++++++++++++++++++++++++------ 1 file changed, 62 insertions(+), 16 deletions(-) diff --git a/kepi/trilby_api/views/timelines.py b/kepi/trilby_api/views/timelines.py index 971156a..2c28708 100644 --- a/kepi/trilby_api/views/timelines.py +++ b/kepi/trilby_api/views/timelines.py @@ -31,6 +31,7 @@ import json import re import random +DEFAULT_TIMELINE_SLICE_LENGTH = 20 class AbstractTimeline(generics.ListAPIView): @@ -39,29 +40,76 @@ class AbstractTimeline(generics.ListAPIView): IsAuthenticated, ] - def get_queryset(self, request): + def get_queryset(self): raise NotImplementedError("cannot query abstract timeline") - def get(self, request): - queryset = self.get_queryset(request) - serializer = self.serializer_class(queryset, - many = True, - context = { - 'request': request, - }) - return Response(serializer.data) + def filter_queryset(self, queryset, + min_id = None, + max_id = None, + since_id = None, + local = False, + remote = False, + limit = DEFAULT_TIMELINE_SLICE_LENGTH, + *args, **kwargs, + ): -PUBLIC_TIMELINE_SLICE_LENGTH = 20 + logger.debug("Timeline queryset: %s", queryset) + + if 'min_id' in self.request.query_params: + queryset = queryset.filter( + id__ge = int(self.request.query_params['min_id']), + ) + logger.debug(" -- after min_id: %s", queryset) + + if 'max_id' in self.request.query_params: + queryset = queryset.filter( + id__le = int(self.request.query_params['max_id']), + ) + logger.debug(" -- after max_id: %s", queryset) + + if 'since_id' in self.request.query_params: + queryset = queryset.filter( + id__gt = int(self.request.query_params['since_id']), + ) + logger.debug(" -- after since_id: %s", queryset) + + if 'local' in self.request.query_params: + queryset = queryset.filter( + local = bool(self.request.query_params['local']), + ) + logger.debug(" -- after local: %s", queryset) + + if 'remote' in self.request.query_params: + queryset = queryset.filter( + remote = bool(self.request.query_params['remote']), + ) + logger.debug(" -- after remote: %s", queryset) + + # Slicing the queryset must be done last, + # since running operations on a sliced queryset + # causes evaluation. + limit = int(self.request.query_params.get('limit', + default = DEFAULT_TIMELINE_SLICE_LENGTH, + )) + + queryset = queryset[:limit] + + logger.debug(" -- after slice of %d: %s", + limit, + queryset, + ) + + return queryset class PublicTimeline(AbstractTimeline): permission_classes = () - def get_queryset(self, request): + def get_queryset(self): result = trilby_models.Status.objects.filter( visibility = trilby_utils.VISIBILITY_PUBLIC, - )[:PUBLIC_TIMELINE_SLICE_LENGTH] + ) return result @@ -71,9 +119,9 @@ class HomeTimeline(AbstractTimeline): IsAuthenticated, ] - def get_queryset(self, request): + def get_queryset(self): - result = request.user.localperson.inbox + result = self.request.user.localperson.inbox logger.debug("Home timeline is %s", result) @@ -82,8 +130,6 @@ class HomeTimeline(AbstractTimeline): ######################################## -######################################## - class UserFeed(View): permission_classes = () From dbdad7037684dd3a4c3ae0c1e9454bdfaab41f35 Mon Sep 17 00:00:00 2001 From: Marnanel Thurman Date: Mon, 15 Feb 2021 18:21:57 +0000 Subject: [PATCH 10/19] LocalPerson.inbox uses Q objects rather than union(), because union precludes filtering later. See: https://stackoverflow.com/questions/49260393/django-filter-a-queryset-made-of-unions-not-working --- kepi/trilby_api/models/person.py | 36 ++++++++++++-------------------- 1 file changed, 13 insertions(+), 23 deletions(-) diff --git a/kepi/trilby_api/models/person.py b/kepi/trilby_api/models/person.py index a48a15c..6951667 100644 --- a/kepi/trilby_api/models/person.py +++ b/kepi/trilby_api/models/person.py @@ -9,6 +9,7 @@ logger = logging.getLogger(name='kepi') from polymorphic.models import PolymorphicModel from django.db import models +from django.db.models import Q from django.db.models.constraints import UniqueConstraint from django.contrib.auth.models import AbstractUser from django.conf import settings @@ -591,41 +592,30 @@ class LocalPerson(Person): import kepi.trilby_api.models as trilby_models - # tags aren't implemented; FIXME - everything_youre_tagged_in = trilby_models.Status.objects.none() + # "Everything you're tagged in": + # tags aren't implemented; FIXME - logger.debug("%s.inbox: tagged in: %s", - self, everything_youre_tagged_in) + all_your_posts = Q(account = self) - all_your_posts = trilby_models.Status.objects.filter( - account = self, - ) - - logger.debug("%s.inbox: all your posts: %s", - self, all_your_posts) - - all_your_friends_public_posts = trilby_models.Status.objects.filter( + all_your_friends_public_posts = Q( visibility = trilby_utils.VISIBILITY_PUBLIC, account__rel_followers__follower = self, ) - logger.debug("%s.inbox: all friends' public: %s", - self, all_your_friends_public_posts) - - all_your_mutuals_private_posts = trilby_models.Status.objects.filter( + all_your_mutuals_private_posts = Q( visibility = trilby_utils.VISIBILITY_PRIVATE, account__rel_following__following = self, account__rel_followers__follower = self, ) - logger.debug("%s.inbox: all mutuals' private: %s", - self, all_your_mutuals_private_posts) + result = trilby_models.Status.objects.filter( + all_your_posts | \ + all_your_friends_public_posts | \ + all_your_mutuals_private_posts + ) - result = everything_youre_tagged_in.union( - all_your_posts, - all_your_friends_public_posts, - all_your_mutuals_private_posts, - ) + logger.debug("%s.inbox: contains %s", + self, result) return result From 5bf75d6c989fa91a85c7f6d2ba8d402035d95ade Mon Sep 17 00:00:00 2001 From: Marnanel Thurman Date: Mon, 15 Feb 2021 18:24:34 +0000 Subject: [PATCH 11/19] Fixed expected results which were wrong --- kepi/trilby_api/tests/test_timelines.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kepi/trilby_api/tests/test_timelines.py b/kepi/trilby_api/tests/test_timelines.py index a26db4e..4492974 100644 --- a/kepi/trilby_api/tests/test_timelines.py +++ b/kepi/trilby_api/tests/test_timelines.py @@ -472,7 +472,7 @@ class TestHomeTimeline(TimelineTestCase): path = '/api/v1/timelines/home', as_user = self.alice, ), - 'ABCDPQ', + 'ADPQ', ) self.assertEqual( @@ -481,7 +481,7 @@ class TestHomeTimeline(TimelineTestCase): data = {'local': True}, as_user = self.alice, ), - 'ABCD', + 'AD', ) class TestTimelinesNotImplemented(TimelineTestCase): From 061ce40101256343273e0394f2c52f2a416e0727 Mon Sep 17 00:00:00 2001 From: Marnanel Thurman Date: Mon, 15 Feb 2021 18:24:48 +0000 Subject: [PATCH 12/19] Remote/local test for Status objects is remote_url__isnull --- kepi/trilby_api/views/timelines.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/kepi/trilby_api/views/timelines.py b/kepi/trilby_api/views/timelines.py index 2c28708..83ebcda 100644 --- a/kepi/trilby_api/views/timelines.py +++ b/kepi/trilby_api/views/timelines.py @@ -75,13 +75,15 @@ class AbstractTimeline(generics.ListAPIView): if 'local' in self.request.query_params: queryset = queryset.filter( - local = bool(self.request.query_params['local']), + remote_url__isnull = \ + bool(self.request.query_params['local']), ) logger.debug(" -- after local: %s", queryset) if 'remote' in self.request.query_params: queryset = queryset.filter( - remote = bool(self.request.query_params['remote']), + remote_url__isnull = \ + not bool(self.request.query_params['remote']), ) logger.debug(" -- after remote: %s", queryset) From 37d53b2e4e12cffaa8d1d1a574c2ff1e71f8511b Mon Sep 17 00:00:00 2001 From: Marnanel Thurman Date: Tue, 16 Feb 2021 22:58:42 +0000 Subject: [PATCH 13/19] Status.content and Status.spoiler_text are now Status.content_source and Status.spoiler_source. HTML renderings of each one are cached. You can access them at Status.content_as_html and Status.spoiler_as_html. --- kepi/bowler_pub/create.py | 4 +- kepi/bowler_pub/serializers.py | 4 +- .../tests/mastodon/test_announce.py | 2 +- kepi/bowler_pub/tests/mastodon/test_create.py | 2 +- kepi/bowler_pub/tests/test_collections.py | 18 +++- kepi/sombrero_sendpub/receivers.py | 2 +- .../migrations/0029_auto_20210216_1914.py | 38 ++++++++ kepi/trilby_api/models/status.py | 86 +++++++++++++++---- kepi/trilby_api/serializers.py | 12 ++- kepi/trilby_api/tests/__init__.py | 11 ++- kepi/trilby_api/tests/test_reblog.py | 6 +- kepi/trilby_api/tests/test_status.py | 2 +- kepi/trilby_api/tests/test_timelines.py | 66 ++++++++++++-- kepi/trilby_api/views/statuses.py | 10 +-- 14 files changed, 213 insertions(+), 50 deletions(-) create mode 100644 kepi/trilby_api/migrations/0029_auto_20210216_1914.py diff --git a/kepi/bowler_pub/create.py b/kepi/bowler_pub/create.py index f3e8afb..276b796 100644 --- a/kepi/bowler_pub/create.py +++ b/kepi/bowler_pub/create.py @@ -231,9 +231,9 @@ def on_note(fields, address): remote_url = fields['id'], account = poster, in_reply_to = in_reply_to, - content = fields['content'], + content_source = fields['content'], sensitive = is_sensitive, - spoiler_text = spoiler_text, + spoiler_source = spoiler_text, visibility = visibility, language = language, ) diff --git a/kepi/bowler_pub/serializers.py b/kepi/bowler_pub/serializers.py index e879abf..f4506de 100644 --- a/kepi/bowler_pub/serializers.py +++ b/kepi/bowler_pub/serializers.py @@ -33,7 +33,7 @@ class StatusObjectSerializer(serializers.ModelSerializer): 'id': status.url, 'url': status.url, 'type': 'Note', - 'summary': status.spoiler_text_as_html, + 'summary': status.spoiler_as_html, 'inReplyTo': status.in_reply_to, 'published': status.created_at, 'attributedTo': status.account.url, @@ -43,7 +43,7 @@ class StatusObjectSerializer(serializers.ModelSerializer): 'conversation': status.conversation, 'content': status.content_as_html, 'contentMap': { - status.language: status.content, + status.language: status.content_source, }, 'attachment': status.media_attachments, 'tag': status.tags, diff --git a/kepi/bowler_pub/tests/mastodon/test_announce.py b/kepi/bowler_pub/tests/mastodon/test_announce.py index 649af6c..40095e7 100644 --- a/kepi/bowler_pub/tests/mastodon/test_announce.py +++ b/kepi/bowler_pub/tests/mastodon/test_announce.py @@ -125,7 +125,7 @@ class Tests(TestCase): ) self.assertEqual( - original_status.content, + original_status.content_source, 'Hello world', msg = 'the status was reblogged at the end', ) diff --git a/kepi/bowler_pub/tests/mastodon/test_create.py b/kepi/bowler_pub/tests/mastodon/test_create.py index 6ac66f0..1788f2d 100644 --- a/kepi/bowler_pub/tests/mastodon/test_create.py +++ b/kepi/bowler_pub/tests/mastodon/test_create.py @@ -69,7 +69,7 @@ class Tests(Create_TestCase): import kepi.trilby_api.models as trilby_models result = trilby_models.Status.objects.filter( - content = content, + content_source = content, ) if result: diff --git a/kepi/bowler_pub/tests/test_collections.py b/kepi/bowler_pub/tests/test_collections.py index a131652..505213d 100644 --- a/kepi/bowler_pub/tests/test_collections.py +++ b/kepi/bowler_pub/tests/test_collections.py @@ -82,8 +82,22 @@ class Tests(TestCase): result = trilby_models.Status( account = self._alice, visibility = trilby_utils.VISIBILITY_PUBLIC, - content = "

Victoria Wood parodying Peter Skellern. I laughed so much at this, though you might have to know both singers' work in order to find it quite as funny.

- love song
- self-doubt
- refs to northern England
- preamble
- piano solo
- brass band
- choir backing
- love is cosy
- heavy rhotic vowels

https://youtu.be/782hqdmnq7g

", - ) + content_source = """Victoria Wood parodying Peter Skellern. +I laughed so much at this, though you might have to know both singers' work +in order to find it quite as funny: + +- love song +- self-doubt +- refs to northern England +- preamble +- piano solo +- brass band +- choir backing +- love is cosy +- heavy rhotic vowels + +https://youtu.be/782hqdmnq7g""", + ) result.save() return result diff --git a/kepi/sombrero_sendpub/receivers.py b/kepi/sombrero_sendpub/receivers.py index ab43947..29f8570 100644 --- a/kepi/sombrero_sendpub/receivers.py +++ b/kepi/sombrero_sendpub/receivers.py @@ -59,7 +59,7 @@ def on_posted(sender, **kwargs): "object": { "type": "Note", "id": sender.url, - "content": sender.content, + "content": sender.content_as_html, } }, sender = sender.account, diff --git a/kepi/trilby_api/migrations/0029_auto_20210216_1914.py b/kepi/trilby_api/migrations/0029_auto_20210216_1914.py new file mode 100644 index 0000000..65dcf9c --- /dev/null +++ b/kepi/trilby_api/migrations/0029_auto_20210216_1914.py @@ -0,0 +1,38 @@ +# Generated by Django 3.0.9 on 2021-02-16 19:14 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('trilby_api', '0028_mention'), + ] + + operations = [ + migrations.RenameField( + model_name='status', + old_name='spoiler_text', + new_name='spoiler_source', + ), + migrations.RemoveField( + model_name='status', + name='content', + ), + migrations.AddField( + model_name='status', + name='content_as_html_denormed', + field=models.TextField(default=None, editable=False, help_text='HTML rendering of content_source. Do not edit!', null=True), + ), + migrations.AddField( + model_name='status', + name='content_source', + field=models.TextField(default='', help_text='Text of the status, as entered'), + preserve_default=False, + ), + migrations.AddField( + model_name='status', + name='spoiler_as_html_denormed', + field=models.CharField(default=None, editable=False, max_length=255, null=True), + ), + ] diff --git a/kepi/trilby_api/models/status.py b/kepi/trilby_api/models/status.py index 826ae04..902bd30 100644 --- a/kepi/trilby_api/models/status.py +++ b/kepi/trilby_api/models/status.py @@ -59,7 +59,15 @@ class Status(PolymorphicModel): blank = True, ) - content = models.TextField( + content_source = models.TextField( + help_text = 'Text of the status, as entered', + ) + + content_as_html_denormed = models.TextField( + help_text = 'HTML rendering of content_source. Do not edit!', + editable = False, + null = True, + default = None, ) created_at = models.DateTimeField( @@ -72,13 +80,20 @@ class Status(PolymorphicModel): default = False, ) - spoiler_text = models.CharField( + spoiler_source = models.CharField( max_length = 255, null = True, blank = True, default = '', ) + spoiler_as_html_denormed = models.CharField( + max_length = 255, + null = True, + editable = False, + default = None, + ) + visibility = models.CharField( max_length = 1, default = trilby_utils.VISIBILITY_PUBLIC, @@ -107,6 +122,44 @@ class Status(PolymorphicModel): default = None, ) + @property + def content_as_html(self): + """ + Returns an HTML rendition of content_source. + The return value will be cached. + Saving the record will clear this cache. + """ + + if self.content_as_html_denormed is not None: + return self.content_as_html_denormed + + if self.content_source is None: + result = '

' + else: + result = markdown.markdown(self.content_source) + + self.content_as_html_denormed = result + return result + + @property + def spoiler_as_html(self): + """ + Returns an HTML rendition of spoiler_source. + The return value will be cached. + Saving the record will clear this cache. + """ + + if self.spoiler_as_html_denormed is not None: + return self.spoiler_as_html_denormed + + if self.spoiler_source is None: + result = '

' + else: + result = markdown.markdown(self.spoiler_source) + + self.spoiler_as_html_denormed = result + return result + @property def emojis(self): return [] # TODO @@ -268,6 +321,19 @@ class Status(PolymorphicModel): if self.in_reply_to == self: raise ValueError("Status can't be a reply to itself") + if not newly_made: + old = self.__class__.objects.get(pk=self.pk) + + if self.content_source != old.content_source: + logger.debug("%s: content changed; flushing HTML cache", + self) + self.content_as_html_denormed = None + + if self.spoiler_source != old.spoiler_source: + logger.debug("%s: spoiler changed; flushing HTML cache", + self) + self.spoiler_as_html_denormed = None + super().save(*args, **kwargs) if send_signal and newly_made: @@ -280,7 +346,7 @@ class Status(PolymorphicModel): def __str__(self): return '%s: %s' % ( self.id, - self.content, + self.content_source, ) @classmethod @@ -350,20 +416,8 @@ class Status(PolymorphicModel): # HTML and one is plain text. But the docs don't # seem to be forthcoming on this point, so we'll # just have to wait until we find out. - return self.content + return self.content_source @property def is_local(self): return self.remote_url is None - - @property - def content_as_html(self): - if not self.content: - return '

' - return markdown.markdown(self.content) - - @property - def spoiler_text_as_html(self): - if not self.spoiler_text: - return '

' - return markdown.markdown(self.spoiler_text) diff --git a/kepi/trilby_api/serializers.py b/kepi/trilby_api/serializers.py index 123d359..0834a76 100644 --- a/kepi/trilby_api/serializers.py +++ b/kepi/trilby_api/serializers.py @@ -208,26 +208,24 @@ class StatusSerializer(serializers.ModelSerializer): # "content" is read-only for HTML; # "status" is write-only for text (or Markdown) - content = serializers.SerializerMethodField( + content = serializers.CharField( + source='content_as_html', read_only = True) status = serializers.CharField( - source='source_text', + source='content_source', write_only = True) - def get_content(self, status): - result = markdown.markdown(status.content) - return result - created_at = serializers.DateTimeField( required = False, read_only = True) - # TODO Media + # TODO Media sensitive = serializers.BooleanField( required = False) spoiler_text = serializers.CharField( + source='spoiler_source', allow_blank = True, required = False) diff --git a/kepi/trilby_api/tests/__init__.py b/kepi/trilby_api/tests/__init__.py index d52774f..ecdbf5c 100644 --- a/kepi/trilby_api/tests/__init__.py +++ b/kepi/trilby_api/tests/__init__.py @@ -1,9 +1,18 @@ +# trilby_api/tests/__init__.py +# +# Part of kepi. +# Copyright (c) 2018-2021 Marnanel Thurman. +# Licensed under the GNU Public License v2. + from django.test import TestCase, Client from rest_framework.test import force_authenticate, APIClient from kepi.trilby_api.models import * from django.conf import settings import json +import logging +logger = logging.getLogger(name='kepi') + ACCOUNT_EXPECTED = { 'username': 'alice', 'acct': 'alice', @@ -154,7 +163,7 @@ def create_local_status( result = Status( remote_url = None, account = posted_by, - content = content, + content_source = content, **kwargs, ) diff --git a/kepi/trilby_api/tests/test_reblog.py b/kepi/trilby_api/tests/test_reblog.py index 691a183..0f71b19 100644 --- a/kepi/trilby_api/tests/test_reblog.py +++ b/kepi/trilby_api/tests/test_reblog.py @@ -23,7 +23,7 @@ class TestReblog(TestCase): ) reblog = create_local_status( - content = original.content, + content = original.content_source, posted_by = bob, reblog_of = original, ) @@ -48,7 +48,7 @@ class TestReblog(TestCase): ) reblog = create_local_status( - content = original.content, + content = original.content_source, posted_by = bob, reblog_of = original, ) @@ -87,7 +87,7 @@ class TestReblog(TestCase): for i in range(1, 20): reblog = create_local_status( - content = original.content, + content = original.content_source, posted_by = bob, reblog_of = original, ) diff --git a/kepi/trilby_api/tests/test_status.py b/kepi/trilby_api/tests/test_status.py index 0b53b57..a489e26 100644 --- a/kepi/trilby_api/tests/test_status.py +++ b/kepi/trilby_api/tests/test_status.py @@ -685,7 +685,7 @@ class TestGetStatus(TrilbyTestCase): remote_url = "https://example.org/people/bob/status/100", account = self._bob, in_reply_to = self._alice_status, - content = "Buttercups our gold.", + content_source = "Buttercups our gold.", ) self._bob_status.save() diff --git a/kepi/trilby_api/tests/test_timelines.py b/kepi/trilby_api/tests/test_timelines.py index 4492974..1eac3ff 100644 --- a/kepi/trilby_api/tests/test_timelines.py +++ b/kepi/trilby_api/tests/test_timelines.py @@ -24,7 +24,7 @@ class TimelineTestCase(TrilbyTestCase): def add_status(self, source, visibility, content): status = Status( account = source, - content = content, + content_source = content, visibility = visibility, ) status.save() @@ -39,12 +39,22 @@ class TimelineTestCase(TrilbyTestCase): as_user = None, ): - details = sorted([x['content'] \ - for x in self.get( + logger.info("Timeline contents for %s as %s...", + path, + as_user) + + found = self.get( path = path, data = data, as_user = as_user, - )]) + ) + + logger.info(" -- retrieved") + + details = sorted([x['content'] for x in found]) + + logger.debug(" -- sorted as %s", + details) result = '' for detail in details: @@ -54,10 +64,7 @@ class TimelineTestCase(TrilbyTestCase): result += detail - logger.info("Timeline contents for %s as %s...", - path, - as_user) - logger.info(" ...are %s", + logger.info(" -- contents are %s", result) return result @@ -449,6 +456,49 @@ class TestHomeTimeline(TimelineTestCase): msg = 'default is 20', ) + def temp_general_test_limit(self, count): + # XXX temp + + self.alice = create_local_person("alice") + self.bob = create_local_person("bob") + self.carol = create_local_person("carol") + + Follow( + follower=self.alice, + following=self.bob, + offer=None).save() + + for i in range(100): + self.add_status( + source=self.bob, + content=str(i), + visibility='A', + ) + + self.add_status( + source=self.carol, + content=str(i), + visibility='A', + ) + + LOOP_COUNT = 50 + for j in range(LOOP_COUNT): + logger.info("----------- Loop: %d of %d", j, LOOP_COUNT) + for i in [count]: + self.assertIsNotNone( + self.timeline_contents( + path = '/api/v1/timelines/home', + data = {'limit': i}, + as_user = self.alice, + ), + ) + + def xxx_test_1(self): + self.temp_general_test_limit(1) + + def xxx_test_100(self): + self.temp_general_test_limit(100) + @httpretty.activate() def test_local(self): diff --git a/kepi/trilby_api/views/statuses.py b/kepi/trilby_api/views/statuses.py index a11ba13..4b1e7ef 100644 --- a/kepi/trilby_api/views/statuses.py +++ b/kepi/trilby_api/views/statuses.py @@ -135,18 +135,18 @@ class Reblog(DoSomethingWithStatus): # https://github.com/tootsuite/mastodon/issues/13479 # For now, I'm assuming that you can. - content = 'RT {}'.format(the_status.content) + content_source = 'RT {}'.format(the_status.content_source) new_status = trilby_models.Status( # Fields which are different in a reblog: account = request.user.localperson, - content = content, + content_source = content_source, reblog_of = the_status, # Fields which are just copied in: sensitive = the_status.sensitive, - spoiler_text = the_status.spoiler_text, + spoiler_source = the_status.spoiler_source, visibility = the_status.visibility, language = the_status.language, in_reply_to = the_status.in_reply_to, @@ -397,9 +397,9 @@ class Statuses(generics.ListCreateAPIView, status = trilby_models.Status( account = request.user.localperson, - content = data.get('status', ''), + content_source = data.get('status', ''), sensitive = data.get('sensitive', False), - spoiler_text = data.get('spoiler_text', ''), + spoiler_source = data.get('spoiler_text', ''), visibility = data.get('visibility', 'public'), language = data.get('language', settings.KEPI['LANGUAGES'][0]), From 3ad00cfae0ddabec5dfa24de072e0f22a118f2a6 Mon Sep 17 00:00:00 2001 From: Marnanel Thurman Date: Wed, 17 Feb 2021 22:44:21 +0000 Subject: [PATCH 14/19] tests for remote and local params on timelines fixed to produce actual remote statuses! --- kepi/trilby_api/tests/test_timelines.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/kepi/trilby_api/tests/test_timelines.py b/kepi/trilby_api/tests/test_timelines.py index 1eac3ff..54ef8fc 100644 --- a/kepi/trilby_api/tests/test_timelines.py +++ b/kepi/trilby_api/tests/test_timelines.py @@ -21,11 +21,13 @@ import httpretty class TimelineTestCase(TrilbyTestCase): - def add_status(self, source, visibility, content): + def add_status(self, source, visibility, content, + remote_url = None): status = Status( account = source, content_source = content, visibility = visibility, + remote_url = remote_url, ) status.save() @@ -159,9 +161,11 @@ class TestPublicTimeline(TimelineTestCase): ) self.add_status(source=alice, content='A', visibility='A') - self.add_status(source=peter, content='B', visibility='A') + self.add_status(source=peter, content='B', visibility='A', + remote_url = 'https://example.com/users/peter/B') self.add_status(source=alice, content='C', visibility='A') - self.add_status(source=peter, content='D', visibility='A') + self.add_status(source=peter, content='D', visibility='A', + remote_url = 'https://example.com/users/peter/D') self.assertEqual( self.timeline_contents( @@ -509,8 +513,15 @@ class TestHomeTimeline(TimelineTestCase): name = "peter", auto_fetch = True, ) - self.add_status(source=self.peter, content='P', visibility='A') - self.add_status(source=self.peter, content='Q', visibility='A') + + for letter in 'PQ': + self.add_status(source=self.peter, + remote_url = 'https://example.com/users/peter/{}'.format( + letter, + ), + content=letter, + visibility='A') + Follow( follower = self.alice, following = self.peter, From 31dede8c2c000d7ef1fcb4c7424061576310663c Mon Sep 17 00:00:00 2001 From: Marnanel Thurman Date: Thu, 18 Feb 2021 18:36:37 +0000 Subject: [PATCH 15/19] Better logging for inbox calculation --- kepi/trilby_api/models/person.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/kepi/trilby_api/models/person.py b/kepi/trilby_api/models/person.py index 6951667..62fc20e 100644 --- a/kepi/trilby_api/models/person.py +++ b/kepi/trilby_api/models/person.py @@ -597,24 +597,46 @@ class LocalPerson(Person): all_your_posts = Q(account = self) + # note: querysets don't get evaluated unless used, + # so the debug logging doesn't cause a db hit + # unless it's actually turned on. + + logger.debug("%s.inbox: your own posts: %s", + self, + trilby_models.Status.objects.filter( + all_your_posts + )) + all_your_friends_public_posts = Q( visibility = trilby_utils.VISIBILITY_PUBLIC, account__rel_followers__follower = self, ) + logger.debug("%s.inbox: your friends' public posts: %s", + self, + trilby_models.Status.objects.filter( + all_your_friends_public_posts + )) + all_your_mutuals_private_posts = Q( visibility = trilby_utils.VISIBILITY_PRIVATE, account__rel_following__following = self, account__rel_followers__follower = self, ) + logger.debug("%s.inbox: your mutuals' private posts: %s", + self, + trilby_models.Status.objects.filter( + all_your_mutuals_private_posts + )) + result = trilby_models.Status.objects.filter( all_your_posts | \ all_your_friends_public_posts | \ all_your_mutuals_private_posts ) - logger.debug("%s.inbox: contains %s", + logger.info("%s.inbox: contains %s", self, result) return result From 46b3dcfbf70de0530e20af861470eadedc8227ce Mon Sep 17 00:00:00 2001 From: Marnanel Thurman Date: Thu, 18 Feb 2021 18:37:11 +0000 Subject: [PATCH 16/19] __ge and __le filters replaced with the correct __gte and __lte --- kepi/trilby_api/views/timelines.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kepi/trilby_api/views/timelines.py b/kepi/trilby_api/views/timelines.py index 83ebcda..2eb7d22 100644 --- a/kepi/trilby_api/views/timelines.py +++ b/kepi/trilby_api/views/timelines.py @@ -57,13 +57,13 @@ class AbstractTimeline(generics.ListAPIView): if 'min_id' in self.request.query_params: queryset = queryset.filter( - id__ge = int(self.request.query_params['min_id']), + id__gte = int(self.request.query_params['min_id']), ) logger.debug(" -- after min_id: %s", queryset) if 'max_id' in self.request.query_params: queryset = queryset.filter( - id__le = int(self.request.query_params['max_id']), + id__lte = int(self.request.query_params['max_id']), ) logger.debug(" -- after max_id: %s", queryset) From a5848e45e2a8f04827473a15b3319d888a389f2c Mon Sep 17 00:00:00 2001 From: Marnanel Thurman Date: Thu, 18 Feb 2021 18:37:43 +0000 Subject: [PATCH 17/19] test_as_follower moved to home timeline test, rather than public timeline test. Test of "since" param, which doesn't exist, replaced with the correct "since_id". Removed a lot of debug code that shouldn't have been checked in. Fixed some comments. --- kepi/trilby_api/tests/test_timelines.py | 122 ++++++++++-------------- 1 file changed, 48 insertions(+), 74 deletions(-) diff --git a/kepi/trilby_api/tests/test_timelines.py b/kepi/trilby_api/tests/test_timelines.py index 54ef8fc..0bb869b 100644 --- a/kepi/trilby_api/tests/test_timelines.py +++ b/kepi/trilby_api/tests/test_timelines.py @@ -1,7 +1,7 @@ # test_timelines.py # # Part of kepi. -# Copyright (c) 2018-2020 Marnanel Thurman. +# Copyright (c) 2018-2021 Marnanel Thurman. # Licensed under the GNU Public License v2. import logging @@ -16,8 +16,10 @@ from django.conf import settings from unittest import skip import httpretty -# Tests for timelines. API docs are here: -# https://docs.joinmastodon.org/methods/statuses/ +""" +Tests for timelines. API docs are here: +https://docs.joinmastodon.org/methods/timelines/ +""" class TimelineTestCase(TrilbyTestCase): @@ -107,31 +109,6 @@ class TestPublicTimeline(TimelineTestCase): 'A', ) - def test_as_follower(self): - - alice = create_local_person("alice") - george = create_local_person("george") - - follow = Follow( - follower = george, - following = alice, - offer = None, - ) - follow.save() - - self.add_status(source=alice, content='A', visibility='A') - self.add_status(source=alice, content='B', visibility='U') - self.add_status(source=alice, content='C', visibility='X') - self.add_status(source=alice, content='D', visibility='D') - - self.assertEqual( - self.timeline_contents( - path = '/api/v1/timelines/public', - as_user = george, - ), - 'AC', - ) - def test_as_stranger(self): alice = create_local_person("alice") @@ -248,7 +225,7 @@ class TestPublicTimeline(TimelineTestCase): self.assertEqual( self.timeline_contents( path = '/api/v1/timelines/public', - data = {'since': status_c.id}, + data = {'since_id': status_c.id}, ), 'D', ) @@ -361,7 +338,7 @@ class TestHomeTimeline(TimelineTestCase): self.assertEqual( self.timeline_contents( path = '/api/v1/timelines/home', - data = {'since': c_id}, + data = {'since_id': c_id}, as_user = self.alice, ), 'D', @@ -390,7 +367,7 @@ class TestHomeTimeline(TimelineTestCase): self.assertEqual( self.timeline_contents( path = '/api/v1/timelines/home', - data = {'since': c_id}, + data = {'since_id': c_id}, as_user = self.alice, ), 'D', @@ -460,49 +437,6 @@ class TestHomeTimeline(TimelineTestCase): msg = 'default is 20', ) - def temp_general_test_limit(self, count): - # XXX temp - - self.alice = create_local_person("alice") - self.bob = create_local_person("bob") - self.carol = create_local_person("carol") - - Follow( - follower=self.alice, - following=self.bob, - offer=None).save() - - for i in range(100): - self.add_status( - source=self.bob, - content=str(i), - visibility='A', - ) - - self.add_status( - source=self.carol, - content=str(i), - visibility='A', - ) - - LOOP_COUNT = 50 - for j in range(LOOP_COUNT): - logger.info("----------- Loop: %d of %d", j, LOOP_COUNT) - for i in [count]: - self.assertIsNotNone( - self.timeline_contents( - path = '/api/v1/timelines/home', - data = {'limit': i}, - as_user = self.alice, - ), - ) - - def xxx_test_1(self): - self.temp_general_test_limit(1) - - def xxx_test_100(self): - self.temp_general_test_limit(100) - @httpretty.activate() def test_local(self): @@ -545,6 +479,46 @@ class TestHomeTimeline(TimelineTestCase): 'AD', ) + def test_as_follower(self): + + alice = create_local_person("alice") + george = create_local_person("george") + + follow = Follow( + follower = george, + following = alice, + offer = None, + ) + follow.save() + + self.add_status(source=alice, content='A', visibility='A') + self.add_status(source=alice, content='B', visibility='U') + self.add_status(source=alice, content='C', visibility='X') + self.add_status(source=alice, content='D', visibility='D') + + self.assertEqual( + self.timeline_contents( + path = '/api/v1/timelines/home', + as_user = george, + ), + 'A', + ) + + follow = Follow( + follower = alice, + following = george, + offer = None, + ) + follow.save() # they are now mutuals + + self.assertEqual( + self.timeline_contents( + path = '/api/v1/timelines/home', + as_user = george, + ), + 'AC', + ) + class TestTimelinesNotImplemented(TimelineTestCase): @skip("to be implemented later") def test_hashtag(self): From 574176b42be3e856a830bebcd8f18e070554e5c3 Mon Sep 17 00:00:00 2001 From: Marnanel Thurman Date: Thu, 18 Feb 2021 18:47:45 +0000 Subject: [PATCH 18/19] "only_media" stub --- kepi/trilby_api/views/timelines.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/kepi/trilby_api/views/timelines.py b/kepi/trilby_api/views/timelines.py index 2eb7d22..50db881 100644 --- a/kepi/trilby_api/views/timelines.py +++ b/kepi/trilby_api/views/timelines.py @@ -87,6 +87,12 @@ class AbstractTimeline(generics.ListAPIView): ) logger.debug(" -- after remote: %s", queryset) + if 'only_media' in self.request.query_params: + # We don't support media at present, so this will give us + # the empty set + queryset = queryset.none() + logger.debug(" -- after only_media: %s", queryset) + # Slicing the queryset must be done last, # since running operations on a sliced queryset # causes evaluation. From 71c6d26ad835b3e29b2c7f2e1c88dc53cbfb4d8c Mon Sep 17 00:00:00 2001 From: Marnanel Thurman Date: Thu, 18 Feb 2021 19:24:48 +0000 Subject: [PATCH 19/19] Param values are strings, not bools. All tests now pass! --- kepi/trilby_api/tests/test_timelines.py | 14 +++++++------- kepi/trilby_api/views/timelines.py | 10 ++++------ 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/kepi/trilby_api/tests/test_timelines.py b/kepi/trilby_api/tests/test_timelines.py index 0bb869b..3b03da5 100644 --- a/kepi/trilby_api/tests/test_timelines.py +++ b/kepi/trilby_api/tests/test_timelines.py @@ -154,7 +154,7 @@ class TestPublicTimeline(TimelineTestCase): self.assertEqual( self.timeline_contents( path = '/api/v1/timelines/public', - data = {'local': True}, + data = {'local': 'true'}, ), 'AC', ) @@ -162,7 +162,7 @@ class TestPublicTimeline(TimelineTestCase): self.assertEqual( self.timeline_contents( path = '/api/v1/timelines/public', - data = {'local': False}, + data = {'local': 'false'}, ), 'ABCD', ) @@ -170,7 +170,7 @@ class TestPublicTimeline(TimelineTestCase): self.assertEqual( self.timeline_contents( path = '/api/v1/timelines/public', - data = {'remote': True}, + data = {'remote': 'true'}, ), 'BD', ) @@ -178,7 +178,7 @@ class TestPublicTimeline(TimelineTestCase): self.assertEqual( self.timeline_contents( path = '/api/v1/timelines/public', - data = {'remote': False}, + data = {'remote': 'false'}, ), 'ABCD', ) @@ -186,7 +186,7 @@ class TestPublicTimeline(TimelineTestCase): self.assertEqual( self.timeline_contents( path = '/api/v1/timelines/public', - data = {'local': True, 'remote': True}, + data = {'local': 'true', 'remote': 'true'}, ), '', ) @@ -206,7 +206,7 @@ class TestPublicTimeline(TimelineTestCase): self.assertEqual( self.timeline_contents( path = '/api/v1/timelines/public', - data = {'only_media': True}, + data = {'only_media': 'true'}, ), '', ) @@ -473,7 +473,7 @@ class TestHomeTimeline(TimelineTestCase): self.assertEqual( self.timeline_contents( path = '/api/v1/timelines/home', - data = {'local': True}, + data = {'local': 'true'}, as_user = self.alice, ), 'AD', diff --git a/kepi/trilby_api/views/timelines.py b/kepi/trilby_api/views/timelines.py index 50db881..5c2c595 100644 --- a/kepi/trilby_api/views/timelines.py +++ b/kepi/trilby_api/views/timelines.py @@ -73,17 +73,15 @@ class AbstractTimeline(generics.ListAPIView): ) logger.debug(" -- after since_id: %s", queryset) - if 'local' in self.request.query_params: + if self.request.query_params.get('local', '')=='true': queryset = queryset.filter( - remote_url__isnull = \ - bool(self.request.query_params['local']), + remote_url__isnull = True, ) logger.debug(" -- after local: %s", queryset) - if 'remote' in self.request.query_params: + if self.request.query_params.get('remote', '')=='true': queryset = queryset.filter( - remote_url__isnull = \ - not bool(self.request.query_params['remote']), + remote_url__isnull = False, ) logger.debug(" -- after remote: %s", queryset)