cache models.get_originals in memory

for #1149

common.py

@@ -456,6 +456,8 @@ def memcache_key(key):
 def memcache_memoize(expire=None):
     """Memoize function decorator that stores the cached value in memcache.
 
+    NOT YET WORKING! CURRENTLY UNUSED!
+
     Only caches non-null/empty values.
 
     Args:

models.py

@@ -1,6 +1,7 @@
 """Datastore model classes."""
 import copy
 from datetime import timedelta, timezone
+from functools import lru_cache
 import itertools
 import json
 import logging
@@ -30,7 +31,6 @@ from common import (
     base64_to_long,
     DOMAIN_RE,
     long_to_base64,
-    memcache_memoize,
     OLD_ACCOUNT_AGE,
     remove,
     report_error,
@@ -1135,7 +1135,7 @@ class Object(StringIdModel):
 
         # batch lookup matching users
         origs = {}  # maps str copy URI to str original URI
-        for obj in get_originals(ids):
+        for obj in get_originals(tuple(ids)):
             for copy in obj.copies:
                 if copy.protocol in (self_proto.LABEL, self_proto.ABBREV):
                     origs[copy.uri] = obj.key.id()
@@ -1559,20 +1559,19 @@ def get_original(copy_id, keys_only=None):
     Returns:
       User or Object:
     """
-    got = get_originals([copy_id], keys_only=keys_only)
+    got = get_originals((copy_id,), keys_only=keys_only)
     if got:
         return got[0]
 
 
-# TODO
+@lru_cache(maxsize=10000)
-# @memcache_memoize(expire=60 * 60)  # 1h
 def get_originals(copy_ids, keys_only=None):
     """Fetches users (across all protocols) for a given set of copies.
 
     Also see :Object:`get_copy` and :User:`get_copy`.
 
     Args:
-      copy_ids (sequence of str)
+      copy_ids (tuple (not list!) of str)
       keys_only (bool): passed through to :class:`google.cloud.ndb.Query`
 
     Returns:

protocol.py

@@ -42,7 +42,6 @@ from ids import (
 )
 from models import (
     Follower,
-    get_originals,
     Object,
     PROTOCOLS,
     PROTOCOLS_BY_KIND,

tests/test_models.py

@@ -918,6 +918,8 @@ class ObjectTest(TestCase):
         obj.resolve_ids()
         self.assert_equals(follow, obj.our_as1)
 
+        models.get_originals.cache_clear()
+
         # matching copy users
         self.make_user('other:alice', cls=OtherFake,
                        copies=[Target(uri='fake:alice', protocol='fake')])
@@ -953,6 +955,8 @@ class ObjectTest(TestCase):
         obj.resolve_ids()
         self.assert_equals(reply, obj.our_as1)
 
+        models.get_originals.cache_clear()
+
         # matching copies
         self.make_user('other:alice', cls=OtherFake,
                        copies=[Target(uri='fake:alice', protocol='fake')])
@@ -991,6 +995,8 @@ class ObjectTest(TestCase):
         obj.resolve_ids()
         self.assert_equals(note, obj.our_as1)
 
+        models.get_originals.cache_clear()
+
         # matching copies
         self.store_object(id='other:a',
                           copies=[Target(uri='fake:a', protocol='fa')])
@@ -1095,23 +1101,18 @@ class ObjectTest(TestCase):
         }, obj.our_as1)
 
     def test_get_originals(self):
-        self.assertEqual([], models.get_originals(['foo', 'did:plc:bar']))
+        self.assertEqual([], models.get_originals(('foo', 'did:plc:bar')))
 
         obj = self.store_object(id='fake:post',
                                 copies=[Target(uri='other:foo', protocol='other')])
         user = self.make_user('other:user', cls=OtherFake,
                               copies=[Target(uri='fake:bar', protocol='fake')])
 
-        # TODO
+        self.assert_entities_equal(
-        # memcache_key = "get_originals-(['other:foo',%20'fake:bar',%20'baz'],)-{}"
+            [obj, user], models.get_originals(('other:foo', 'fake:bar', 'baz')))
-        # self.assertIsNone(common.memcache.get(memcache_key))
 
         self.assert_entities_equal(
-            [obj, user], models.get_originals(['other:foo', 'fake:bar', 'baz']))
+            [obj, user], models.get_originals(('other:foo', 'fake:bar', 'baz')))
-
-        # self.assertIsNotNone(common.memcache.get(memcache_key))
-        self.assert_entities_equal(
-            [obj, user], models.get_originals(['other:foo', 'fake:bar', 'baz']))
 
     def test_get_copy(self):
         obj = Object(id='x')

tests/test_protocol.py

@@ -12,7 +12,7 @@ from google.cloud import ndb
 from google.cloud.ndb.global_cache import _InProcessGlobalCache
 from granary import as2
 from granary.tests.test_bluesky import ACTOR_PROFILE_BSKY
-from oauth_dropins.webutil import appengine_info, models, util
+from oauth_dropins.webutil import appengine_info, util
 from oauth_dropins.webutil.appengine_config import ndb_client
 from oauth_dropins.webutil.flask_util import NoContent
 from oauth_dropins.webutil.testutil import NOW, requests_response
@@ -26,6 +26,7 @@ from activitypub import ActivityPub
 from app import app
 from atproto import ATProto
 import common
+import models
 from models import Follower, Object, PROTOCOLS, Target, User
 import protocol
 from protocol import ErrorButDoNotRetryTask, Protocol
@@ -2295,6 +2296,7 @@ class ProtocolReceiveTest(TestCase):
                        copies=[Target(uri='fake:bob', protocol='fake')])
 
         common.memcache.clear()
+        models.get_originals.cache_clear()
 
         obj.new = True
         OtherFake.fetchable = {
@@ -2327,6 +2329,7 @@ class ProtocolReceiveTest(TestCase):
                           copies=[Target(uri='fake:post', protocol='fake')])
 
         common.memcache.clear()
+        models.get_originals.cache_clear()
         obj.new = True
 
         _, code = Fake.receive(obj, authed_as='fake:alice')
@@ -2375,7 +2378,7 @@ class ProtocolReceiveTest(TestCase):
             id='fake:post', our_as1={'foo': 9}, source_protocol='fake',
             copies=[Target(uri='other:post', protocol='other')])
 
-        common.memcache.clear()
+        models.get_originals.cache_clear()
 
         obj.new = True
         self.assertEqual(('OK', 202),

tests/testutil.py

@@ -280,6 +280,7 @@ class TestCase(unittest.TestCase, testutil.Asserts):
 
         memcache.clear()
         global_cache.clear()
+        models.get_originals.cache_clear()
         activitypub.WEB_OPT_OUT_DOMAINS = set()
 
         # clear datastore