diff --git a/activitypub.py b/activitypub.py index 047ddd2..009d3aa 100644 --- a/activitypub.py +++ b/activitypub.py @@ -1034,7 +1034,7 @@ def inbox(protocol=None, id=None): logger.info(f'Discarding, {id} is on an opted out domain') return '', 204 - memcache_key = f'AP-id-{id}' + memcache_key = common.memcache_key(f'AP-id-{id}') if memcache.get(memcache_key): logger.info(f'Already seen this activity {id}') return '', 204 diff --git a/common.py b/common.py index ae5fbe8..c1d6aa5 100644 --- a/common.py +++ b/common.py @@ -91,6 +91,9 @@ RUN_TASKS_INLINE = False # overridden by unit tests # for Protocol.REQUIRES_OLD_ACCOUNT, how old is old enough OLD_ACCOUNT_AGE = timedelta(days=14) +# https://github.com/memcached/memcached/wiki/Commands#standard-protocol +MEMCACHE_KEY_MAX_LEN = 250 + if appengine_info.DEBUG or appengine_info.LOCAL_SERVER: logger.info('Using in memory mock memcache') memcache = MockMemcacheClient() @@ -98,7 +101,8 @@ if appengine_info.DEBUG or appengine_info.LOCAL_SERVER: else: logger.info('Using production Memorystore memcache') memcache = pymemcache.client.base.PooledClient( - '10.126.144.3', timeout=10, connect_timeout=10) # seconds + '10.126.144.3', timeout=10, connect_timeout=10, # seconds + allow_unicode_keys=True) global_cache = MemcacheCache(memcache) @@ -397,3 +401,15 @@ def global_cache_timeout_policy(key): return int(timedelta(hours=2).total_seconds()) return int(timedelta(minutes=30).total_seconds()) + + +def memcache_key(key): + """Preprocesses a memcache key. Right now just truncates it to 250 chars. + + https://pymemcache.readthedocs.io/en/latest/apidoc/pymemcache.client.base.html + https://github.com/memcached/memcached/wiki/Commands#standard-protocol + + TODO: truncate to 250 *UTF-8* chars, to handle Unicode chars in URLs. Related: + pymemcache Client's allow_unicode_keys constructor kwarg. + """ + return key[:MEMCACHE_KEY_MAX_LEN] diff --git a/protocol.py b/protocol.py index 9443d6f..08df343 100644 --- a/protocol.py +++ b/protocol.py @@ -770,7 +770,7 @@ class Protocol: error(f'Activity {id} is blocklisted') # lease this object atomically - lease_memcache_key = f'receive-{id}' + lease_memcache_key = common.memcache_key(f'receive-{id}') if not common.memcache.add(lease_memcache_key, 'leased', noreply=False, expire=5 * 60): # 5 min error('This object is already being received elsewhere', status=204)