diff --git a/models.py b/models.py index dd10ea9..60aa444 100644 --- a/models.py +++ b/models.py @@ -1,6 +1,6 @@ """Datastore model classes.""" import base64 -from datetime import timezone +from datetime import timedelta, timezone import difflib import itertools import logging @@ -36,6 +36,19 @@ PROTOCOLS = ('activitypub', 'bluesky', 'ostatus', 'webmention', 'ui') KEY_BITS = 1024 if DEBUG else 2048 PAGE_SIZE = 20 +# auto delete old objects of these types via the Object.expire property +# https://cloud.google.com/datastore/docs/ttl +OBJECT_EXPIRE_TYPES = ( + 'post', + 'update', + 'delete', + 'accept', + 'reject', + 'undo', + None +) +OBJECT_EXPIRE_AGE = timedelta(days=90) + logger = logging.getLogger(__name__) @@ -331,7 +344,6 @@ class Object(StringIdModel): def _object_ids(self): # id(s) of inner objects if self.as1: return common.redirect_unwrap(as1.get_ids(self.as1, 'object')) - object_ids = ndb.ComputedProperty(_object_ids, repeated=True) deleted = ndb.BooleanProperty() @@ -343,6 +355,16 @@ class Object(StringIdModel): created = ndb.DateTimeProperty(auto_now_add=True) updated = ndb.DateTimeProperty(auto_now=True) + # For certain types, automatically delete this Object after 90d using a + # TTL policy: + # https://cloud.google.com/datastore/docs/ttl#ttl_properties_and_indexes + # They recommend not indexing TTL properties: + # https://cloud.google.com/datastore/docs/ttl#ttl_properties_and_indexes + def _expire(self): + if self.type in OBJECT_EXPIRE_TYPES: + return (self.updated or util.now()) + OBJECT_EXPIRE_AGE + expire = ndb.ComputedProperty(_expire, indexed=False) + def _pre_put_hook(self): assert '^^' not in self.key.id() diff --git a/tests/test_models.py b/tests/test_models.py index 40afa37..7b3caa2 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -4,11 +4,11 @@ from unittest import mock from flask import g, get_flashed_messages from granary import as2 -from oauth_dropins.webutil.testutil import requests_response +from oauth_dropins.webutil.testutil import NOW, requests_response from app import app import common -from models import Follower, Object, User +from models import Follower, Object, OBJECT_EXPIRE_AGE, User import protocol from protocol import Protocol from . import testutil @@ -329,6 +329,10 @@ class ObjectTest(testutil.TestCase): def test_computed_properties_without_as1(self): Object(id='a').put() + def test_expire(self): + obj = Object(id='a', our_as1={'objectType': 'activity', 'verb': 'update'}) + self.assertEqual(NOW + OBJECT_EXPIRE_AGE, obj.expire) + def test_put_adds_removes_activity_label(self): obj = Object(id='x#y', our_as1={}) obj.put() diff --git a/tests/testutil.py b/tests/testutil.py index 704672c..ac35a87 100644 --- a/tests/testutil.py +++ b/tests/testutil.py @@ -182,8 +182,8 @@ class TestCase(unittest.TestCase, testutil.Asserts): got.mf2.pop('url', None) self.assert_entities_equal(Object(id=id, **props), got, - ignore=['as1', 'created', 'object_ids', - 'type', 'updated']) + ignore=['as1', 'created', 'expire', + 'object_ids', 'type', 'updated']) def assert_equals(self, expected, actual, msg=None, ignore=(), **kwargs): return super().assert_equals(