initial design of new Object model, implement it in render.py

models.py

@@ -227,78 +227,50 @@ class User(StringIdModel):
         return self
 
 
-class Activity(StringIdModel):
+class Object(StringIdModel):
-    """A reply, like, repost, or other interaction that we've relayed.
+    """An activity or other object, eg actor.
 
-    Key name is 'SOURCE_URL TARGET_URL', e.g. 'http://a/reply http://orig/post'.
+    Key name is the id. We synthesize ids if necessary.
     """
-    STATUSES = ('new', 'complete', 'error', 'ignored')
+    STATUSES = ('new', 'in progress', 'complete', 'ignored')
-    PROTOCOLS = ('activitypub', 'ostatus')
+    PROTOCOLS = ('activitypub', 'bluesky', 'webmention')
-    DIRECTIONS = ('out', 'in')
+    LABELS = ('feed', 'notification')
 
     # domains of the Bridgy Fed users this activity is to or from
-    domain = ndb.StringProperty(repeated=True)
+    domains = ndb.StringProperty(repeated=True)
     status = ndb.StringProperty(choices=STATUSES, default='new')
-    protocol = ndb.StringProperty(choices=PROTOCOLS)
+    source_protocol = ndb.StringProperty(choices=PROTOCOLS)
-    direction = ndb.StringProperty(choices=DIRECTIONS)
+    labels = ndb.StringProperty(repeated=True, choices=LABELS)
 
-    # usually only one of these at most will be populated.
+    # these are all JSON. They're TextProperty, and not JsonProperty, so that
-    source_mf2 = ndb.TextProperty()  # JSON
+    # their plain text is visible in the App Engine admin console. (JsonProperty
-    source_as2 = ndb.TextProperty()  # JSON
+    # uses a blob.)
-    source_atom = ndb.TextProperty()
+    as1 = ndb.TextProperty(required=True)  # converted from source data
-    target_as2 = ndb.TextProperty()  # JSON
+    as2 = ndb.TextProperty()  # only one of the rest will be populated...
+    bsky = ndb.TextProperty()  # Bluesky / AT Protocol
+    mf2 = ndb.TextProperty()  # HTML microformats2
+
+    type = ndb.StringProperty()  # AS1 objectType, or verb if it's an activity
+    deleted = ndb.BooleanProperty(default=False)
+    object_ids = ndb.StringProperty(repeated=True)  # id(s) of inner objects
+
+    # ActivityPub inbox delivery
+    ap_delivered = ndb.StringProperty(repeated=True)
+    ap_undelivered = ndb.StringProperty(repeated=True)
+    ap_failed = ndb.StringProperty(repeated=True)
 
     created = ndb.DateTimeProperty(auto_now_add=True)
     updated = ndb.DateTimeProperty(auto_now=True)
 
-    @classmethod
-    def _get_kind(cls):
-        return 'Response'
-
-    def __init__(self, source=None, target=None, **kwargs):
-        if source and target:
-            assert 'id' not in kwargs
-            kwargs['id'] = self._id(source, target)
-            logger.info(f"Activity id (source target): {kwargs['id']}")
-        super(Activity, self).__init__(**kwargs)
-
-    @classmethod
-    def get_or_create(cls, source=None, target=None, **kwargs):
-        logger.info(f'Activity source target: {source} {target}')
-        return cls.get_or_insert(cls._id(source, target), **kwargs)
-
-    def source(self):
-        return self.key.id().split()[0]
-
-    def target(self):
-        return self.key.id().split()[1]
-
     def proxy_url(self):
         """Returns the Bridgy Fed proxy URL to render this post as HTML."""
-        if self.source_mf2 or self.source_as2 or self.source_atom:
+        return common.host_url('render?' +
-            source, target = self.key.id().split(' ')
+                               urllib.parse.urlencode({'id': self.key.id()}))
-            return common.host_url('render?' + urllib.parse.urlencode({
-                'source': source,
-                'target': target,
-            }))
-
-    def to_as1(self):
-        """Returns this activity as an ActivityStreams 1 dict, if available."""
-        if self.source_mf2:
-            mf2 = json_loads(self.source_mf2)
-            items = mf2.get('items')
-            if items:
-                mf2 = items[0]
-            return microformats2.json_to_object(mf2)
-        if self.source_as2:
-            return as2.to_as1(json_loads(self.source_as2))
-        if self.source_atom:
-            return atom.atom_to_activity(self.source_atom)
 
     def actor_link(self, as1=None):
         """Returns a pretty actor link with their name and profile picture."""
-        if self.direction == 'out' and self.domain:
+        if self.direction == 'out' and self.domains:
-            return User.get_by_id(self.domain[0]).user_page_link()
+            return User.get_by_id(self.domains[0]).user_page_link()
 
         if not as1:
            as1 = self.to_as1()
@@ -319,20 +291,6 @@ class Activity(StringIdModel):
           {util.ellipsize(name, chars=40)}
         </a>"""
 
-    @classmethod
-    def _id(cls, source, target):
-        assert source
-        assert target
-        return f'{cls._encode(source)} {cls._encode(target)}'
-
-    @classmethod
-    def _encode(cls, val):
-        return val.replace('#', '__')
-
-    @classmethod
-    def _decode(cls, val):
-        return val.replace('__', '#')
-
 
 class Follower(StringIdModel):
     """A follower of a Bridgy Fed user.

render.py

@@ -1,8 +1,10 @@
 # coding=utf-8
-"""Renders mf2 proxy pages based on stored Activity entities."""
+"""Renders mf2 proxy pages based on stored Object entities."""
 import datetime
+import logging
+from urllib.parse import urlencode
 
-from flask import request
+from flask import redirect, request
 from granary import as2, atom, microformats2
 from oauth_dropins.webutil import flask_util
 from oauth_dropins.webutil.flask_util import error
@@ -11,34 +13,41 @@ from oauth_dropins.webutil.util import json_loads
 
 from app import app, cache
 import common
-from models import Activity
+from models import Object
+
+logger = logging.getLogger(__name__)
 
 
 @app.get('/render')
 @flask_util.cached(cache, common.CACHE_TIME)
 def render():
-    """Fetches a stored Activity and renders it as HTML."""
+    """Fetches a stored Object and renders it as HTML."""
-    source = flask_util.get_required_param('source')
+    id = flask_util.get_required_param('id')
-    target = flask_util.get_required_param('target')
+    obj = Object.get_by_id(id)
+    if not obj:
+        error(f'No stored object for {id}', status=404)
+    elif not obj.as1:
+        error(f'Stored object for {id} has no AS1', status=404)
 
-    id = f'{source} {target}'
+    as1 = json_loads(obj.as1)
-    activity = Activity.get_by_id(id)
+    if (as1.get('objectType') == 'activity' and
-    if not activity:
+        as1.get('verb') in ('post', 'update', 'delete')):
-        error(f'No stored activity for {id}', status=404)
+        # redirect to inner object
-
+        obj_id = as1.get('object')
-    if activity.source_mf2:
+        if isinstance(obj_id, dict):
-        as1 = microformats2.json_to_object(json_loads(activity.source_mf2))
+            obj_id = obj_id.get('id')
-    elif activity.source_as2:
+        if not obj_id:
-        as1 = as2.to_as1(json_loads(activity.source_as2))
+            error(f'Stored {type} activity has no object id!', status=404)
-    elif activity.source_atom:
+        logger.info(f'{type} activity, redirecting to object id {obj_id}')
-        as1 = atom.atom_to_activity(activity.source_atom)
+        return redirect('/render?' + urlencode({'id': obj_id}), code=301)
-    else:
-        error(f'Stored activity for {id} has no data', status=404)
 
     # add HTML meta redirect to source page. should trigger for end users in
     # browsers but not for webmention receivers (hopefully).
     html = microformats2.activities_to_html([as1])
     utf8 = '<meta charset="utf-8">'
-    url = util.get_url(as1) or as1.get('id') or source
+    url = util.get_url(as1)
-    refresh = f'<meta http-equiv="refresh" content="0;url={url}">'
+    if url:
-    return html.replace(utf8, utf8 + '\n' + refresh)
+        refresh = f'<meta http-equiv="refresh" content="0;url={url}">'
+        html = html.replace(utf8, utf8 + '\n' + refresh)
+
+    return html

tests/test_models.py

@@ -7,7 +7,7 @@ from oauth_dropins.webutil.testutil import requests_response
 from oauth_dropins.webutil.util import json_dumps, json_loads
 
 from app import app
-from models import Activity, Follower, User
+from models import Follower, Object, User
 from . import testutil
 
 from .test_activitypub import ACTOR
@@ -167,30 +167,12 @@ http://this/404s
         self.assertEqual(root_user.key, User.get_or_create('www.y.z').key)
 
 
-class ActivityTest(testutil.TestCase):
+class ObjectTest(testutil.TestCase):
-
-    def test_constructor(self):
-        activity = Activity('abc', 'xyz')
-        self.assertEqual('abc xyz', activity.key.id())
-
-        activity = Activity('abc#1', 'xyz#Z')
-        self.assertEqual('abc__1 xyz__Z', activity.key.id())
-
-    def test_get_or_create(self):
-        activity = Activity.get_or_create('abc', 'xyz')
-        self.assertEqual('abc xyz', activity.key.id())
-
-        activity = Activity.get_or_create('abc#1', 'xyz#Z')
-        self.assertEqual('abc__1 xyz__Z', activity.key.id())
 
     def test_proxy_url(self):
         with app.test_request_context('/'):
-            activity = Activity.get_or_create('abc', 'xyz')
+            obj = Object.get_or_insert('abc')
-            self.assertIsNone(activity.proxy_url())
+            self.assertEqual('http://localhost/render?id=abc', obj.proxy_url())
-
-            activity.source_as2 = 'as2'
-            self.assertEqual('http://localhost/render?source=abc&target=xyz',
-                             activity.proxy_url())
 
 
 class FollowerTest(testutil.TestCase):

tests/test_render.py

@@ -1,93 +1,81 @@
 # coding=utf-8
 """Unit tests for render.py."""
+import copy
+
+from granary.tests.test_as1 import COMMENT, DELETE_OF_ID, UPDATE
 from oauth_dropins.webutil.util import json_dumps
 
 import common
-from models import Activity
+from models import Object
 import render
 from . import testutil
 
-
+EXPECTED_HTML = """\
-class RenderTest(testutil.TestCase):
-
-    def setUp(self):
-        super().setUp()
-        self.as2 = {
-            '@context': 'https://www.w3.org/ns/activitystreams',
-            'type': 'Note',
-            'id': 'http://this/reply',
-            'url': 'http://this/reply',
-            'content': 'A ☕ reply',
-            'inReplyTo': 'http://orig/post',
-        }
-        self.mf2 = {
-            'type': ['h-entry'],
-            'properties': {
-                'uid': ['http://this/reply'],
-                'url': ['http://this/reply'],
-                'content': [{'value': 'A ☕ reply'}],
-                'in-reply-to': ['http://orig/post'],
-            },
-        }
-        self.atom = """\
-<?xml version="1.0" encoding="UTF-8"?>
-<entry xmlns="http://www.w3.org/2005/Atom"
-       xmlns:thr="http://purl.org/syndication/thread/1.0">
-
-<uri>http://this/reply</uri>
-<thr:in-reply-to href="http://orig/post" />
-<content>A ☕ reply</content>
-</entry>
-"""
-        self.html = """\
 <!DOCTYPE html>
 <html>
 <head><meta charset="utf-8">
-<meta http-equiv="refresh" content="0;url=http://this/reply"></head>
+<meta http-equiv="refresh" content="0;url=https://fake.com/123456"></head>
 <body class="">
 <article class="h-entry">
-  <span class="p-uid">http://this/reply</span>
+  <span class="p-uid">tag:fake.com:123456</span>
-  <a class="u-url" href="http://this/reply">http://this/reply</a>
+  <time class="dt-published" datetime="2012-12-05T00:58:26+00:00">2012-12-05T00:58:26+00:00</time>
+  <a class="u-url" href="https://fake.com/123456">https://fake.com/123456</a>
   <div class="e-content p-name">
   A ☕ reply
   </div>
-  <a class="u-in-reply-to" href="http://orig/post"></a>
+  <a class="u-in-reply-to" href="https://fake.com/123"></a>
 </article>
 </body>
 </html>
 """
 
+class RenderTest(testutil.TestCase):
+
+    def setUp(self):
+        super().setUp()
+
     def test_render_errors(self):
-        for source, target in ('', ''), ('abc', ''), ('', 'xyz'):
+        resp = self.client.get(f'/render?id=')
-            resp = self.client.get(f'/render?source={source}&target={target}')
+        self.assertEqual(400, resp.status_code)
-            self.assertEqual(400, resp.status_code, resp.get_data(as_text=True))
 
-        # no Activity
+        resp = self.client.get(f'/render')
-        resp = self.client.get('/render?source=abc&target=xyz')
+        self.assertEqual(400, resp.status_code)
+
+        # no Object
+        resp = self.client.get('/render?id=abc')
         self.assertEqual(404, resp.status_code)
 
-        # no source data
+    def test_render(self):
-        Activity(id='abc xyz').put()
+        Object(id='abc', as1=json_dumps(COMMENT)).put()
-        resp = self.client.get('/render?source=abc&target=xyz')
+        resp = self.client.get('/render?id=abc')
-        self.assertEqual(404, resp.status_code)
-
-    def test_render_as2(self):
-        Activity(id='abc xyz', source_as2=json_dumps(self.as2)).put()
-        resp = self.client.get('/render?source=abc&target=xyz')
         self.assertEqual(200, resp.status_code)
-        self.assert_multiline_equals(self.html, resp.get_data(as_text=True),
+        self.assert_multiline_equals(EXPECTED_HTML, resp.get_data(as_text=True), ignore_blanks=True)
+
+    def test_render_no_url(self):
+        comment = copy.deepcopy(COMMENT)
+        del comment['url']
+        Object(id='abc', as1=json_dumps(comment)).put()
+
+        resp = self.client.get('/render?id=abc')
+        self.assertEqual(200, resp.status_code)
+        expected = EXPECTED_HTML.replace(
+            '\n<meta http-equiv="refresh" content="0;url=https://fake.com/123456">', ''
+            ).replace('<a class="u-url" href="https://fake.com/123456">https://fake.com/123456</a>', '')
+        self.assert_multiline_equals(expected, resp.get_data(as_text=True),
                                      ignore_blanks=True)
 
-    def test_render_mf2(self):
+    def test_render_update_redirect(self):
-        Activity(id='abc xyz', source_mf2=json_dumps(self.mf2)).put()
+        # UPDATE's object field is a full object
-        resp = self.client.get('/render?source=abc&target=xyz')
+        Object(id='abc', as1=json_dumps(UPDATE)).put()
-        self.assertEqual(200, resp.status_code)
+        resp = self.client.get('/render?id=abc')
-        self.assert_multiline_equals(self.html, resp.get_data(as_text=True),
+        self.assertEqual(301, resp.status_code)
-                                     ignore_blanks=True)
+        self.assertEqual(f'/render?id=tag%3Afake.com%3A123456',
+                         resp.headers['Location'])
 
-    def test_render_atom(self):
+    def test_render_delete_redirect(self):
-        Activity(id='abc xyz', source_atom=self.atom).put()
+        # DELETE_OF_ID's object field is a bare string id
-        resp = self.client.get('/render?source=abc&target=xyz')
+        Object(id='abc', as1=json_dumps(DELETE_OF_ID)).put()
-        self.assertEqual(200, resp.status_code)
+        resp = self.client.get('/render?id=abc')
-        self.assert_multiline_equals(self.html, resp.get_data(as_text=True),
+        self.assertEqual(301, resp.status_code)
-                                     ignore_blanks=True)
+        self.assertEqual(f'/render?id=tag%3Afake.com%3A123456',
+                         resp.headers['Location'])