Add media type and rendered content to entities with raw_content

Entities with `raw_content` now also contain a `_media_type` and
`rendered_content`.

The default `_media_type` is `text/markdown` except for ActivityPub
originating posts it defaults to `text/html`. If the ActivityPub
payload contains a `source`, that mediaType will be used instead.

CHANGELOG.md

@@ -30,6 +30,10 @@
 
 * All ActivityPub payloads are added a `pyfed: https://docs.jasonrobinson.me/ns/python-federation` context to identify payloads sent by this library.
 
+* Entities with `raw_content` now also contain a `_media_type` and `rendered_content`.
+
+  The default `_media_type` is `text/markdown` except for ActivityPub originating posts it defaults to `text/html`. If the ActivityPub payload contains a `source`, that mediaType will be used instead.
+
 ### Changed
 
 * **Backwards incompatible.** Lowest compatible Python version is now 3.6.

federation/entities/activitypub/entities.py

@@ -3,7 +3,6 @@ import uuid
 from typing import Dict
 
 import attr
-from commonmark import commonmark
 
 from federation.entities.activitypub.constants import (
     CONTEXTS_DEFAULT, CONTEXT_MANUALLY_APPROVES_FOLLOWERS, CONTEXT_SENSITIVE, CONTEXT_HASHTAG,
@@ -51,7 +50,7 @@ class ActivitypubNoteMixin(AttachImagesMixin, CleanContentMixin, ActivitypubEnti
                 "id": self.id,
                 "type": self._type,
                 "attributedTo": self.actor_id,
-                "content": commonmark(self.raw_content).strip(),
+                "content": self.rendered_content,
                 "published": self.created_at.isoformat(),
                 "inReplyTo": None,
                 "sensitive": True if "nsfw" in self.tags else False,
@@ -60,7 +59,7 @@ class ActivitypubNoteMixin(AttachImagesMixin, CleanContentMixin, ActivitypubEnti
                 "url": self.url,
                 'source': {
                     'content': self.raw_content,
-                    'mediaType': 'text/markdown',
+                    'mediaType': self._media_type,
                 },
             },
             "published": self.created_at.isoformat(),

federation/entities/activitypub/mappers.py

@@ -218,7 +218,9 @@ def message_to_objects(
     return element_to_objects(message)
 
 
-def transform_attribute(key: str, value: Union[str, Dict, int], transformed: Dict, cls, is_object: bool) -> None:
+def transform_attribute(
+        key: str, value: Union[str, Dict, int], transformed: Dict, cls, is_object: bool, payload: Dict,
+) -> None:
     if value is None:
         value = ""
     if key == "id":
@@ -236,8 +238,15 @@ def transform_attribute(key: str, value: Union[str, Dict, int], transformed: Dic
         transformed["actor_id"] = value
     elif key == "attributedTo" and is_object:
         transformed["actor_id"] = value
-    elif key == "content":
+    elif key in ("content", "source"):
+        if payload.get('source') and isinstance(payload.get("source"), dict):
+            transformed["raw_content"] = payload.get('source').get('content')
+            transformed["_media_type"] = payload.get('source').get('mediaType')
+            transformed["_rendered_content"] = payload.get('content')
+        else:
             transformed["raw_content"] = value
+            # Assume HTML by convention
+            transformed["_media_type"] = "text/html"
     elif key == "inboxes" and isinstance(value, dict):
         if "inboxes" not in transformed:
             transformed["inboxes"] = {"private": None, "public": None}
@@ -300,5 +309,5 @@ def transform_attributes(payload: Dict, cls, transformed: Dict = None, is_object
     if not transformed:
         transformed = {}
     for key, value in payload.items():
-        transform_attribute(key, value, transformed, cls, is_object)
+        transform_attribute(key, value, transformed, cls, is_object, payload)
     return transformed

federation/entities/activitypub/mixins.py

@@ -10,6 +10,8 @@ class AttachImagesMixin(RawContentMixin):
         """
         Attach any embedded images from raw_content.
         """
+        if self._media_type != "text/markdown":
+            return
         regex = r"!\[([\w ]*)\]\((https?://[\w\d\-\./]+\.[\w]*((?<=jpg)|(?<=gif)|(?<=png)|(?<=jpeg)))\)"
         matches = re.finditer(regex, self.raw_content, re.MULTILINE | re.IGNORECASE)
         for match in matches:

federation/entities/mixins.py

@@ -3,6 +3,8 @@ import importlib
 import warnings
 from typing import List, Set
 
+from commonmark import commonmark
+
 from federation.entities.activitypub.enums import ActivityType
 
 
@@ -175,12 +177,23 @@ class CreatedAtMixin(BaseEntity):
 
 
 class RawContentMixin(BaseEntity):
-    raw_content = ""
+    _media_type: str = "text/markdown"
+    _rendered_content: str = ""
+    raw_content: str = ""
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self._required += ["raw_content"]
 
+    @property
+    def rendered_content(self) -> str:
+        """Returns the rendered version of raw_content, or just raw_content."""
+        if self._rendered_content:
+            return self._rendered_content
+        elif self._media_type == "text/markdown" and self.raw_content:
+            return commonmark(self.raw_content).strip()
+        return self.raw_content
+
     @property
     def tags(self):
         """Returns a `set` of unique tags contained in `raw_content`."""

federation/tests/entities/activitypub/test_mappers.py

@@ -10,7 +10,7 @@ from federation.entities.base import Accept, Follow, Profile, Post, Comment, Ima
 from federation.tests.fixtures.payloads import (
     ACTIVITYPUB_FOLLOW, ACTIVITYPUB_PROFILE, ACTIVITYPUB_PROFILE_INVALID, ACTIVITYPUB_UNDO_FOLLOW, ACTIVITYPUB_POST,
     ACTIVITYPUB_COMMENT, ACTIVITYPUB_RETRACTION, ACTIVITYPUB_SHARE, ACTIVITYPUB_RETRACTION_SHARE,
-    ACTIVITYPUB_POST_IMAGES)
+    ACTIVITYPUB_POST_IMAGES, ACTIVITYPUB_POST_WITH_SOURCE)
 from federation.types import UserType, ReceiverVariant
 
 
@@ -67,9 +67,26 @@ class TestActivitypubEntityMappersReceive:
         assert isinstance(post, Post)
         assert post.raw_content == '<p><span class="h-card"><a href="https://dev.jasonrobinson.me/u/jaywink/" ' \
                                    'class="u-url mention">@<span>jaywink</span></a></span> boom</p>'
+        assert post.rendered_content == post.raw_content
         assert post.id == "https://diaspodon.fr/users/jaywink/statuses/102356911717767237"
         assert post.actor_id == "https://diaspodon.fr/users/jaywink"
         assert post.public is True
+        assert post._media_type == "text/html"
+        assert getattr(post, "target_id", None) is None
+
+    def test_message_to_objects_simple_post__with_source(self):
+        entities = message_to_objects(ACTIVITYPUB_POST_WITH_SOURCE, "https://diaspodon.fr/users/jaywink")
+        assert len(entities) == 1
+        post = entities[0]
+        assert isinstance(post, ActivitypubPost)
+        assert isinstance(post, Post)
+        assert post.rendered_content == '<p><span class="h-card"><a href="https://dev.jasonrobinson.me/u/jaywink/" ' \
+                                        'class="u-url mention">@<span>jaywink</span></a></span> boom</p>'
+        assert post.raw_content == "@jaywink boom"
+        assert post.id == "https://diaspodon.fr/users/jaywink/statuses/102356911717767237"
+        assert post.actor_id == "https://diaspodon.fr/users/jaywink"
+        assert post.public is True
+        assert post._media_type == "text/markdown"
         assert getattr(post, "target_id", None) is None
 
     def test_message_to_objects_post_with_photos(self):

federation/tests/fixtures/payloads/activitypub.py

@@ -292,6 +292,60 @@ ACTIVITYPUB_POST = {
   'signatureValue': 'SjDACS7Z/Cb1SEC3AtxEokID5SHAYl7kpys/hhmaRbpXuFKCxfj2P9BmH8QhLnuam3sENZlrnBOcB5NlcBhIfwo/Xh242RZBmPQf+edTVYVCe1j19dihcftNCHtnqAcKwp/51dNM/OlKu2730FrwvOUXVIPtB7iVqkseO9TRzDYIDj+zBTksnR/NAYtq6SUpmefXfON0uW3N3Uq6PGfExJaS+aeqRf8cPGkZFSIUQZwOLXbIpb7BFjJ1+y1OMOAJueqvikUprAit3v6BiNWurAvSQpC7WWMFUKyA79/xtkO9kIPA/Q4C9ryqdzxZJ0jDhXiaIIQj2JZfIADdjLZHJA=='}
 }
 
+ACTIVITYPUB_POST_WITH_SOURCE = {
+  '@context': ['https://www.w3.org/ns/activitystreams',
+  {'ostatus': 'http://ostatus.org#',
+   'atomUri': 'ostatus:atomUri',
+   'inReplyToAtomUri': 'ostatus:inReplyToAtomUri',
+   'conversation': 'ostatus:conversation',
+   'sensitive': 'as:sensitive',
+   'Hashtag': 'as:Hashtag',
+   'toot': 'http://joinmastodon.org/ns#',
+   'Emoji': 'toot:Emoji',
+   'focalPoint': {'@container': '@list', '@id': 'toot:focalPoint'},
+   'blurhash': 'toot:blurhash'}],
+ 'id': 'https://diaspodon.fr/users/jaywink/statuses/102356911717767237/activity',
+ 'type': 'Create',
+ 'actor': 'https://diaspodon.fr/users/jaywink',
+ 'published': '2019-06-29T21:08:45Z',
+ 'to': 'https://www.w3.org/ns/activitystreams#Public',
+ 'cc': ['https://diaspodon.fr/users/jaywink/followers',
+  'https://dev.jasonrobinson.me/p/d4574854-a5d7-42be-bfac-f70c16fcaa97/'],
+ 'object': {'id': 'https://diaspodon.fr/users/jaywink/statuses/102356911717767237',
+  'type': 'Note',
+  'summary': None,
+  'inReplyTo': None,
+  'published': '2019-06-29T21:08:45Z',
+  'url': 'https://diaspodon.fr/@jaywink/102356911717767237',
+  'attributedTo': 'https://diaspodon.fr/users/jaywink',
+  'to': 'https://www.w3.org/ns/activitystreams#Public',
+  'cc': ['https://diaspodon.fr/users/jaywink/followers',
+   'https://dev.jasonrobinson.me/p/d4574854-a5d7-42be-bfac-f70c16fcaa97/'],
+  'sensitive': False,
+  'atomUri': 'https://diaspodon.fr/users/jaywink/statuses/102356911717767237',
+  'inReplyToAtomUri': None,
+  'conversation': 'tag:diaspodon.fr,2019-06-28:objectId=2347687:objectType=Conversation',
+  'content': '<p><span class="h-card"><a href="https://dev.jasonrobinson.me/u/jaywink/" class="u-url mention">@<span>jaywink</span></a></span> boom</p>',
+  'source': {
+      'content': "@jaywink boom",
+      'mediaType': "text/markdown",
+  },
+  'contentMap': {'en': '<p><span class="h-card"><a href="https://dev.jasonrobinson.me/u/jaywink/" class="u-url mention">@<span>jaywink</span></a></span> boom</p>'},
+  'attachment': [],
+  'tag': [{'type': 'Mention',
+    'href': 'https://dev.jasonrobinson.me/p/d4574854-a5d7-42be-bfac-f70c16fcaa97/',
+    'name': '@jaywink@dev.jasonrobinson.me'}],
+  'replies': {'id': 'https://diaspodon.fr/users/jaywink/statuses/102356911717767237/replies',
+   'type': 'Collection',
+   'first': {'type': 'CollectionPage',
+    'partOf': 'https://diaspodon.fr/users/jaywink/statuses/102356911717767237/replies',
+    'items': []}}},
+ 'signature': {'type': 'RsaSignature2017',
+  'creator': 'https://diaspodon.fr/users/jaywink#main-key',
+  'created': '2019-06-29T21:08:45Z',
+  'signatureValue': 'SjDACS7Z/Cb1SEC3AtxEokID5SHAYl7kpys/hhmaRbpXuFKCxfj2P9BmH8QhLnuam3sENZlrnBOcB5NlcBhIfwo/Xh242RZBmPQf+edTVYVCe1j19dihcftNCHtnqAcKwp/51dNM/OlKu2730FrwvOUXVIPtB7iVqkseO9TRzDYIDj+zBTksnR/NAYtq6SUpmefXfON0uW3N3Uq6PGfExJaS+aeqRf8cPGkZFSIUQZwOLXbIpb7BFjJ1+y1OMOAJueqvikUprAit3v6BiNWurAvSQpC7WWMFUKyA79/xtkO9kIPA/Q4C9ryqdzxZJ0jDhXiaIIQj2JZfIADdjLZHJA=='}
+}
+
 ACTIVITYPUB_POST_OBJECT = {
     'id': 'https://diaspodon.fr/users/jaywink/statuses/102356911717767237',
     'type': 'Note',