Add media type and rendered content to entities with raw_content

Entities with `raw_content` now also contain a `_media_type` and `rendered_content`. The default `_media_type` is `text/markdown` except for ActivityPub originating posts it defaults to `text/html`. If the ActivityPub payload contains a `source`, that mediaType will be used instead.
2019-08-18 22:37:18 +03:00 · 2019-08-18 22:37:18 +03:00 · 61a0fc442b
commit 61a0fc442b
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@ -30,6 +30,10 @@

 * All ActivityPub payloads are added a `pyfed: https://docs.jasonrobinson.me/ns/python-federation` context to identify payloads sent by this library.

+* Entities with `raw_content` now also contain a `_media_type` and `rendered_content`.
+
+  The default `_media_type` is `text/markdown` except for ActivityPub originating posts it defaults to `text/html`. If the ActivityPub payload contains a `source`, that mediaType will be used instead.
+
 ### Changed

 * **Backwards incompatible.** Lowest compatible Python version is now 3.6.
--- a/federation/entities/activitypub/entities.py
+++ b/federation/entities/activitypub/entities.py
@ -3,7 +3,6 @@ import uuid
 from typing import Dict

 import attr
-from commonmark import commonmark

 from federation.entities.activitypub.constants import (
    CONTEXTS_DEFAULT, CONTEXT_MANUALLY_APPROVES_FOLLOWERS, CONTEXT_SENSITIVE, CONTEXT_HASHTAG,
@ -51,7 +50,7 @@ class ActivitypubNoteMixin(AttachImagesMixin, CleanContentMixin, ActivitypubEnti
                "id": self.id,
                "type": self._type,
                "attributedTo": self.actor_id,
-                "content": commonmark(self.raw_content).strip(),
+                "content": self.rendered_content,
                "published": self.created_at.isoformat(),
                "inReplyTo": None,
                "sensitive": True if "nsfw" in self.tags else False,
@ -60,7 +59,7 @@ class ActivitypubNoteMixin(AttachImagesMixin, CleanContentMixin, ActivitypubEnti
                "url": self.url,
                'source': {
                    'content': self.raw_content,
-                    'mediaType': 'text/markdown',
+                    'mediaType': self._media_type,
                },
            },
            "published": self.created_at.isoformat(),
--- a/federation/entities/activitypub/mappers.py
+++ b/federation/entities/activitypub/mappers.py
@ -218,7 +218,9 @@ def message_to_objects(
    return element_to_objects(message)


-def transform_attribute(key: str, value: Union[str, Dict, int], transformed: Dict, cls, is_object: bool) -> None:
+def transform_attribute(
+        key: str, value: Union[str, Dict, int], transformed: Dict, cls, is_object: bool, payload: Dict,
+) -> None:
    if value is None:
        value = ""
    if key == "id":
@ -236,8 +238,15 @@ def transform_attribute(key: str, value: Union[str, Dict, int], transformed: Dic
        transformed["actor_id"] = value
    elif key == "attributedTo" and is_object:
        transformed["actor_id"] = value
-    elif key == "content":
-        transformed["raw_content"] = value
+    elif key in ("content", "source"):
+        if payload.get('source') and isinstance(payload.get("source"), dict):
+            transformed["raw_content"] = payload.get('source').get('content')
+            transformed["_media_type"] = payload.get('source').get('mediaType')
+            transformed["_rendered_content"] = payload.get('content')
+        else:
+            transformed["raw_content"] = value
+            # Assume HTML by convention
+            transformed["_media_type"] = "text/html"
    elif key == "inboxes" and isinstance(value, dict):
        if "inboxes" not in transformed:
            transformed["inboxes"] = {"private": None, "public": None}
@ -300,5 +309,5 @@ def transform_attributes(payload: Dict, cls, transformed: Dict = None, is_object
    if not transformed:
        transformed = {}
    for key, value in payload.items():
-        transform_attribute(key, value, transformed, cls, is_object)
+        transform_attribute(key, value, transformed, cls, is_object, payload)
    return transformed
--- a/federation/entities/activitypub/mixins.py
+++ b/federation/entities/activitypub/mixins.py
@ -10,6 +10,8 @@ class AttachImagesMixin(RawContentMixin):
        """
        Attach any embedded images from raw_content.
        """
+        if self._media_type != "text/markdown":
+            return
        regex = r"!\[([\w ]*)\]\((https?://[\w\d\-\./]+\.[\w]*((?<=jpg)|(?<=gif)|(?<=png)|(?<=jpeg)))\)"
        matches = re.finditer(regex, self.raw_content, re.MULTILINE | re.IGNORECASE)
        for match in matches:
--- a/federation/entities/mixins.py
+++ b/federation/entities/mixins.py
@ -3,6 +3,8 @@ import importlib
 import warnings
 from typing import List, Set

+from commonmark import commonmark
+
 from federation.entities.activitypub.enums import ActivityType


@ -175,12 +177,23 @@ class CreatedAtMixin(BaseEntity):


 class RawContentMixin(BaseEntity):
-    raw_content = ""
+    _media_type: str = "text/markdown"
+    _rendered_content: str = ""
+    raw_content: str = ""

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self._required += ["raw_content"]

+    @property
+    def rendered_content(self) -> str:
+        """Returns the rendered version of raw_content, or just raw_content."""
+        if self._rendered_content:
+            return self._rendered_content
+        elif self._media_type == "text/markdown" and self.raw_content:
+            return commonmark(self.raw_content).strip()
+        return self.raw_content
+
    @property
    def tags(self):
        """Returns a `set` of unique tags contained in `raw_content`."""
--- a/federation/tests/entities/activitypub/test_mappers.py
+++ b/federation/tests/entities/activitypub/test_mappers.py
@ -10,7 +10,7 @@ from federation.entities.base import Accept, Follow, Profile, Post, Comment, Ima
 from federation.tests.fixtures.payloads import (
    ACTIVITYPUB_FOLLOW, ACTIVITYPUB_PROFILE, ACTIVITYPUB_PROFILE_INVALID, ACTIVITYPUB_UNDO_FOLLOW, ACTIVITYPUB_POST,
    ACTIVITYPUB_COMMENT, ACTIVITYPUB_RETRACTION, ACTIVITYPUB_SHARE, ACTIVITYPUB_RETRACTION_SHARE,
-    ACTIVITYPUB_POST_IMAGES)
+    ACTIVITYPUB_POST_IMAGES, ACTIVITYPUB_POST_WITH_SOURCE)
 from federation.types import UserType, ReceiverVariant


@ -67,9 +67,26 @@ class TestActivitypubEntityMappersReceive:
        assert isinstance(post, Post)
        assert post.raw_content == '<p><span class="h-card"><a href="https://dev.jasonrobinson.me/u/jaywink/" ' \
                                   'class="u-url mention">@<span>jaywink</span></a></span> boom</p>'
+        assert post.rendered_content == post.raw_content
        assert post.id == "https://diaspodon.fr/users/jaywink/statuses/102356911717767237"
        assert post.actor_id == "https://diaspodon.fr/users/jaywink"
        assert post.public is True
+        assert post._media_type == "text/html"
+        assert getattr(post, "target_id", None) is None
+
+    def test_message_to_objects_simple_post__with_source(self):
+        entities = message_to_objects(ACTIVITYPUB_POST_WITH_SOURCE, "https://diaspodon.fr/users/jaywink")
+        assert len(entities) == 1
+        post = entities[0]
+        assert isinstance(post, ActivitypubPost)
+        assert isinstance(post, Post)
+        assert post.rendered_content == '<p><span class="h-card"><a href="https://dev.jasonrobinson.me/u/jaywink/" ' \
+                                        'class="u-url mention">@<span>jaywink</span></a></span> boom</p>'
+        assert post.raw_content == "@jaywink boom"
+        assert post.id == "https://diaspodon.fr/users/jaywink/statuses/102356911717767237"
+        assert post.actor_id == "https://diaspodon.fr/users/jaywink"
+        assert post.public is True
+        assert post._media_type == "text/markdown"
        assert getattr(post, "target_id", None) is None

    def test_message_to_objects_post_with_photos(self):
--- a/federation/tests/fixtures/payloads/activitypub.py
+++ b/federation/tests/fixtures/payloads/activitypub.py
@ -292,6 +292,60 @@ ACTIVITYPUB_POST = {
  'signatureValue': 'SjDACS7Z/Cb1SEC3AtxEokID5SHAYl7kpys/hhmaRbpXuFKCxfj2P9BmH8QhLnuam3sENZlrnBOcB5NlcBhIfwo/Xh242RZBmPQf+edTVYVCe1j19dihcftNCHtnqAcKwp/51dNM/OlKu2730FrwvOUXVIPtB7iVqkseO9TRzDYIDj+zBTksnR/NAYtq6SUpmefXfON0uW3N3Uq6PGfExJaS+aeqRf8cPGkZFSIUQZwOLXbIpb7BFjJ1+y1OMOAJueqvikUprAit3v6BiNWurAvSQpC7WWMFUKyA79/xtkO9kIPA/Q4C9ryqdzxZJ0jDhXiaIIQj2JZfIADdjLZHJA=='}
 }

+ACTIVITYPUB_POST_WITH_SOURCE = {
+  '@context': ['https://www.w3.org/ns/activitystreams',
+  {'ostatus': 'http://ostatus.org#',
+   'atomUri': 'ostatus:atomUri',
+   'inReplyToAtomUri': 'ostatus:inReplyToAtomUri',
+   'conversation': 'ostatus:conversation',
+   'sensitive': 'as:sensitive',
+   'Hashtag': 'as:Hashtag',
+   'toot': 'http://joinmastodon.org/ns#',
+   'Emoji': 'toot:Emoji',
+   'focalPoint': {'@container': '@list', '@id': 'toot:focalPoint'},
+   'blurhash': 'toot:blurhash'}],
+ 'id': 'https://diaspodon.fr/users/jaywink/statuses/102356911717767237/activity',
+ 'type': 'Create',
+ 'actor': 'https://diaspodon.fr/users/jaywink',
+ 'published': '2019-06-29T21:08:45Z',
+ 'to': 'https://www.w3.org/ns/activitystreams#Public',
+ 'cc': ['https://diaspodon.fr/users/jaywink/followers',
+  'https://dev.jasonrobinson.me/p/d4574854-a5d7-42be-bfac-f70c16fcaa97/'],
+ 'object': {'id': 'https://diaspodon.fr/users/jaywink/statuses/102356911717767237',
+  'type': 'Note',
+  'summary': None,
+  'inReplyTo': None,
+  'published': '2019-06-29T21:08:45Z',
+  'url': 'https://diaspodon.fr/@jaywink/102356911717767237',
+  'attributedTo': 'https://diaspodon.fr/users/jaywink',
+  'to': 'https://www.w3.org/ns/activitystreams#Public',
+  'cc': ['https://diaspodon.fr/users/jaywink/followers',
+   'https://dev.jasonrobinson.me/p/d4574854-a5d7-42be-bfac-f70c16fcaa97/'],
+  'sensitive': False,
+  'atomUri': 'https://diaspodon.fr/users/jaywink/statuses/102356911717767237',
+  'inReplyToAtomUri': None,
+  'conversation': 'tag:diaspodon.fr,2019-06-28:objectId=2347687:objectType=Conversation',
+  'content': '<p><span class="h-card"><a href="https://dev.jasonrobinson.me/u/jaywink/" class="u-url mention">@<span>jaywink</span></a></span> boom</p>',
+  'source': {
+      'content': "@jaywink boom",
+      'mediaType': "text/markdown",
+  },
+  'contentMap': {'en': '<p><span class="h-card"><a href="https://dev.jasonrobinson.me/u/jaywink/" class="u-url mention">@<span>jaywink</span></a></span> boom</p>'},
+  'attachment': [],
+  'tag': [{'type': 'Mention',
+    'href': 'https://dev.jasonrobinson.me/p/d4574854-a5d7-42be-bfac-f70c16fcaa97/',
+    'name': '@jaywink@dev.jasonrobinson.me'}],
+  'replies': {'id': 'https://diaspodon.fr/users/jaywink/statuses/102356911717767237/replies',
+   'type': 'Collection',
+   'first': {'type': 'CollectionPage',
+    'partOf': 'https://diaspodon.fr/users/jaywink/statuses/102356911717767237/replies',
+    'items': []}}},
+ 'signature': {'type': 'RsaSignature2017',
+  'creator': 'https://diaspodon.fr/users/jaywink#main-key',
+  'created': '2019-06-29T21:08:45Z',
+  'signatureValue': 'SjDACS7Z/Cb1SEC3AtxEokID5SHAYl7kpys/hhmaRbpXuFKCxfj2P9BmH8QhLnuam3sENZlrnBOcB5NlcBhIfwo/Xh242RZBmPQf+edTVYVCe1j19dihcftNCHtnqAcKwp/51dNM/OlKu2730FrwvOUXVIPtB7iVqkseO9TRzDYIDj+zBTksnR/NAYtq6SUpmefXfON0uW3N3Uq6PGfExJaS+aeqRf8cPGkZFSIUQZwOLXbIpb7BFjJ1+y1OMOAJueqvikUprAit3v6BiNWurAvSQpC7WWMFUKyA79/xtkO9kIPA/Q4C9ryqdzxZJ0jDhXiaIIQj2JZfIADdjLZHJA=='}
+}
+
 ACTIVITYPUB_POST_OBJECT = {
    'id': 'https://diaspodon.fr/users/jaywink/statuses/102356911717767237',
    'type': 'Note',