Got up to incoming posts working

.pre-commit-config.yaml

@@ -18,7 +18,7 @@ repos:
     rev: 22.10.0
     hooks:
       - id: black
-        args: ["--target-version=py37"]
+        language_version: python3.10
 
   - repo: https://github.com/pycqa/isort
     rev: 5.10.1
@@ -35,4 +35,4 @@ repos:
     rev: v0.982
     hooks:
       - id: mypy
-        additional_dependencies: [types-pyopenssl]
+        additional_dependencies: [types-pyopenssl, types-bleach]

activities/admin.py

@@ -0,0 +1,15 @@
+from django.contrib import admin
+
+from activities.models import Post, TimelineEvent
+
+
+@admin.register(Post)
+class PostAdmin(admin.ModelAdmin):
+    list_display = ["id", "author", "created"]
+    raw_id_fields = ["to", "mentions"]
+
+
+@admin.register(TimelineEvent)
+class TimelineEventAdmin(admin.ModelAdmin):
+    list_display = ["id", "identity", "created", "type"]
+    raw_id_fields = ["identity", "subject_post", "subject_identity"]

activities/apps.py

@@ -1,6 +1,6 @@
 from django.apps import AppConfig
 
 
-class StatusesConfig(AppConfig):
+class ActivitiesConfig(AppConfig):
     default_auto_field = "django.db.models.BigAutoField"
-    name = "statuses"
+    name = "activities"

activities/migrations/0001_initial.py

@@ -0,0 +1,155 @@
+# Generated by Django 4.1.3 on 2022-11-11 20:02
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+import activities.models.post
+import stator.models
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        ("users", "0001_initial"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="Post",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("state_ready", models.BooleanField(default=False)),
+                ("state_changed", models.DateTimeField(auto_now_add=True)),
+                ("state_attempted", models.DateTimeField(blank=True, null=True)),
+                ("state_locked_until", models.DateTimeField(blank=True, null=True)),
+                (
+                    "state",
+                    stator.models.StateField(
+                        choices=[("new", "new"), ("fanned_out", "fanned_out")],
+                        default="new",
+                        graph=activities.models.post.PostStates,
+                        max_length=100,
+                    ),
+                ),
+                ("local", models.BooleanField()),
+                ("object_uri", models.CharField(blank=True, max_length=500, null=True)),
+                (
+                    "visibility",
+                    models.IntegerField(
+                        choices=[
+                            (0, "Public"),
+                            (1, "Unlisted"),
+                            (2, "Followers"),
+                            (3, "Mentioned"),
+                        ],
+                        default=0,
+                    ),
+                ),
+                ("content", models.TextField()),
+                ("sensitive", models.BooleanField(default=False)),
+                ("summary", models.TextField(blank=True, null=True)),
+                ("url", models.CharField(blank=True, max_length=500, null=True)),
+                (
+                    "in_reply_to",
+                    models.CharField(blank=True, max_length=500, null=True),
+                ),
+                ("created", models.DateTimeField(auto_now_add=True)),
+                ("updated", models.DateTimeField(auto_now=True)),
+                (
+                    "author",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.PROTECT,
+                        related_name="statuses",
+                        to="users.identity",
+                    ),
+                ),
+                (
+                    "mentions",
+                    models.ManyToManyField(
+                        related_name="posts_mentioning", to="users.identity"
+                    ),
+                ),
+                (
+                    "to",
+                    models.ManyToManyField(
+                        related_name="posts_to", to="users.identity"
+                    ),
+                ),
+            ],
+            options={
+                "abstract": False,
+            },
+        ),
+        migrations.CreateModel(
+            name="TimelineEvent",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                (
+                    "type",
+                    models.CharField(
+                        choices=[
+                            ("post", "Post"),
+                            ("mention", "Mention"),
+                            ("like", "Like"),
+                            ("follow", "Follow"),
+                            ("boost", "Boost"),
+                        ],
+                        max_length=100,
+                    ),
+                ),
+                ("created", models.DateTimeField(auto_now_add=True)),
+                (
+                    "identity",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="timeline_events",
+                        to="users.identity",
+                    ),
+                ),
+                (
+                    "subject_identity",
+                    models.ForeignKey(
+                        blank=True,
+                        null=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="timeline_events_about_us",
+                        to="users.identity",
+                    ),
+                ),
+                (
+                    "subject_post",
+                    models.ForeignKey(
+                        blank=True,
+                        null=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="timeline_events_about_us",
+                        to="activities.post",
+                    ),
+                ),
+            ],
+            options={
+                "index_together": {
+                    ("identity", "type", "subject_post", "subject_identity"),
+                    ("identity", "type", "subject_identity"),
+                },
+            },
+        ),
+    ]

activities/models/__init__.py

@@ -0,0 +1,2 @@
+from .post import Post  # noqa
+from .timeline_event import TimelineEvent  # noqa

activities/models/post.py

@@ -0,0 +1,161 @@
+import urlman
+from django.db import models
+
+from activities.models.timeline_event import TimelineEvent
+from core.html import sanitize_post
+from stator.models import State, StateField, StateGraph, StatorModel
+from users.models.follow import Follow
+from users.models.identity import Identity
+
+
+class PostStates(StateGraph):
+    new = State(try_interval=300)
+    fanned_out = State()
+
+    new.transitions_to(fanned_out)
+
+    @classmethod
+    async def handle_new(cls, instance: "Post"):
+        """
+        Creates all needed fan-out objects for a new Post.
+        """
+        pass
+
+
+class Post(StatorModel):
+    """
+    A post (status, toot) that is either local or remote.
+    """
+
+    class Visibilities(models.IntegerChoices):
+        public = 0
+        unlisted = 1
+        followers = 2
+        mentioned = 3
+
+    # The author (attributedTo) of the post
+    author = models.ForeignKey(
+        "users.Identity",
+        on_delete=models.PROTECT,
+        related_name="posts",
+    )
+
+    # The state the post is in
+    state = StateField(PostStates)
+
+    # If it is our post or not
+    local = models.BooleanField()
+
+    # The canonical object ID
+    object_uri = models.CharField(max_length=500, blank=True, null=True)
+
+    # Who should be able to see this Post
+    visibility = models.IntegerField(
+        choices=Visibilities.choices,
+        default=Visibilities.public,
+    )
+
+    # The main (HTML) content
+    content = models.TextField()
+
+    # If the contents of the post are sensitive, and the summary (content
+    # warning) to show if it is
+    sensitive = models.BooleanField(default=False)
+    summary = models.TextField(blank=True, null=True)
+
+    # The public, web URL of this Post on the original server
+    url = models.CharField(max_length=500, blank=True, null=True)
+
+    # The Post it is replying to as an AP ID URI
+    # (as otherwise we'd have to pull entire threads to use IDs)
+    in_reply_to = models.CharField(max_length=500, blank=True, null=True)
+
+    # The identities the post is directly to (who can see it if not public)
+    to = models.ManyToManyField(
+        "users.Identity",
+        related_name="posts_to",
+        blank=True,
+    )
+
+    # The identities mentioned in the post
+    mentions = models.ManyToManyField(
+        "users.Identity",
+        related_name="posts_mentioning",
+        blank=True,
+    )
+
+    created = models.DateTimeField(auto_now_add=True)
+    updated = models.DateTimeField(auto_now=True)
+
+    class urls(urlman.Urls):
+        view = "{self.identity.urls.view}posts/{self.id}/"
+
+    def __str__(self):
+        return f"{self.author} #{self.id}"
+
+    @property
+    def safe_content(self):
+        return sanitize_post(self.content)
+
+    ### Local creation ###
+
+    @classmethod
+    def create_local(cls, author: Identity, content: str) -> "Post":
+        post = cls.objects.create(
+            author=author,
+            content=content,
+            local=True,
+        )
+        post.object_uri = post.author.actor_uri + f"posts/{post.id}/"
+        post.url = post.object_uri
+        post.save()
+        return post
+
+    ### ActivityPub (outgoing) ###
+
+    ### ActivityPub (incoming) ###
+
+    @classmethod
+    def by_ap(cls, data, create=False) -> "Post":
+        """
+        Retrieves a Post instance by its ActivityPub JSON object.
+
+        Optionally creates one if it's not present.
+        Raises KeyError if it's not found and create is False.
+        """
+        # Do we have one with the right ID?
+        try:
+            return cls.objects.get(object_uri=data["id"])
+        except cls.DoesNotExist:
+            if create:
+                # Resolve the author
+                author = Identity.by_actor_uri(data["attributedTo"], create=create)
+                return cls.objects.create(
+                    author=author,
+                    content=sanitize_post(data["content"]),
+                    summary=data.get("summary", None),
+                    sensitive=data.get("as:sensitive", False),
+                    url=data.get("url", None),
+                    local=False,
+                    # TODO: to
+                    # TODO: mentions
+                    # TODO: visibility
+                )
+            else:
+                raise KeyError(f"No post with ID {data['id']}", data)
+
+    @classmethod
+    def handle_create_ap(cls, data):
+        """
+        Handles an incoming create request
+        """
+        # Ensure the Create actor is the Post's attributedTo
+        if data["actor"] != data["object"]["attributedTo"]:
+            raise ValueError("Create actor does not match its Post object", data)
+        # Create it
+        post = cls.by_ap(data["object"], create=True)
+        # Make timeline events as appropriate
+        for follow in Follow.objects.filter(target=post.author, source__local=True):
+            TimelineEvent.add_post(follow.source, post)
+        # Force it into fanned_out as it's not ours
+        post.transition_perform(PostStates.fanned_out)

activities/models/timeline_event.py

@@ -0,0 +1,85 @@
+from django.db import models
+
+
+class TimelineEvent(models.Model):
+    """
+    Something that has happened to an identity that we want them to see on one
+    or more timelines, like posts, likes and follows.
+    """
+
+    class Types(models.TextChoices):
+        post = "post"
+        mention = "mention"
+        like = "like"
+        follow = "follow"
+        boost = "boost"
+
+    # The user this event is for
+    identity = models.ForeignKey(
+        "users.Identity",
+        on_delete=models.CASCADE,
+        related_name="timeline_events",
+    )
+
+    # What type of event it is
+    type = models.CharField(max_length=100, choices=Types.choices)
+
+    # The subject of the event (which is used depends on the type)
+    subject_post = models.ForeignKey(
+        "activities.Post",
+        on_delete=models.CASCADE,
+        blank=True,
+        null=True,
+        related_name="timeline_events_about_us",
+    )
+    subject_identity = models.ForeignKey(
+        "users.Identity",
+        on_delete=models.CASCADE,
+        blank=True,
+        null=True,
+        related_name="timeline_events_about_us",
+    )
+
+    created = models.DateTimeField(auto_now_add=True)
+
+    class Meta:
+        index_together = [
+            # This relies on a DB that can use left subsets of indexes
+            ("identity", "type", "subject_post", "subject_identity"),
+            ("identity", "type", "subject_identity"),
+        ]
+
+    ### Alternate constructors ###
+
+    @classmethod
+    def add_follow(cls, identity, source_identity):
+        """
+        Adds a follow to the timeline if it's not there already
+        """
+        return cls.objects.get_or_create(
+            identity=identity,
+            type=cls.Types.follow,
+            subject_identity=source_identity,
+        )[0]
+
+    @classmethod
+    def add_post(cls, identity, post):
+        """
+        Adds a post to the timeline if it's not there already
+        """
+        return cls.objects.get_or_create(
+            identity=identity,
+            type=cls.Types.post,
+            subject_post=post,
+        )[0]
+
+    @classmethod
+    def add_like(cls, identity, post):
+        """
+        Adds a like to the timeline if it's not there already
+        """
+        return cls.objects.get_or_create(
+            identity=identity,
+            type=cls.Types.like,
+            subject_post=post,
+        )[0]

activities/views/home.py

@@ -1,17 +1,18 @@
 from django import forms
 from django.shortcuts import redirect
+from django.template.defaultfilters import linebreaks_filter
 from django.utils.decorators import method_decorator
 from django.views.generic import FormView
 
+from activities.models import Post, TimelineEvent
 from core.forms import FormHelper
-from statuses.models import Status
 from users.decorators import identity_required
 
 
 @method_decorator(identity_required, name="dispatch")
 class Home(FormView):
 
-    template_name = "statuses/home.html"
+    template_name = "activities/home.html"
 
     class form_class(forms.Form):
         text = forms.CharField()
@@ -22,14 +23,20 @@ class Home(FormView):
         context = super().get_context_data()
         context.update(
             {
-                "statuses": self.request.identity.statuses.all()[:100],
+                "timeline_posts": [
+                    te.subject_post
+                    for te in TimelineEvent.objects.filter(
+                        identity=self.request.identity,
+                        type=TimelineEvent.Types.post,
+                    ).order_by("-created")[:100]
+                ],
             }
         )
         return context
 
     def form_valid(self, form):
-        Status.create_local(
+        Post.create_local(
-            identity=self.request.identity,
+            author=self.request.identity,
-            text=form.cleaned_data["text"],
+            content=linebreaks_filter(form.cleaned_data["text"]),
         )
         return redirect(".")

core/html.py

@@ -0,0 +1,11 @@
+import bleach
+from django.utils.safestring import mark_safe
+
+
+def sanitize_post(post_html: str) -> str:
+    """
+    Only allows a, br, p and span tags, and class attributes.
+    """
+    return mark_safe(
+        bleach.clean(post_html, tags=["a", "br", "p", "span"], attributes=["class"])
+    )

core/ld.py

@@ -1,4 +1,5 @@
 import urllib.parse as urllib_parse
+from typing import Dict, List, Union
 
 from pyld import jsonld
 from pyld.jsonld import JsonLdError
@@ -299,24 +300,27 @@ def builtin_document_loader(url: str, options={}):
             )
 
 
-def canonicalise(json_data, include_security=False):
+def canonicalise(json_data: Dict, include_security: bool = False) -> Dict:
     """
     Given an ActivityPub JSON-LD document, round-trips it through the LD
     systems to end up in a canonicalised, compacted format.
 
+    If no context is provided, supplies one automatically.
+
     For most well-structured incoming data this won't actually do anything,
     but it's probably good to abide by the spec.
     """
-    if not isinstance(json_data, (dict, list)):
+    if not isinstance(json_data, dict):
         raise ValueError("Pass decoded JSON data into LDDocument")
-    return jsonld.compact(
+    context: Union[str, List[str]]
-        jsonld.expand(json_data),
+    if include_security:
-        (
+        context = [
-            [
+            "https://www.w3.org/ns/activitystreams",
-                "https://www.w3.org/ns/activitystreams",
+            "https://w3id.org/security/v1",
-                "https://w3id.org/security/v1",
+        ]
-            ]
+    else:
-            if include_security
+        context = "https://www.w3.org/ns/activitystreams"
-            else "https://www.w3.org/ns/activitystreams"
+    if "@context" not in json_data:
-        ),
+        json_data["@context"] = context
-    )
+
+    return jsonld.compact(jsonld.expand(json_data), context)

core/views.py

@@ -1,6 +1,6 @@
 from django.views.generic import TemplateView
 
-from statuses.views.home import Home
+from activities.views.home import Home
 from users.models import Identity
 
 

requirements.txt

@@ -8,4 +8,5 @@ httpx~=0.23
 pyOpenSSL~=22.1.0
 uvicorn~=0.19
 gunicorn~=20.1.0
-psycopg2==2.9.5
+psycopg2~=2.9.5
+bleach~=5.0.1

static/css/style.css

@@ -290,3 +290,41 @@ h1.identity small {
 .system-note a {
     color: inherit;
 }
+
+/* Posts */
+
+.post {
+    margin-bottom: 20px;
+    overflow: hidden;
+}
+
+.post .icon {
+    height: 48px;
+    width: auto;
+    float: left;
+}
+
+.post .author {
+    padding-left: 64px;
+}
+
+.post .author a,
+.post time a {
+    color: inherit;
+    text-decoration: none;
+}
+
+.post .author small {
+    font-weight: normal;
+    color: var(--color-text-dull);
+}
+
+.post time {
+    display: block;
+    padding-left: 64px;
+    color: var(--color-text-duller);
+}
+
+.post .content {
+    padding-left: 64px;
+}

stator/models.py

@@ -1,4 +1,5 @@
 import datetime
+import pprint
 import traceback
 from typing import ClassVar, List, Optional, Type, Union, cast
 
@@ -218,10 +219,16 @@ class StatorError(models.Model):
         instance: StatorModel,
         exception: Optional[BaseException] = None,
     ):
+        detail = traceback.format_exc()
+        if exception and len(exception.args) > 1:
+            detail += "\n\n" + "\n\n".join(
+                pprint.pformat(arg) for arg in exception.args
+            )
+
         return await cls.objects.acreate(
             model_label=instance._meta.label_lower,
             instance_pk=str(instance.pk),
             state=instance.state,
             error=str(exception),
-            error_details=traceback.format_exc(),
+            error_details=detail,
         )

statuses/admin.py

@@ -1,8 +0,0 @@
-from django.contrib import admin
-
-from statuses.models import Status
-
-
-@admin.register(Status)
-class StatusAdmin(admin.ModelAdmin):
-    pass

statuses/migrations/0001_initial.py

@@ -1,56 +0,0 @@
-# Generated by Django 4.1.3 on 2022-11-10 05:58
-
-import django.db.models.deletion
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    initial = True
-
-    dependencies = [
-        ("users", "0001_initial"),
-    ]
-
-    operations = [
-        migrations.CreateModel(
-            name="Status",
-            fields=[
-                (
-                    "id",
-                    models.BigAutoField(
-                        auto_created=True,
-                        primary_key=True,
-                        serialize=False,
-                        verbose_name="ID",
-                    ),
-                ),
-                ("local", models.BooleanField()),
-                ("uri", models.CharField(blank=True, max_length=500, null=True)),
-                (
-                    "visibility",
-                    models.IntegerField(
-                        choices=[
-                            (0, "Public"),
-                            (1, "Unlisted"),
-                            (2, "Followers"),
-                            (3, "Mentioned"),
-                        ],
-                        default=0,
-                    ),
-                ),
-                ("text", models.TextField()),
-                ("created", models.DateTimeField(auto_now_add=True)),
-                ("updated", models.DateTimeField(auto_now=True)),
-                ("deleted", models.DateTimeField(blank=True, null=True)),
-                (
-                    "identity",
-                    models.ForeignKey(
-                        on_delete=django.db.models.deletion.PROTECT,
-                        related_name="statuses",
-                        to="users.identity",
-                    ),
-                ),
-            ],
-        ),
-    ]

statuses/models/__init__.py

@@ -1 +0,0 @@
-from .status import Status  # noqa

statuses/models/status.py

@@ -1,42 +0,0 @@
-import urlman
-from django.db import models
-
-
-class Status(models.Model):
-    class StatusVisibility(models.IntegerChoices):
-        public = 0
-        unlisted = 1
-        followers = 2
-        mentioned = 3
-
-    identity = models.ForeignKey(
-        "users.Identity",
-        on_delete=models.PROTECT,
-        related_name="statuses",
-    )
-
-    local = models.BooleanField()
-    uri = models.CharField(max_length=500, blank=True, null=True)
-    visibility = models.IntegerField(
-        choices=StatusVisibility.choices,
-        default=StatusVisibility.public,
-    )
-    text = models.TextField()
-
-    created = models.DateTimeField(auto_now_add=True)
-    updated = models.DateTimeField(auto_now=True)
-    deleted = models.DateTimeField(null=True, blank=True)
-
-    class Meta:
-        verbose_name_plural = "statuses"
-
-    @classmethod
-    def create_local(cls, identity, text: str):
-        return cls.objects.create(
-            identity=identity,
-            text=text,
-            local=True,
-        )
-
-    class urls(urlman.Urls):
-        view = "{self.identity.urls.view}statuses/{self.id}/"

takahe/settings.py

@@ -24,7 +24,7 @@ INSTALLED_APPS = [
     "django.contrib.staticfiles",
     "crispy_forms",
     "core",
-    "statuses",
+    "activities",
     "users",
     "stator",
 ]

templates/activities/_post.html

@@ -0,0 +1,19 @@
+{% load static %}
+<div class="post">
+
+    {% if post.author.icon_uri %}
+        <img src="{{post.author.icon_uri}}" class="icon">
+    {% else %}
+        <img src="{% static "img/unknown-icon-128.png" %}" class="icon">
+    {% endif %}
+
+    <h3 class="author">
+        {{ post.author.name_or_handle }} <small>@{{ post.author.handle }}</small>
+    </h3>
+    <time>
+        <a href="{{ post.urls.view }}">{{ post.created | timesince }} ago</a>
+    </time>
+    <div class="content">
+        {{ post.safe_content }}
+    </div>
+</div>

templates/activities/home.html

@@ -7,9 +7,9 @@
 
     {% crispy form form.helper %}
 
-    {% for status in statuses %}
+    {% for post in timeline_posts %}
-        {% include "statuses/_status.html" %}
+        {% include "activities/_post.html" %}
     {% empty %}
-        No statuses yet.
+        No posts yet.
     {% endfor %}
 {% endblock %}

templates/identity/view.html

@@ -39,9 +39,9 @@
         </form>
     {% endif %}
 
-    {% for status in statuses %}
+    {% for post in posts %}
-        {% include "statuses/_status.html" %}
+        {% include "activities/_post.html" %}
     {% empty %}
-        No statuses yet.
+        No posts yet.
     {% endfor %}
 {% endblock %}

templates/statuses/_status.html

@@ -1,12 +0,0 @@
-<div class="status">
-    <h3 class="author">
-        <a href="{{ status.identity.urls.view }}">
-            {{ status.identity }}
-            <small>{{ status.identity.handle }}</small>
-        </a>
-    </h3>
-    <time>
-        <a href="{{ status.urls.view }}">{{ status.created | timesince }} ago</a>
-    </time>
-    {{ status.text | linebreaks }}
-</div>

users/admin.py

@@ -21,16 +21,18 @@ class UserEventAdmin(admin.ModelAdmin):
 @admin.register(Identity)
 class IdentityAdmin(admin.ModelAdmin):
     list_display = ["id", "handle", "actor_uri", "state", "local"]
+    raw_id_fields = ["users"]
 
 
 @admin.register(Follow)
 class FollowAdmin(admin.ModelAdmin):
     list_display = ["id", "source", "target", "state"]
+    raw_id_fields = ["source", "target"]
 
 
 @admin.register(InboxMessage)
 class InboxMessageAdmin(admin.ModelAdmin):
-    list_display = ["id", "state", "message_type"]
+    list_display = ["id", "state", "state_attempted", "message_type"]
     actions = ["reset_state"]
 
     @admin.action(description="Reset State")

users/migrations/0001_initial.py

@@ -1,4 +1,4 @@
-# Generated by Django 4.1.3 on 2022-11-10 05:58
+# Generated by Django 4.1.3 on 2022-11-11 20:02
 
 import functools
 
@@ -296,11 +296,14 @@ class Migration(migrations.Migration):
                     "state",
                     stator.models.StateField(
                         choices=[
-                            ("pending", "pending"),
+                            ("unrequested", "unrequested"),
-                            ("requested", "requested"),
+                            ("local_requested", "local_requested"),
+                            ("remote_requested", "remote_requested"),
                             ("accepted", "accepted"),
+                            ("undone_locally", "undone_locally"),
+                            ("undone_remotely", "undone_remotely"),
                         ],
-                        default="pending",
+                        default="unrequested",
                         graph=users.models.follow.FollowStates,
                         max_length=100,
                     ),

users/models/domain.py

@@ -49,10 +49,7 @@ class Domain(models.Model):
 
     @classmethod
     def get_remote_domain(cls, domain: str) -> "Domain":
-        try:
+        return cls.objects.get_or_create(domain=domain, local=False)[0]
-            return cls.objects.get(domain=domain, local=False)
-        except cls.DoesNotExist:
-            return cls.objects.create(domain=domain, local=False)
 
     @classmethod
     def get_domain(cls, domain: str) -> Optional["Domain"]:
@@ -93,3 +90,4 @@ class Domain(models.Model):
                 raise ValueError(
                     f"Service domain {self.service_domain} is already a domain elsewhere!"
                 )
+        super().save(*args, **kwargs)

users/models/follow.py

@@ -5,10 +5,11 @@ from django.db import models
 from core.ld import canonicalise
 from core.signatures import HttpSignature
 from stator.models import State, StateField, StateGraph, StatorModel
+from users.models.identity import Identity
 
 
 class FollowStates(StateGraph):
-    unrequested = State(try_interval=30)
+    unrequested = State(try_interval=300)
     local_requested = State(try_interval=24 * 60 * 60)
     remote_requested = State(try_interval=24 * 60 * 60)
     accepted = State(externally_progressed=True)
@@ -24,26 +25,19 @@ class FollowStates(StateGraph):
 
     @classmethod
     async def handle_unrequested(cls, instance: "Follow"):
-        # Re-retrieve the follow with more things linked
+        """
-        follow = await Follow.objects.select_related(
+        Follows that are unrequested need us to deliver the Follow object
-            "source", "source__domain", "target"
+        to the target server.
-        ).aget(pk=instance.pk)
+        """
+        follow = await instance.afetch_full()
         # Remote follows should not be here
         if not follow.source.local:
             return cls.remote_requested
-        # Construct the request
-        request = canonicalise(
-            {
-                "@context": "https://www.w3.org/ns/activitystreams",
-                "id": follow.uri,
-                "type": "Follow",
-                "actor": follow.source.actor_uri,
-                "object": follow.target.actor_uri,
-            }
-        )
         # Sign it and send it
         await HttpSignature.signed_request(
-            follow.target.inbox_uri, request, follow.source
+            uri=follow.target.inbox_uri,
+            body=canonicalise(follow.to_ap()),
+            identity=follow.source,
         )
         return cls.local_requested
 
@@ -54,56 +48,28 @@ class FollowStates(StateGraph):
 
     @classmethod
     async def handle_remote_requested(cls, instance: "Follow"):
-        # Re-retrieve the follow with more things linked
+        """
-        follow = await Follow.objects.select_related(
+        Items in remote_requested need us to send an Accept object to the
-            "source", "source__domain", "target"
+        source server.
-        ).aget(pk=instance.pk)
+        """
-        # Send an accept
+        follow = await instance.afetch_full()
-        request = canonicalise(
-            {
-                "@context": "https://www.w3.org/ns/activitystreams",
-                "id": follow.target.actor_uri + f"follow/{follow.pk}/#accept",
-                "type": "Follow",
-                "actor": follow.source.actor_uri,
-                "object": {
-                    "id": follow.uri,
-                    "type": "Follow",
-                    "actor": follow.source.actor_uri,
-                    "object": follow.target.actor_uri,
-                },
-            }
-        )
-        # Sign it and send it
         await HttpSignature.signed_request(
-            follow.source.inbox_uri,
+            uri=follow.source.inbox_uri,
-            request,
+            body=canonicalise(follow.to_accept_ap()),
             identity=follow.target,
         )
         return cls.accepted
 
     @classmethod
     async def handle_undone_locally(cls, instance: "Follow"):
-        follow = Follow.objects.select_related(
+        """
-            "source", "source__domain", "target"
+        Delivers the Undo object to the target server
-        ).get(pk=instance.pk)
+        """
-        # Construct the request
+        follow = await instance.afetch_full()
-        request = canonicalise(
-            {
-                "@context": "https://www.w3.org/ns/activitystreams",
-                "id": follow.uri + "#undo",
-                "type": "Undo",
-                "actor": follow.source.actor_uri,
-                "object": {
-                    "id": follow.uri,
-                    "type": "Follow",
-                    "actor": follow.source.actor_uri,
-                    "object": follow.target.actor_uri,
-                },
-            }
-        )
-        # Sign it and send it
         await HttpSignature.signed_request(
-            follow.target.inbox_uri, request, follow.source
+            uri=follow.target.inbox_uri,
+            body=canonicalise(follow.to_undo_ap()),
+            identity=follow.source,
         )
         return cls.undone_remotely
 
@@ -135,6 +101,11 @@ class Follow(StatorModel):
     class Meta:
         unique_together = [("source", "target")]
 
+    def __str__(self):
+        return f"#{self.id}: {self.source} → {self.target}"
+
+    ### Alternate fetchers/constructors ###
+
     @classmethod
     def maybe_get(cls, source, target) -> Optional["Follow"]:
         """
@@ -164,22 +135,122 @@ class Follow(StatorModel):
             follow.save()
         return follow
 
-    @classmethod
+    ### Async helpers ###
-    def remote_created(cls, source, target, uri):
+
-        follow = cls.maybe_get(source=source, target=target)
+    async def afetch_full(self):
-        if follow is None:
+        """
-            follow = Follow.objects.create(source=source, target=target, uri=uri)
+        Returns a version of the object with all relations pre-loaded
-        if follow.state == FollowStates.unrequested:
+        """
-            follow.transition_perform(FollowStates.remote_requested)
+        return await Follow.objects.select_related(
+            "source", "source__domain", "target"
+        ).aget(pk=self.pk)
+
+    ### ActivityPub (outbound) ###
+
+    def to_ap(self):
+        """
+        Returns the AP JSON for this object
+        """
+        return {
+            "type": "Follow",
+            "id": self.uri,
+            "actor": self.source.actor_uri,
+            "object": self.target.actor_uri,
+        }
+
+    def to_accept_ap(self):
+        """
+        Returns the AP JSON for this objects' accept.
+        """
+        return {
+            "type": "Accept",
+            "id": self.uri + "#accept",
+            "actor": self.target.actor_uri,
+            "object": self.to_ap(),
+        }
+
+    def to_undo_ap(self):
+        """
+        Returns the AP JSON for this objects' undo.
+        """
+        return {
+            "type": "Undo",
+            "id": self.uri + "#undo",
+            "actor": self.source.actor_uri,
+            "object": self.to_ap(),
+        }
+
+    ### ActivityPub (inbound) ###
 
     @classmethod
-    def remote_accepted(cls, source, target):
+    def by_ap(cls, data, create=False) -> "Follow":
-        print(f"accepted follow source {source} target {target}")
+        """
+        Retrieves a Follow instance by its ActivityPub JSON object.
+
+        Optionally creates one if it's not present.
+        Raises KeyError if it's not found and create is False.
+        """
+        # Resolve source and target and see if a Follow exists
+        source = Identity.by_actor_uri(data["actor"], create=create)
+        target = Identity.by_actor_uri(data["object"])
         follow = cls.maybe_get(source=source, target=target)
-        print(f"accepting follow {follow}")
+        # If it doesn't exist, create one in the remote_requested state
+        if follow is None:
+            if create:
+                return cls.objects.create(
+                    source=source,
+                    target=target,
+                    uri=data["id"],
+                    state=FollowStates.remote_requested,
+                )
+            else:
+                raise KeyError(
+                    f"No follow with source {source} and target {target}", data
+                )
+        else:
+            return follow
+
+    @classmethod
+    def handle_request_ap(cls, data):
+        """
+        Handles an incoming follow request
+        """
+        follow = cls.by_ap(data, create=True)
+        # Force it into remote_requested so we send an accept
+        follow.transition_perform(FollowStates.remote_requested)
+
+    @classmethod
+    def handle_accept_ap(cls, data):
+        """
+        Handles an incoming Follow Accept for one of our follows
+        """
+        # Ensure the Accept actor is the Follow's object
+        if data["actor"] != data["object"]["object"]:
+            raise ValueError("Accept actor does not match its Follow object", data)
+        # Resolve source and target and see if a Follow exists (it really should)
+        try:
+            follow = cls.by_ap(data["object"])
+        except KeyError:
+            raise ValueError("No Follow locally for incoming Accept", data)
+        # If the follow was waiting to be accepted, transition it
         if follow and follow.state in [
             FollowStates.unrequested,
             FollowStates.local_requested,
         ]:
             follow.transition_perform(FollowStates.accepted)
-            print("accepted")
+
+    @classmethod
+    def handle_undo_ap(cls, data):
+        """
+        Handles an incoming Follow Undo for one of our follows
+        """
+        # Ensure the Undo actor is the Follow's actor
+        if data["actor"] != data["object"]["actor"]:
+            raise ValueError("Undo actor does not match its Follow object", data)
+        # Resolve source and target and see if a Follow exists (it hopefully does)
+        try:
+            follow = cls.by_ap(data["object"])
+        except KeyError:
+            raise ValueError("No Follow locally for incoming Undo", data)
+        # Delete the follow
+        follow.delete()

users/models/identity.py

@@ -55,7 +55,11 @@ class Identity(StatorModel):
     state = StateField(IdentityStates)
 
     local = models.BooleanField()
-    users = models.ManyToManyField("users.User", related_name="identities")
+    users = models.ManyToManyField(
+        "users.User",
+        related_name="identities",
+        blank=True,
+    )
 
     username = models.CharField(max_length=500, blank=True, null=True)
     # Must be a display domain if present
@@ -141,18 +145,14 @@ class Identity(StatorModel):
             return None
 
     @classmethod
-    def by_actor_uri(cls, uri) -> Optional["Identity"]:
+    def by_actor_uri(cls, uri, create=False) -> "Identity":
         try:
             return cls.objects.get(actor_uri=uri)
         except cls.DoesNotExist:
-            return None
+            if create:
-
+                return cls.objects.create(actor_uri=uri, local=False)
-    @classmethod
+            else:
-    def by_actor_uri_with_create(cls, uri) -> "Identity":
+                raise KeyError(f"No identity found matching {uri}")
-        try:
-            return cls.objects.get(actor_uri=uri)
-        except cls.DoesNotExist:
-            return cls.objects.create(actor_uri=uri, local=False)
 
     ### Dynamic properties ###
 
@@ -236,7 +236,7 @@ class Identity(StatorModel):
             self.outbox_uri = document.get("outbox")
             self.summary = document.get("summary")
             self.username = document.get("preferredUsername")
-            if "@value" in self.username:
+            if self.username and "@value" in self.username:
                 self.username = self.username["@value"]
             self.manually_approves_followers = document.get(
                 "as:manuallyApprovesFollowers"

users/models/inbox_message.py

@@ -2,7 +2,6 @@ from asgiref.sync import sync_to_async
 from django.db import models
 
 from stator.models import State, StateField, StateGraph, StatorModel
-from users.models import Follow, Identity
 
 
 class InboxMessageStates(StateGraph):
@@ -13,23 +12,38 @@ class InboxMessageStates(StateGraph):
 
     @classmethod
     async def handle_received(cls, instance: "InboxMessage"):
-        type = instance.message_type
+        from activities.models import Post
-        if type == "follow":
+        from users.models import Follow
-            await instance.follow_request()
+
-        elif type == "accept":
+        match instance.message_type:
-            inner_type = instance.message["object"]["type"].lower()
+            case "follow":
-            if inner_type == "follow":
+                await sync_to_async(Follow.handle_request_ap)(instance.message)
-                await instance.follow_accepted()
+            case "create":
-            else:
+                match instance.message_object_type:
-                raise ValueError(f"Cannot handle activity of type accept.{inner_type}")
+                    case "note":
-        elif type == "undo":
+                        await sync_to_async(Post.handle_create_ap)(instance.message)
-            inner_type = instance.message["object"]["type"].lower()
+                    case unknown:
-            if inner_type == "follow":
+                        raise ValueError(
-                await instance.follow_undo()
+                            f"Cannot handle activity of type create.{unknown}"
-            else:
+                        )
-                raise ValueError(f"Cannot handle activity of type undo.{inner_type}")
+            case "accept":
-        else:
+                match instance.message_object_type:
-            raise ValueError(f"Cannot handle activity of type {type}")
+                    case "follow":
+                        await sync_to_async(Follow.handle_accept_ap)(instance.message)
+                    case unknown:
+                        raise ValueError(
+                            f"Cannot handle activity of type accept.{unknown}"
+                        )
+            case "undo":
+                match instance.message_object_type:
+                    case "follow":
+                        await sync_to_async(Follow.handle_undo_ap)(instance.message)
+                    case unknown:
+                        raise ValueError(
+                            f"Cannot handle activity of type undo.{unknown}"
+                        )
+            case unknown:
+                raise ValueError(f"Cannot handle activity of type {unknown}")
         return cls.processed
 
 
@@ -45,35 +59,10 @@ class InboxMessage(StatorModel):
 
     state = StateField(InboxMessageStates)
 
-    @sync_to_async
-    def follow_request(self):
-        """
-        Handles an incoming follow request
-        """
-        Follow.remote_created(
-            source=Identity.by_actor_uri_with_create(self.message["actor"]),
-            target=Identity.by_actor_uri(self.message["object"]),
-            uri=self.message["id"],
-        )
-
-    @sync_to_async
-    def follow_accepted(self):
-        """
-        Handles an incoming acceptance of one of our follow requests
-        """
-        target = Identity.by_actor_uri_with_create(self.message["actor"])
-        source = Identity.by_actor_uri(self.message["object"]["actor"])
-        if source is None:
-            raise ValueError(
-                f"Follow-Accept has invalid source {self.message['object']['actor']}"
-            )
-        Follow.remote_accepted(source=source, target=target)
-
     @property
     def message_type(self):
         return self.message["type"].lower()
 
-    async def follow_undo(self):
+    @property
-        """
+    def message_object_type(self):
-        Handles an incoming follow undo
+        return self.message["object"]["type"].lower()
-        """

users/views/identity.py

@@ -222,7 +222,7 @@ class Inbox(View):
         # Find the Identity by the actor on the incoming item
         # This ensures that the signature used for the headers matches the actor
         # described in the payload.
-        identity = Identity.by_actor_uri_with_create(document["actor"])
+        identity = Identity.by_actor_uri(document["actor"], create=True)
         if not identity.public_key:
             # See if we can fetch it right now
             async_to_sync(identity.fetch_actor)()