From e52dffe9ff5b36ae60377ddc005f34f829940690 Mon Sep 17 00:00:00 2001 From: Marnanel Thurman Date: Fri, 1 May 2020 13:48:14 +0100 Subject: [PATCH] Split out the part of bowler_pub that sends ActivityPub notifications into a new app, sombrero_sendpub. Tests not yet passing. --- kepi/bowler_pub/create.py | 192 --------- kepi/bowler_pub/find.py | 345 ---------------- kepi/bowler_pub/forms.py | 26 -- kepi/bowler_pub/tests/test_audience.py | 97 ----- kepi/bowler_pub/tests/test_collection.py | 260 ------------ kepi/bowler_pub/tests/test_find.py | 96 ----- kepi/bowler_pub/tests/test_send_to_outbox.py | 388 ------------------ kepi/bowler_pub/utils.py | 12 + kepi/bowler_pub/views/activitypub.py | 3 +- .../templates/host-meta.xml | 0 kepi/kepi/settings.py | 1 + kepi/sombrero_sendpub/__init__.py | 0 kepi/sombrero_sendpub/admin.py | 3 + .../delivery.py | 0 .../middleware.py | 0 kepi/sombrero_sendpub/migrations/__init__.py | 0 kepi/sombrero_sendpub/models.py | 3 + kepi/sombrero_sendpub/tests/__init__.py | 0 .../tests/test_deliver.py | 0 kepi/sombrero_sendpub/views.py | 3 + kepi/trilby_api/admin.py | 1 - kepi/trilby_api/models/follow.py | 1 - kepi/trilby_api/models/like.py | 1 - kepi/trilby_api/models/notification.py | 1 - kepi/trilby_api/models/person.py | 1 - kepi/trilby_api/models/status.py | 1 - 26 files changed, 23 insertions(+), 1412 deletions(-) delete mode 100644 kepi/bowler_pub/create.py delete mode 100644 kepi/bowler_pub/find.py delete mode 100644 kepi/bowler_pub/forms.py delete mode 100644 kepi/bowler_pub/tests/test_audience.py delete mode 100644 kepi/bowler_pub/tests/test_collection.py delete mode 100644 kepi/bowler_pub/tests/test_find.py delete mode 100644 kepi/bowler_pub/tests/test_send_to_outbox.py rename kepi/{bowler_pub => busby_1st}/templates/host-meta.xml (100%) create mode 100644 kepi/sombrero_sendpub/__init__.py create mode 100644 kepi/sombrero_sendpub/admin.py rename kepi/{bowler_pub => sombrero_sendpub}/delivery.py (100%) rename kepi/{bowler_pub => sombrero_sendpub}/middleware.py (100%) create mode 100644 kepi/sombrero_sendpub/migrations/__init__.py create mode 100644 kepi/sombrero_sendpub/models.py create mode 100644 kepi/sombrero_sendpub/tests/__init__.py rename kepi/{bowler_pub => sombrero_sendpub}/tests/test_deliver.py (100%) create mode 100644 kepi/sombrero_sendpub/views.py diff --git a/kepi/bowler_pub/create.py b/kepi/bowler_pub/create.py deleted file mode 100644 index 929e4af..0000000 --- a/kepi/bowler_pub/create.py +++ /dev/null @@ -1,192 +0,0 @@ -# create.py -# -# Part of kepi, an ActivityPub daemon and library. -# Copyright (c) 2018-2019 Marnanel Thurman. -# Licensed under the GNU Public License v2. - -""" -This module contains create(), which creates objects. -""" - -import django.db.utils -import logging -import json - -logger = logging.getLogger(name='kepi') - -def _fix_value_type(v): - if type(v) in [str, int, bool]: - # simple types are fine - return v - - try: - result = v.id - except AttributeError: - result = v - - return result - -def create( - value=None, - is_local_user=True, - run_side_effects=True, - run_delivery=True, - send_signal=True, - incoming=False, - **kwargs): - - """ - Creates a bowler_pub object. - - Keyword arguments: - value -- the fields of the new object, as a dict. - Must contain a key "type". - is_local_user -- ignored (FIXME) - run_side_effects -- whether the new object should cause - its usual side effects. For example, creating a Delete - object should have the side-effect of deleting something. - run_delivery -- whether we should attempt to deliver the - new object to whatever audiences it lists. - send_signal -- whether we should send signals.created - about the new object - - Any extra keyword arguments are taken to be fields of the - new object, just as if they had appeared in "value". - Any of these keywords may be prefixed with "f_" to avoid - confusing Python's parser. For example, you could write - f_type = "Create" - to set the "type" field to "Create". - - Don't confuse create() with objects of type Create! - """ - - from kepi.bowler_pub.delivery import deliver - from kepi.bowler_pub.models.activity import AcActivity - from kepi.bowler_pub.models.acobject import AcObject - - if value is None: - value = {} - - # Remove the "f_" prefix, which exists so that we can write - # things like f_type or f_object without using reserved keywords. - for k,v in kwargs.items(): - if k.startswith('f_'): - value[k[2:]] = v - else: - value[k] = v - - logger.info("Create begins: source is %s; run side effects? %s", - value, run_side_effects) - - if value is None: - logger.warn(" -- it's ludicrous to create an Object with no value") - return None - - # Now, let's fix the types of keys and values. - - if 'type' not in value: - logger.warn("Objects must have a type; dropping message") - return None - - value['type'] = value['type'].title() - - for k,v in value.copy().items(): - if not isinstance(k, str): - logger.warn('Objects can only have keys which are strings: %s', - str(k)) - del value[k] - - class_name = 'Ac'+value['type'] - try: - import kepi.bowler_pub.models as bowler_pub_models - cls = getattr(locals()['bowler_pub_models'], class_name) - except AttributeError: - logger.warn("There's no type called %s", - class_name) - return None - except KeyError: - logger.warn("The class '%s' wasn't exported properly. "+\ - "This shouldn't happen.", - class_name) - return None - - logger.debug('Class for %s is %s', value['type'], cls) - del value['type'] - - if 'id' in value and 'url' in value: - if value['id']!=value['url']: - logger.warn('id and url differ (%s vs %s)', - value['id'], value['url']) - del value['url'] - - ######################## - - # Split out the values which have a f_* field in the class. - - primary_values = {} - - class_fields = dir(cls) - for f,v in list(value.items()): - - if f=='id': - new_fieldname = f - else: - new_fieldname = 'f_'+f - - if new_fieldname in class_fields: - - primary_values[new_fieldname] = _fix_value_type(v) - del value[f] - - logger.debug('Primary values are %s; others are %s', - primary_values, value) - - ######################## - - # Right, we need to create an object. - - try: - result = cls( - **primary_values, - ) - result.save() - logger.info(' -- created object %s', - result) - except django.db.utils.IntegrityError: - logger.warn('We already have an object with id=%s; ignoring', - value['id']) - return None - - for f,v in value.items(): - try: - result[f] = _fix_value_type(v) - except django.db.utils.Error as pe: - logger.warn('Can\'t set %s=%s on the new object (%s); bailing', - f, v, pe) - return None - - if hasattr(result, '_after_create'): - result._after_create() - - if run_side_effects: - success = result.run_side_effects( - send_signal = send_signal, - ) - if not success: - logger.debug(' -- side-effects failed; deleting original object') - result.delete() - return None - - if run_delivery and isinstance(result, AcActivity): - deliver(result.id, - incoming = incoming) - - if send_signal: - logger.debug(' -- sending "created" signal') - signals.created.send( - sender = AcObject, - value = result, - ) - - return result - diff --git a/kepi/bowler_pub/find.py b/kepi/bowler_pub/find.py deleted file mode 100644 index 11fcc79..0000000 --- a/kepi/bowler_pub/find.py +++ /dev/null @@ -1,345 +0,0 @@ -# find.py -# -# Part of kepi, an ActivityPub daemon. -# Copyright (c) 2018-2019 Marnanel Thurman. -# Licensed under the GNU Public License v2. - -""" -This module contains find(), which finds objects. -""" - -from django.db import models -import requests -import logging -from django.conf import settings -import django.urls -from urllib.parse import urlparse -from django.http.request import HttpRequest -from kepi.bowler_pub.create import create -from django.utils import timezone -from kepi.bowler_pub.utils import is_short_id -import json -import mimeparse - -logger = logging.getLogger(name='kepi') - -class Fetch(models.Model): - - """ - A record of something having been fetched from a - particular URL at a particular time. It doesn't - contain the actual data; it just keeps the cache fresh. - """ - - url = models.URLField( - primary_key = True, - ) - - date = models.DateTimeField( - default = timezone.now, - ) - -class ActivityRequest(HttpRequest): - - """ - These are fake HttpRequests which we send to the views - as an ACTIVITY_GET or ACTIVITY_STORE method. - - For more information, see the docstring in views/. - """ - - def __init__(self, path, object_to_store): - super().__init__() - - self.path = path - - if object_to_store is None: - self.method = 'ACTIVITY_GET' - else: - self.method = 'ACTIVITY_STORE' - self.activity = object_to_store - -class TombstoneException(Exception): - # XXX obsolete; remove - def __init__(self, tombstone, *args, **kwargs): - self.tombstone = tombstone - super().__init__(*args, **kwargs) - - def __str__(self): - return self.tombstone.__str__() - -################################### - -COLLECTION_TYPES = set([ - 'OrderedCollection', - 'OrderedCollectionPage', - ]) - -class Collection(dict): - """ - The contents of a remote object of any type in COLLECTION_TYPES. - """ - pass - -################################### - -def find_local(path, - object_to_store=None): - - """ - Finds a local object and returns it. - Optionally, passes it another object. - - path -- the path to the object. Note: not the URL. - The URL is redundant because we know this object is local. - object_to_store -- something to give the object when we find it. - For example, if "path" refers to someone's inbox, - "object_to_store" might be an activity to add to it. - """ - - from django.urls import resolve - - try: - resolved = resolve(path) - except django.urls.Resolver404: - logger.debug('%s: -- not found', path) - return None - - logger.debug('%s: handled by %s, %s, %s', - path, - str(resolved.func), - str(resolved.args), - str(resolved.kwargs), - ) - - request = ActivityRequest( - path=path, - object_to_store=object_to_store, - ) - result = resolved.func(request, - **resolved.kwargs) - - if result: - logger.debug('%s: resulting in %s', path, str(result)) - - return result - -def find_remote(url, - do_not_fetch=False, - run_delivery=False): - """ - Finds a remote object and returns it. - We return None if the object couldn't be found, or was - somehow invalid. Otherwise, we create a local copy of - the object and return that. - - As a special case, if the remote object is a collection type, - we return a find.Collection (which is just a dict subclass) - containing its fields. - - url -- the URL of the remote object. - do_not_fetch -- True if we should give up and return None - when the cache doesn't hold the remote object; False - (the default) if we should go and get it via HTTP. - run_delivery -- whether to deliver the found object to - its stated audiences. This is usually not what you want. - """ - - from kepi.bowler_pub.models.acobject import AcObject - - logger.debug('%s: find remote', url) - - try: - fetch = Fetch.objects.get( - url=url, - ) - - # TODO: cache timeouts. - # FIXME: honour cache headers etc - - # We fetched it in the past. - - try: - result = AcObject.objects.get( - id = url, - ) - logger.debug('%s: already fetched, and it\'s %s', - url, result) - - return result - except AcObject.DoesNotExist: - logger.debug('%s: we already know it wasn\'t there', - url) - - return None - - except Fetch.DoesNotExist: - # We haven't fetched it before. - # So we need to fetch it now. - pass - - if do_not_fetch: - logger.info('%s: do_not_fetch was set, so returning None', - url) - return None - - logger.info('%s: performing the GET', url) - response = requests.get(url, - headers={'Accept': 'application/activity+json'}, - ) - - fetch_record = Fetch(url=url) - fetch_record.save() - - if response.status_code!=200: - logger.warn('%s: remote server responded %s %s' % ( - url, - response.status_code, response.reason)) - return None - - mime_type = mimeparse.parse_mime_type( - response.headers['Content-Type']) - mime_type = '/'.join(mime_type[0:2]) - - if mime_type not in [ - 'application/activity+json', - 'application/json', - 'text/json', - 'text/plain', - ]: - logger.warn('%s: response had the wrong Content-Type, %s' % ( - url, response.headers['Content-Type'], - )) - return None - - logger.debug('%s: response was: %s' % ( - url, response.text, - )) - - try: - content = json.loads(response.text) - except json.JSONDecodeError: - logger.warn('%s: response was not JSON' % ( - url, - )) - return None - - if not isinstance(content, dict): - logger.warn('%s: response was not a JSON dict' % ( - url, - )) - return None - - content_without_at = dict([ - (f, v) - for f, v in content.items() - if not f.startswith('@') - ]) - - if content['type'] in COLLECTION_TYPES: - # It might be better if find() downloaded - # an entire Collection if it finds the index. - # At present we expect the caller to do it. - result = Collection(content_without_at) - else: - result = create( - is_local_user = False, - value = content_without_at, - id = url, - run_delivery = run_delivery, - ) - - return result - -def is_local(url): - """ - True if "url" resides on the local server. - - False otherwise. Returns False even if - the string argument is not in fact a URL. - """ - if hasattr(url, 'url'): - url = url.url - - if is_short_id(url): - return True - - parsed_url = urlparse(url) - return parsed_url.hostname in settings.ALLOWED_HOSTS - -def _short_id_lookup(number): - """ - Helper function to find an object when we actually have - its short_id number. - - "number" is the number preceded by a slash - (for example, "/2bad4dec"), - or a name preceded by an atpersat - (for example, "@alice"). - - """ - - from kepi.bowler_pub.models import AcObject - - try: - result = AcObject.objects.get( - id=number, - ) - logger.debug('%s: found %s', - number, result) - - return result - - except AcObject.DoesNotExist: - logger.debug('%s: does not exist', - number) - - return None - -def find(url, - local_only=False, - do_not_fetch=False, - object_to_store=None): - """ - Finds an object. - - Keyword arguments: - address -- the URL of the object. - local_only -- whether to restrict ourselves to local URLs. - object_to_store -- if the address is a local collection, - this is an object to add to that collection. - - If the address is local, we pass it to find_local(), - which will look the object up using Django's usual dispatcher. - If it isn't found, we return None. - - If the address is remote, we pass it to find_remote(), - which will look for the object in the cache and return it. - If it's not in the cache, and "do_not_fetch" is True, - we return None. Otherwise, we fetch the object over HTTP. - - The reason for using "local_only" instead of just calling - find_local() is that this function parses URLs for you. - """ - - if not url: - return None - - if is_short_id(url): - return _short_id_lookup(url) - - parsed_url = urlparse(url) - is_local = parsed_url.hostname in settings.ALLOWED_HOSTS - - if is_local: - return find_local(parsed_url.path, - object_to_store=object_to_store) - else: - if local_only: - logger.info('find: url==%s but is_local==False; ignoring', - url) - return None - - return find_remote( - url=url, - do_not_fetch=do_not_fetch) diff --git a/kepi/bowler_pub/forms.py b/kepi/bowler_pub/forms.py deleted file mode 100644 index 3478ba7..0000000 --- a/kepi/bowler_pub/forms.py +++ /dev/null @@ -1,26 +0,0 @@ -# forms.py -# -# Part of kepi, an ActivityPub daemon. -# Copyright (c) 2018-2019 Marnanel Thurman. -# Licensed under the GNU Public License v2. - -""" -This module contains some forms which are used by -the admin interface. It's not very elaborate yet. -""" - -from django import forms -import kepi.bowler_pub.models as bowler_pub_models - -class NoteAdminForm(forms.ModelForm): - - class Meta: - model = bowler_pub_models.AcNote - - fields = [ - 'f_content', - 'f_attributedTo', - ] - - f_content = forms.CharField( - widget = forms.Textarea) diff --git a/kepi/bowler_pub/tests/test_audience.py b/kepi/bowler_pub/tests/test_audience.py deleted file mode 100644 index 3c44bf4..0000000 --- a/kepi/bowler_pub/tests/test_audience.py +++ /dev/null @@ -1,97 +0,0 @@ -from django.test import TestCase -from kepi.bowler_pub.models import Audience -from kepi.bowler_pub.create import create -from . import create_local_person, REMOTE_FRED, REMOTE_JIM - -class TestAudience(TestCase): - - def setUp(self): - self._narcissus = create_local_person( - name = 'narcissus', - ) - - def test_add_audiences_for(self): - - self._like = create( - f_type = 'Like', - f_actor = self._narcissus, - f_object = self._narcissus, - run_side_effects = False, - run_delivery = False, - ) - - a = Audience.add_audiences_for( - thing = self._like, - field = 'to', - value = [ - REMOTE_FRED, - REMOTE_JIM, - ], - ) - - results = Audience.objects.filter( - parent = self._like, - ) - - self.assertEqual(len(results), 2) - self.assertEqual(results[0].recipient, REMOTE_FRED) - self.assertEqual(results[1].recipient, REMOTE_JIM) - - def test_create(self): - - self._like = create( - f_type = 'Like', - f_actor = self._narcissus, - f_object = self._narcissus, - to = [ REMOTE_FRED, REMOTE_JIM, ], - run_side_effects = False, - run_delivery = False, - ) - - results = Audience.objects.filter( - parent = self._like, - ) - - self.assertEqual(len(results), 2) - self.assertEqual(results[0].recipient, REMOTE_FRED) - self.assertEqual(results[1].recipient, REMOTE_JIM) - - def test_get_audiences_for(self): - - self._like = create( - f_type = 'Like', - f_actor = self._narcissus, - f_object = self._narcissus, - run_side_effects = False, - run_delivery = False, - ) - - for fieldname in ['to', 'cc', 'bcc']: - a = Audience.add_audiences_for( - thing = self._like, - field = fieldname, - value = [ - REMOTE_FRED, - REMOTE_JIM, - ], - ) - - self.assertDictEqual( - Audience.get_audiences_for(self._like), - {'to': ['https://remote.example.org/users/fred', - 'https://remote.example.org/users/jim'], - 'cc': ['https://remote.example.org/users/fred', - 'https://remote.example.org/users/jim'], - 'bcc': ['https://remote.example.org/users/fred', - 'https://remote.example.org/users/jim'], - }) - - self.assertDictEqual( - Audience.get_audiences_for(self._like, - hide_blind = True, - ), - {'to': ['https://remote.example.org/users/fred', - 'https://remote.example.org/users/jim'], - 'cc': ['https://remote.example.org/users/fred', - 'https://remote.example.org/users/jim'], - }) diff --git a/kepi/bowler_pub/tests/test_collection.py b/kepi/bowler_pub/tests/test_collection.py deleted file mode 100644 index 1e714fb..0000000 --- a/kepi/bowler_pub/tests/test_collection.py +++ /dev/null @@ -1,260 +0,0 @@ -from django.test import TestCase, Client -from unittest import skip -from kepi.bowler_pub.models import * -import datetime -import json -from kepi.bowler_pub.find import find -from kepi.bowler_pub.utils import as_json -from . import * -import logging - -logger = logging.Logger('kepi') - -EXAMPLE_SERVER = 'http://testserver' -JSON_TYPE = 'application/activity+json; charset=utf-8' -PAGE_LENGTH = 50 - -class CollectionTests(TestCase): - - def setUp(self): - settings.KEPI['LOCAL_OBJECT_HOSTNAME'] = 'testserver' - self.maxDiff = None - - def check_collection(self, - path, - expectedTotalItems): - - c = Client( - HTTP_ACCEPT = JSON_TYPE, - ) - - response = c.get(path) - - if response.status_code!=200: - raise RuntimeError(response.content) - - self.assertEqual(response['Content-Type'], JSON_TYPE) - - result = json.loads(response.content.decode(encoding='UTF-8')) - - for field in [ - '@context', - 'id', - 'totalItems', - 'type', - ]: - self.assertIn(field, result) - - if expectedTotalItems==0: - self.assertNotIn('first', result) - else: - self.assertIn('first', result) - self.assertEqual(result['first'], EXAMPLE_SERVER+path+'?page=1') - - self.assertIn('last', result) - - lastpage = 1 + int((expectedTotalItems+1)/PAGE_LENGTH) - - self.assertEqual(result['last'], EXAMPLE_SERVER+path+\ - ('?page=%d' % (lastpage,))) - - self.assertEqual(result['id'], EXAMPLE_SERVER+path) - self.assertEqual(result['totalItems'], expectedTotalItems) - self.assertEqual(result['type'], 'OrderedCollection') - - def check_collection_page(self, - path, - page_number, - expectedTotalItems, - expectedOnPage, - ): - - def full_path(page): - if page is None: - query = '' - else: - query = '?page={}'.format(page) - - return EXAMPLE_SERVER + path + query - - c = Client( - HTTP_ACCEPT = JSON_TYPE, - ) - - response = c.get(full_path(page_number)) - - if response['Content-Type']=='text/html': - # let's just give up here so they have a chance - # of figuring out the error message from the server. - raise RuntimeError(response.content) - - self.assertEqual(response['Content-Type'], JSON_TYPE) - - result = json.loads(response.content.decode(encoding='UTF-8')) - - for field in [ - '@context', - 'id', - 'totalItems', - 'type', - 'partOf', - ]: - self.assertIn(field, result) - - self.assertEqual(result['id'], full_path(page_number)) - self.assertEqual(result['totalItems'], expectedTotalItems) - self.assertEqual(result['type'], 'OrderedCollectionPage') - self.assertEqual(result['partOf'], full_path(None)) - self.assertEqual(len(expectedOnPage), len(result['orderedItems'])) - - for actual, expected in zip(result['orderedItems'], expectedOnPage): - if type(expected)==dict: - self.assertDictContainsSubset(expected, actual) - else: - self.assertEqual(expected, actual) - - if page_number!=1: - self.assertIn('prev', result) - self.assertEqual(result['prev'], full_path(page_number-1)) - else: - self.assertNotIn('prev', result) - - if (page_number-1)