kopia lustrzana https://gitlab.com/marnanel/chapeau
Partial checkin. Still busy on this.
Marnanel Thurman
rodzic
48f76e7c5c
commit
171d9b51d3
37
TODO.md
37
TODO.md
|
|
@ -4,3 +4,40 @@
|
|||
2) Can I easily create an Activity for creating a local Article?
|
||||
(Bear in mind that this instruction might have come in via the
|
||||
user's outbox, rather than locally.)
|
||||
|
||||
------
|
||||
|
||||
New challenge:
|
||||
Auto-send Accept responses for incoming follow requests.
|
||||
|
||||
------
|
||||
|
||||
Activities need to allow lists etc.
|
||||
Can't we just store a JSON blob or something?
|
||||
|
||||
------
|
||||
|
||||
Delivery needs to be implemented.
|
||||
What's the longest chain of lookups?
|
||||
|
||||
We'll probably need a Delivery model.
|
||||
|
||||
XXX This should all be done through Celery.
|
||||
How good is Celery at task resumption even after a reboot?
|
||||
|
||||
------
|
||||
|
||||
Let's look at validation, done using Celery throughout.
|
||||
|
||||
We create an IncomingMessage, and save it, always.
|
||||
Then we ask Celery to do the rest: validate_task.
|
||||
|
||||
One single function.
|
||||
Based on CachedText; this does the fetch etc.
|
||||
|
||||
------
|
||||
|
||||
Is there something weird going on with our webfinger implementation?
|
||||
|
||||
-------
|
||||
|
||||
|
|
|
|||
|
|
@ -50,35 +50,6 @@ def implements_activity_type(f_type):
|
|||
return cls
|
||||
return register
|
||||
|
||||
def find(identifier, f_type=None):
|
||||
|
||||
if f_type is None:
|
||||
f_type = object_type_registry.keys()
|
||||
elif not isinstance(f_type, list):
|
||||
f_type = [f_type]
|
||||
|
||||
logger.debug('Here we go: %s %s', identifier, f_type)
|
||||
for t in f_type:
|
||||
|
||||
logger.debug('How about %s? %s', t, str(object_type_registry[t]))
|
||||
if t not in object_type_registry:
|
||||
continue
|
||||
|
||||
for cls in object_type_registry[t]:
|
||||
logging.info('finding %s for %s', identifier, str(cls))
|
||||
try:
|
||||
result = cls.activity_find(url=identifier)
|
||||
except cls.DoesNotExist:
|
||||
result = None
|
||||
|
||||
if result is not None:
|
||||
logger.debug('find: %s(%s)==%s', identifier, str(f_type),
|
||||
result)
|
||||
return result
|
||||
|
||||
logger.debug('find: %s(%s) was not found', identifier, str(f_type))
|
||||
return None
|
||||
|
||||
def create(fields):
|
||||
|
||||
if 'type' not in fields:
|
||||
|
|
|
|||
|
|
@ -0,0 +1,171 @@
|
|||
from django.db import models
|
||||
import requests
|
||||
import logging
|
||||
from django.conf import settings
|
||||
import django.urls
|
||||
from urllib.parse import urlparse
|
||||
from django.http.request import HttpRequest
|
||||
|
||||
logger = logging.getLogger(name='django_kepi')
|
||||
|
||||
class CachedRemoteText(models.Model):
|
||||
|
||||
address = models.URLField(
|
||||
primary_key = True,
|
||||
)
|
||||
|
||||
content = models.TextField(
|
||||
default = None,
|
||||
null = True,
|
||||
)
|
||||
# XXX We should probably also have a cache timeout
|
||||
|
||||
def is_gone(self):
|
||||
return self.content is None
|
||||
|
||||
def __str__(self):
|
||||
if self.key is None:
|
||||
return '(%s: "%s")' % (self.owner, self.content[:20])
|
||||
else:
|
||||
return '(%s is GONE)' % (self.owner)
|
||||
|
||||
@classmethod
|
||||
def fetch(cls,
|
||||
fetch_url,
|
||||
post_data):
|
||||
"""
|
||||
Fetch a file over HTTPS (and other protocols).
|
||||
This function blocks; don't call it while
|
||||
serving a request.
|
||||
|
||||
fetch_url: the URL of the file you want.
|
||||
FIXME: What happens if fetch_url is local?
|
||||
|
||||
post_data: If this is a dict, then the request
|
||||
will be a POST, with the contents of
|
||||
that dict as parameters to the remote server.
|
||||
If this is None, then the request will
|
||||
be a GET.
|
||||
|
||||
Returns: None, if post_data was a dict.
|
||||
If post_data was None, returns a CachedRemoteText.
|
||||
If fetch_url existed in the cache, this will be the cached
|
||||
record; otherwise it will be a new record, which
|
||||
has already been saved.
|
||||
|
||||
If the request was not successful, the is_gone()
|
||||
method of the returned CachedRemoteText will return True.
|
||||
All error codes, including notably 404, 410, and 500,
|
||||
are handled alike. (Is there any reason not to do this?)
|
||||
|
||||
FIXME: What does it do if the request returned a redirect?
|
||||
|
||||
"""
|
||||
|
||||
if post_data is None:
|
||||
|
||||
# This is a GET, so the answer might be cached.
|
||||
# (FIXME: honour HTTP caching headers etc)
|
||||
existing = cls.objects.find(address==fetch_url)
|
||||
if existing is not None:
|
||||
logger.info('fetch %s: in cache', fetch_url)
|
||||
return existing
|
||||
|
||||
logger.info('fetch %s: GET', fetch_url)
|
||||
|
||||
fetch = requests.get(fetch_url)
|
||||
else:
|
||||
logger.info('fetch %s: POST', fetch_url)
|
||||
logger.debug('fetch %s: with data: %s',
|
||||
fetch_url, post_data)
|
||||
|
||||
fetch = requests.post(fetch_url,
|
||||
data=post_data)
|
||||
|
||||
logger.info('fetch %s: response code was %d',
|
||||
fetch_url, fetch.status_code, fetch.text)
|
||||
logger.debug('fetch %s: body was %d',
|
||||
fetch_url, fetch.text)
|
||||
|
||||
result = None
|
||||
|
||||
if post_data is None:
|
||||
# This was a GET, so cache it
|
||||
# (FIXME: honour HTTP caching headers etc)
|
||||
# XXX: race condition: catch duplicate entry exception and ignore
|
||||
|
||||
if fetch.status_code==200:
|
||||
content = fetch.text
|
||||
else:
|
||||
content = None
|
||||
|
||||
result = cls(
|
||||
address = fetch_url,
|
||||
content = content,
|
||||
)
|
||||
result.save()
|
||||
|
||||
return result
|
||||
|
||||
def _obviously_belongs_to(self, actor):
|
||||
return self.address.startswith(actor+'#')
|
||||
|
||||
class ActivityRequest(HttpRequest):
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
|
||||
self.method = 'ACTIVITY'
|
||||
|
||||
def find_local(path):
|
||||
logger.debug('%s: find local', path)
|
||||
|
||||
try:
|
||||
resolved = django.urls.resolve(path)
|
||||
except django.urls.Resolver404:
|
||||
logger.debug('%s: -- not found', path)
|
||||
return None
|
||||
|
||||
logger.debug('%s: handled by %s', path, str(resolved.func))
|
||||
logger.debug('%s: %s', path, str(resolved.args))
|
||||
logger.debug('%s: %s', path, str(resolved.kwargs))
|
||||
|
||||
request = ActivityRequest()
|
||||
result = resolved.func(request,
|
||||
**resolved.kwargs)
|
||||
logger.debug('%s: resulting in %s', path, str(result))
|
||||
|
||||
return None # XXX
|
||||
|
||||
def find_remote(url):
|
||||
logger.debug('%s: find remote', url)
|
||||
return None # XXX
|
||||
|
||||
def find(url):
|
||||
"""
|
||||
Finds an object.
|
||||
|
||||
address: the URL of the object.
|
||||
|
||||
If the address is local, we look the object up using
|
||||
Django's usual dispatcher.
|
||||
|
||||
Otherwise, we check the cache. If the JSON source of the
|
||||
object is in the cache, we parse it and return it.
|
||||
|
||||
Otherwise, we dereference the URL.
|
||||
|
||||
The result can be None, if the object doesn't exist locally
|
||||
or isn't found remotely, and for remote objects this fact
|
||||
may be cached.
|
||||
|
||||
Results other than None are guaranteed to be subscriptable.
|
||||
"""
|
||||
|
||||
parsed_url = urlparse(url)
|
||||
is_local = parsed_url.hostname in settings.ALLOWED_HOSTS
|
||||
|
||||
if is_local:
|
||||
return find_local(parsed_url.path)
|
||||
else:
|
||||
return find_remote(url)
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
# Generated by Django 2.1.5 on 2019-04-28 10:34
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('django_kepi', '0012_activitymodel'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.CreateModel(
|
||||
name='CachedRemoteText',
|
||||
fields=[
|
||||
('address', models.URLField(primary_key=True, serialize=False)),
|
||||
('content', models.TextField(default=None, null=True)),
|
||||
],
|
||||
),
|
||||
migrations.DeleteModel(
|
||||
name='CachedRemoteUser',
|
||||
),
|
||||
]
|
||||
|
|
@ -1,26 +1,10 @@
|
|||
from django.db import models
|
||||
from django_kepi import object_type_registry, find, register_type
|
||||
from django.contrib.contenttypes.fields import GenericForeignKey
|
||||
from django.contrib.contenttypes.models import ContentType
|
||||
from django.conf import settings
|
||||
import django_kepi.tasks
|
||||
import logging
|
||||
import random
|
||||
import json
|
||||
import datetime
|
||||
import warnings
|
||||
import uuid
|
||||
|
||||
from django_kepi.activity_model import *
|
||||
from django_kepi.cache_model import *
|
||||
from django_kepi.something_model import *
|
||||
from django_kepi.validation import *
|
||||
from django_kepi.activity_model import Activity, new_activity_identifier
|
||||
from django_kepi.something_model import ActivityModel
|
||||
|
||||
#######################
|
||||
|
||||
__all__ = [
|
||||
'Activity',
|
||||
'Cache',
|
||||
'Person',
|
||||
'new_activity_identifier',
|
||||
'ActivityModel',
|
||||
]
|
||||
|
|
|
|||
|
|
@ -1,42 +1,79 @@
|
|||
from __future__ import absolute_import, unicode_literals
|
||||
from celery import shared_task
|
||||
import requests
|
||||
from django_kepi.validation import IncomingMessage
|
||||
from django_kepi import find
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(name='django_kepi')
|
||||
|
||||
@shared_task()
|
||||
def fetch(
|
||||
fetch_url,
|
||||
post_data,
|
||||
result_url,
|
||||
def validate(
|
||||
message_id,
|
||||
):
|
||||
logger.info('%s: begin validation',
|
||||
message_id)
|
||||
|
||||
if post_data is None:
|
||||
logger.info('batch %s: GET', fetch_url)
|
||||
message = IncomingMessage.objects.find(id=message_id)
|
||||
|
||||
fetch = requests.get(fetch_url)
|
||||
actor = message.actor
|
||||
key_id = message.key_id
|
||||
|
||||
logger.debug('%s: message signature is: %s',
|
||||
message, message.signature)
|
||||
logger.debug('%s: message body is: %s',
|
||||
message, message.body)
|
||||
|
||||
if _is_local_user(actor):
|
||||
logger.debug('%s: actor %s is local', message, actor)
|
||||
|
||||
local_user = find(actor, 'Actor')
|
||||
|
||||
if local_user is None:
|
||||
logger.info('%s: local actor %s does not exist; dropping message',
|
||||
message, actor)
|
||||
return
|
||||
|
||||
key = local_user.key
|
||||
_do_validation(message, key)
|
||||
return
|
||||
|
||||
if not _obviously_belongs_to(actor, key_id):
|
||||
logger.info('%s: key_id %s is not obviously owned by '+\
|
||||
'actor %s; dropping message',
|
||||
message, key_id, actor)
|
||||
return
|
||||
|
||||
try:
|
||||
remote_key = CachedRemoteUser.objects.get(owner=actor)
|
||||
except CachedRemoteUser.DoesNotExist:
|
||||
remote_key = None
|
||||
|
||||
if remote_key is not None:
|
||||
|
||||
if remote_key.is_gone():
|
||||
# XXX This should probably trigger a clean-out of everything
|
||||
# we know about that user
|
||||
logger.info('%s: remote actor %s is gone; dropping message',
|
||||
actor, message)
|
||||
return
|
||||
|
||||
logger.debug('%s: we have the remote key', message)
|
||||
_do_validation(message, remote_key.key)
|
||||
return
|
||||
|
||||
logger.debug('%s: we don\'t have the key', message)
|
||||
|
||||
if second_pass:
|
||||
logger.warning('%s: we apparently both do and don\'t have the key',
|
||||
message)
|
||||
return
|
||||
|
||||
message.waiting_for = actor
|
||||
message.save()
|
||||
|
||||
if len(IncomingMessage.objects.filter(waiting_for=actor))==1:
|
||||
logger.debug('%s: starting background task', message)
|
||||
_kick_off_background_fetch(actor)
|
||||
else:
|
||||
logger.info('batch %s: POST with form data: %s',
|
||||
fetch_url, post_data)
|
||||
logger.debug('%s: not starting background task', message)
|
||||
|
||||
fetch = requests.post(fetch_url,
|
||||
data=post_data)
|
||||
|
||||
logger.info('batch %s: response code was %d, body was %s',
|
||||
fetch_url, fetch.status_code, fetch.text)
|
||||
|
||||
if result_url is not None:
|
||||
|
||||
response = requests.post(
|
||||
result_url,
|
||||
params={
|
||||
'code': int(fetch.status_code),
|
||||
'url': fetch_url,
|
||||
},
|
||||
data=fetch.text,
|
||||
)
|
||||
|
||||
if response.status_code!=200:
|
||||
logger.warn('batch %s: notifying %s FAILED with code %d',
|
||||
fetch_url, result_url, response.status_code)
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ from django.conf import settings
|
|||
from urllib.parse import urlparse
|
||||
from django_kepi import find
|
||||
from httpsig.verify import HeaderVerifier
|
||||
import django_kepi.tasks
|
||||
|
||||
logger = logging.getLogger(name='django_kepi')
|
||||
|
||||
|
|
@ -61,31 +62,6 @@ logger = logging.getLogger(name='django_kepi')
|
|||
# If there's been fewer than "n" tries, recreate the background task.
|
||||
# Otherwise, report the error and drop the message.
|
||||
|
||||
class CachedRemoteUser(models.Model):
|
||||
|
||||
owner = models.URLField(
|
||||
primary_key = True,
|
||||
)
|
||||
|
||||
key = models.TextField(
|
||||
default = None,
|
||||
null = True,
|
||||
)
|
||||
|
||||
inbox = models.URLField()
|
||||
outbox = models.URLField()
|
||||
|
||||
# XXX We should probably also have a cache timeout
|
||||
|
||||
def is_gone(self):
|
||||
return self.key is None
|
||||
|
||||
def __str__(self):
|
||||
if self.key is None:
|
||||
return '(%s: public key)' % (self.owner)
|
||||
else:
|
||||
return '(%s is GONE)' % (self.owner)
|
||||
|
||||
class IncomingMessage(models.Model):
|
||||
|
||||
id = models.UUIDField(
|
||||
|
|
@ -123,16 +99,12 @@ class IncomingMessage(models.Model):
|
|||
def fields(self):
|
||||
return json.loads(self.body)
|
||||
|
||||
def validate(self):
|
||||
tasks.validate(self)
|
||||
|
||||
def is_local_user(url):
|
||||
return urlparse(url).hostname in settings.ALLOWED_HOSTS
|
||||
|
||||
def _obviously_belongs_to(actor, key_id):
|
||||
return key_id.startswith(actor+'#')
|
||||
|
||||
def _kick_off_background_fetch(url):
|
||||
# XXX actually do it
|
||||
pass
|
||||
|
||||
def _do_validation(message, key):
|
||||
logger.debug('%s: running actual validation', message)
|
||||
fields = message.fields
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ from django.http import HttpResponse, JsonResponse, Http404
|
|||
from django.contrib.auth.decorators import login_required
|
||||
from django.core.exceptions import ValidationError
|
||||
from django.conf import settings
|
||||
from django_kepi.models import Activity
|
||||
import logging
|
||||
import urllib.parse
|
||||
import json
|
||||
|
|
@ -18,23 +19,35 @@ logger = logging.getLogger(name='django_kepi')
|
|||
PAGE_LENGTH = 50
|
||||
PAGE_FIELD = 'page'
|
||||
|
||||
class ActivityObjectView(django.views.View):
|
||||
class KepiView(django.views.View):
|
||||
|
||||
def get(self, request, *args, **kwargs):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
|
||||
url = 'https://' + \
|
||||
settings.KEPI['LOCAL_OBJECT_HOSTNAME'] + \
|
||||
request.path
|
||||
f_type = kwargs.get('f_type', None)
|
||||
self.http_method_names.append('activity')
|
||||
|
||||
activity_object = find(url, f_type)
|
||||
class ActivityObjectView(KepiView):
|
||||
|
||||
def activity(self, request, *args, **kwargs):
|
||||
|
||||
url = settings.KEPI['ACTIVITY_URL_FORMAT'] % (kwargs['id'],)
|
||||
logger.debug('url:%s', url)
|
||||
|
||||
activity_object = Activity.objects.get(
|
||||
identifier=url,
|
||||
)
|
||||
|
||||
if activity_object is None:
|
||||
logger.info('%s(%s) is unknown', url, f_type)
|
||||
logger.info('%s: unknown', url, f_type)
|
||||
raise Http404('Unknown object')
|
||||
|
||||
result = activity_object.activity_form(*args, **kwargs)
|
||||
|
||||
return result
|
||||
|
||||
def get(self, request, *args, **kwargs):
|
||||
|
||||
result = self.activity(request, *args, **kwargs)
|
||||
return self._render(result)
|
||||
|
||||
def _make_query_page(
|
||||
|
|
|
|||
|
|
@ -3,7 +3,6 @@ from .celery import app as celery_app
|
|||
from django.test import TestCase
|
||||
import httpretty
|
||||
import logging
|
||||
from django_kepi.tasks import fetch
|
||||
|
||||
logger = logging.getLogger(name='things_for_testing')
|
||||
|
||||
|
|
|
|||
Ładowanie…
Reference in New Issue