From dc8edbc7e682f1f9ebad5715526b4700d34906d8 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Sun, 17 Mar 2019 19:39:55 +0200 Subject: [PATCH] Refactor handle_send function recipients and delivery code The outbound function `outbound.handle_send` parameter `recipients` structure has changed. It must now be a list of dictionaries, containing at minimum the following: `fid` for the recipient endpoint, `protocol` for the protocol to use and `public` as a boolean whether the payload should be treated as visible to anyone. For Diaspora private deliveries, also a `public_key` is required containing the receiver public key. Note that passing in handles as recipients is not any more possible - always pass in a url for `fid`. --- CHANGELOG.md | 4 +- federation/outbound.py | 148 +++++++++++++++++------------- federation/tests/test_outbound.py | 63 +++++++++---- setup.py | 1 + 4 files changed, 132 insertions(+), 84 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d2f27fe..9e5bc94 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,7 +36,9 @@ * NodeInfo2 parser now returns the admin user in `handle` format instead of a Diaspora format URL. * The high level inbound and outbound functions `inbound.handle_receive`, `outbound.handle_send` parameter `user` must now receive a `UserType` compatible object. This must have the attributes `id` and `private_key`. If Diaspora support is required then also `handle` and `guid` should exist. The type can be found as a class in `types.UserType`. * The high level inbound function `inbound.handle_receive` first parameter has been changed to `request` which must be a `RequestType` compatible object. This must have the attribute `body` which corrresponds to the old `payload` parameter. For ActivityPub inbound requests the object must also contain `headers`, `method` and `url`. - * The outbound function `outbound.handle_send` parameter `recipients` structure has changed. It must now for Diaspora contain either a `handle` (public delivery) or tuple of `handle, RSAPublicKey, guid` for private delivery. For AP delivery either `url ID` for public delivery or tuple of `url ID, RSAPublicKey` for private delivery. + * The outbound function `outbound.handle_send` parameter `recipients` structure has changed. It must now be a list of dictionaries, containing at minimum the following: `fid` for the recipient endpoint, `protocol` for the protocol to use and `public` as a boolean whether the payload should be treated as visible to anyone. + + For Diaspora private deliveries, also a `public_key` is required containing the receiver public key. Note that passing in handles as recipients is not any more possible - always pass in a url for `fid`. * The outbound function `outbound.handle_create_payload` now requires an extra third parameter for the protocol to use. This function should rarely need to be called directly - use `handle_send` instead which can handle both ActivityPub and Diaspora protocols. * **Backwards incompatible.** Generator `RFC3033Webfinger` and the related `rfc3033_webfinger_view` have been renamed to `RFC7033Webfinger` and `rfc7033_webfinger_view` to reflect the right RFC number. diff --git a/federation/outbound.py b/federation/outbound.py index d60517f..c02f4a2 100644 --- a/federation/outbound.py +++ b/federation/outbound.py @@ -1,16 +1,15 @@ import importlib import json import logging -from typing import List, Tuple, Union +from typing import List, Dict from Crypto.PublicKey.RSA import RsaKey +from iteration_utilities import unique_everseen from federation.entities.mixins import BaseEntity from federation.protocols.activitypub.signing import get_http_authentication from federation.types import UserType -from federation.utils.diaspora import get_public_endpoint, get_private_endpoint from federation.utils.network import send_document -from federation.utils.protocols import identify_recipient_protocol logger = logging.getLogger("federation") @@ -49,32 +48,56 @@ def handle_create_payload( def handle_send( entity: BaseEntity, author_user: UserType, - recipients: List[Union[Tuple[str, RsaKey], Tuple[str, RsaKey, str], str]] = None, + recipients: List[Dict], parent_user: UserType = None, ) -> None: """Send an entity to remote servers. - Using this we will build a list of payloads per protocol, after resolving any that need to be guessed or - looked up over the network. After that, each recipient will get the generated protocol payload delivered. + Using this we will build a list of payloads per protocol. After that, each recipient will get the generated + protocol payload delivered. Delivery to the same endpoint will only be done once so it's ok to include + the same endpoint as a receiver multiple times. - Any given user arguments must have ``private_key`` and ``id`` attributes. + Any given user arguments must have ``private_key`` and ``fid`` attributes. :arg entity: Entity object to send. Can be a base entity or a protocol specific one. :arg author_user: User authoring the object. - :arg recipients: A list of recipients to delivery to. Each recipient is a tuple - containing at minimum the "id". - For private deliveries, optionally "public key" (all protocols) and "guid" (diaspora - protocol) are required. - Instead of a tuple, for public deliveries the "id" as str is also ok. - If public key and guid are provided, Diaspora protocol delivery will be made as an encrypted - private delivery. + :arg recipients: A list of recipients to delivery to. Each recipient is a dict + containing at minimum the "fid", "public" and "protocol" keys. + + For ActivityPub and Diaspora payloads, "fid" should be an URL of the endpoint to deliver to. + + The "protocol" should be a protocol name that is known for this recipient. + + The "public" value should be a boolean to indicate whether the payload should be flagged as a + public payload. + + TODO: support guessing the protocol over networks? Would need caching of results + + For private deliveries to Diaspora protocol recipients, "public_key" is also required. + For example [ - ("user@domain.tld", , '1234-5678-0123-4567'), - ("user@domain2.tld", None, None), - "user@domain3.tld", - "https://domain4.tld/sharedinbox/", - ("https://domain4.tld/sharedinbox/", ), + { + "fid": "https://domain.tld/receive/users/1234-5678-0123-4567", + "protocol": "diaspora", + "public": False, + "public_key": , + }, + { + "fid": "https://domain2.tld/receive/public", + "protocol": "diaspora", + "public": True, + }, + { + "fid": "https://domain4.tld/sharedinbox/", + "protocol": "activitypub", + "public": True, + }, + { + "fid": "https://domain4.tld/profiles/jill", + "protocol": "activitypub", + "public": False, + }, ] :arg parent_user: (Optional) User object of the parent object, if there is one. This must be given for the Diaspora protocol if a parent object exists, so that a proper ``parent_author_signature`` can @@ -94,63 +117,60 @@ def handle_send( }, } + # Flatten to unique recipients + unique_recipients = unique_everseen(recipients) + # Generate payloads and collect urls - for recipient in recipients: - id = recipient[0] if isinstance(recipient, tuple) else recipient - public_key = recipient[1] if isinstance(recipient, tuple) and len(recipient) > 1 else None - recipient_protocol = identify_recipient_protocol(id) - # TODO for now send all AP payloads as "private" ie one per url - if public_key or recipient_protocol == "activitypub": - # Private payload - if recipient_protocol == 'activitypub': - try: - payload = handle_create_payload( - entity, author_user, "activitypub", to_user_key=public_key, parent_user=parent_user, + for recipient in unique_recipients: + fid = recipient["fid"] + public_key = recipient.get("public_key") + protocol = recipient["protocol"] + public = recipient["public"] + + if protocol == "activitypub": + try: + payload = handle_create_payload(entity, author_user, protocol, parent_user=parent_user) + if public: + payload["to"] = "https://www.w3.org/ns/activitystreams#Public" + else: + payload["to"] = fid + payload = json.dumps(payload).encode("utf-8") + except Exception as ex: + logger.error("handle_send - failed to generate private payload for %s: %s", fid, ex) + continue + payloads.append({ + "auth": get_http_authentication(author_user.private_key, f"{author_user.id}#main-key"), + "payload": payload, + "content_type": 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"', + "urls": {fid}, + }) + elif protocol == "diaspora": + if public: + if public_key: + raise ValueError("handle_send - Diaspora recipient cannot be public and use encrypted delivery") + if not public_payloads[protocol]["payload"]: + public_payloads[protocol]["payload"] = handle_create_payload( + entity, author_user, protocol, parent_user=parent_user, ) - payload["to"] = id - payload = json.dumps(payload).encode("utf-8") - except Exception as ex: - logger.error("handle_send - failed to generate private payload for %s: %s", id, ex) - continue - payloads.append({ - "auth": get_http_authentication(author_user.private_key, f"{author_user.id}#main-key"), - "payload": payload, - "content_type": 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"', - "urls": {id}, - }) - elif recipient_protocol == 'diaspora': + public_payloads["diaspora"]["urls"].add(fid) + else: + if not public_key: + raise ValueError("handle_send - Diaspora recipient cannot be private without a public key for " + "encrypted delivery") + # Private payload try: payload = handle_create_payload( entity, author_user, "diaspora", to_user_key=public_key, parent_user=parent_user, ) payload = json.dumps(payload) except Exception as ex: - logger.error("handle_send - failed to generate private payload for %s: %s", id, ex) + logger.error("handle_send - failed to generate private payload for %s: %s", fid, ex) continue - guid = recipient[2] if len(recipient) > 2 else None - url = get_private_endpoint(id, guid=guid) payloads.append({ - "urls": {url}, "payload": payload, "content_type": "application/json", "auth": None, + "urls": {fid}, "payload": payload, "content_type": "application/json", "auth": None, }) - else: - if not public_payloads[recipient_protocol]["payload"]: - public_payloads[recipient_protocol]["payload"] = handle_create_payload( - entity, author_user, recipient_protocol, parent_user=parent_user, - ) - if recipient_protocol == 'activitypub': - public_payloads["activitypub"]["urls"].add(id) - elif recipient_protocol == 'diaspora': - url = get_public_endpoint(id) - public_payloads["diaspora"]["urls"].add(url) - # Add public payload - if public_payloads["activitypub"]["payload"]: - payloads.append({ - "auth": get_http_authentication(author_user.private_key, f"{author_user.id}#main-key"), - "urls": public_payloads["activitypub"]["urls"], - "payload": public_payloads["activitypub"]["payload"], - "content_type": 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"', - }) + # Add public diaspora payload if public_payloads["diaspora"]["payload"]: payloads.append({ "urls": public_payloads["diaspora"]["urls"], "payload": public_payloads["diaspora"]["payload"], diff --git a/federation/tests/test_outbound.py b/federation/tests/test_outbound.py index 329a7b0..d79867b 100644 --- a/federation/tests/test_outbound.py +++ b/federation/tests/test_outbound.py @@ -1,8 +1,11 @@ from unittest.mock import Mock, patch +import pytest + from federation.entities.diaspora.entities import DiasporaPost from federation.outbound import handle_create_payload, handle_send from federation.tests.fixtures.keys import get_dummy_private_key +from federation.utils.text import encode_if_text class TestHandleCreatePayloadBuildsAPayload: @@ -21,12 +24,26 @@ class TestHandleSend: def test_calls_handle_create_payload(self, mock_send, profile): key = get_dummy_private_key() recipients = [ - ("foo@127.0.0.1", key.publickey(), "xyz"), - ("https://127.0.0.1/foobar", key.publickey()), - ("foo@example.com", None), - "foo@example.net", - "qwer@example.net", # Same host twice to ensure one delivery only per host for public payloads - "https://example.net/foobar", # On the same host there is an AP actor + { + "fid": "https://127.0.0.1/receive/users/1234", "public_key": key.publickey(), "public": False, + "protocol": "diaspora", + }, + { + "fid": "https://example.com/receive/public", "public": True, "protocol": "diaspora", + }, + { + "fid": "https://example.net/receive/public", "public": True, "protocol": "diaspora", + }, + # Same twice to ensure one delivery only per unique + { + "fid": "https://example.net/receive/public", "public": True, "protocol": "diaspora", + }, + { + "fid": "https://example.net/foobar", "public": False, "protocol": "activitypub", + }, + { + "fid": "https://example.net/inbox", "public": True, "protocol": "activitypub", + } ] mock_author = Mock( private_key=key, id="foo@example.com", handle="foo@example.com", @@ -35,32 +52,40 @@ class TestHandleSend: # Ensure first call is a private diaspora payload args, kwargs = mock_send.call_args_list[0] - assert args[0] == "https://127.0.0.1/receive/users/xyz" + assert args[0] == "https://127.0.0.1/receive/users/1234" assert "aes_key" in args[1] assert "encrypted_magic_envelope" in args[1] assert kwargs['headers'] == {'Content-Type': 'application/json'} # Ensure second call is a private activitypub payload args, kwargs = mock_send.call_args_list[1] - assert args[0] == "https://127.0.0.1/foobar" + assert args[0] == "https://example.net/foobar" assert kwargs['headers'] == { 'Content-Type': 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"', } + assert encode_if_text("https://www.w3.org/ns/activitystreams#Public") not in args[1] - # Ensure public payloads and recipients, one per unique host - args1, kwargs1 = mock_send.call_args_list[2] - args2, kwargs2 = mock_send.call_args_list[3] - args3, kwargs3 = mock_send.call_args_list[4] - public_endpoints = {args1[0], args2[0], args3[0]} + # Ensure third call is a public activitypub payload + args, kwargs = mock_send.call_args_list[2] + assert args[0] == "https://example.net/inbox" + assert kwargs['headers'] == { + 'Content-Type': 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"', + } + assert encode_if_text("https://www.w3.org/ns/activitystreams#Public") in args[1] + + # Ensure diaspora public payloads and recipients, one per unique host + args3, kwargs3 = mock_send.call_args_list[3] + args4, kwargs4 = mock_send.call_args_list[4] + public_endpoints = {args3[0], args4[0]} assert public_endpoints == { "https://example.net/receive/public", "https://example.com/receive/public", - "https://example.net/foobar", } - assert args2[1].startswith("=0.3.0", "lxml>=3.4.0", "ipdata>=2.6", + "iteration_utilities", "jsonschema>=2.0.0", "pycryptodome>=3.4.10", "python-dateutil>=2.4.0",