diff --git a/CHANGELOG.md b/CHANGELOG.md index 61e86e9..b789df7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,10 +2,27 @@ ## [unreleased] +### Added + +* Enable generating encrypted JSON payloads with the Diaspora protocol which adds private message support. ([related issue](https://github.com/jaywink/federation/issues/82)) + + JSON encrypted payload encryption and decryption is handled by the Diaspora `EncryptedPayload` class. + ### Changed -* Send outbound Diaspora payloads in new format. Remove possibility to generate legacy MagicEnvelope payloads. ([related issue](https://github.com/jaywink/federation/issues/82)) +* Send outbound Diaspora payloads in new format. Remove possibility to generate legacy MagicEnvelope payloads. ([related issue](https://github.com/jaywink/federation/issues/82)) +* **Backwards incompatible**. Refactor `handle_send` function + + Now handle_send high level outbound helper function also allows delivering private payloads using the Diaspora protocol. ([related issue](https://github.com/jaywink/federation/issues/82)) + + The signature has changed. Parameter `recipients` should now be a list of recipients to delivery to. Each recipient should either be an `id` or a tuple of `(id, public key)`. If public key is provided, Diaspora protocol delivery will be made as an encrypted private delivery. + +* **Backwards incompatible**. Change `handle_create_payload` function signature. + + Parameter `to_user` is now `to_user_key` and thus instead of an object containing the `key` attribute it should now be an RSA public key object instance. This simplifies things since we only need the key from the user, nothing else. + + ## [0.15.0] - 2018-02-12 ### Added diff --git a/docs/usage.rst b/docs/usage.rst index d599c11..415cf1f 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -109,6 +109,7 @@ Diaspora .. autofunction:: federation.utils.diaspora.fetch_public_key .. autofunction:: federation.utils.diaspora.get_fetch_content_endpoint .. autofunction:: federation.utils.diaspora.get_public_endpoint +.. autofunction:: federation.utils.diaspora.get_private_endpoint .. autofunction:: federation.utils.diaspora.parse_diaspora_uri .. autofunction:: federation.utils.diaspora.parse_profile_from_hcard .. autofunction:: federation.utils.diaspora.retrieve_and_parse_content diff --git a/federation/outbound.py b/federation/outbound.py index 8eb1ac6..eb65fa7 100644 --- a/federation/outbound.py +++ b/federation/outbound.py @@ -1,17 +1,17 @@ from federation.entities.diaspora.mappers import get_outbound_entity from federation.protocols.diaspora.protocol import Protocol -from federation.utils.diaspora import get_public_endpoint +from federation.utils.diaspora import get_public_endpoint, get_private_endpoint from federation.utils.network import send_document -def handle_create_payload(entity, author_user, to_user=None, parent_user=None): +def handle_create_payload(entity, author_user, to_user_key=None, parent_user=None): """Create a payload with the correct protocol. Any given user arguments must have ``private_key`` and ``handle`` attributes. :arg entity: Entity object to send. Can be a base entity or a protocol specific one. :arg author_user: User authoring the object. - :arg to_user: Profile entry to send to (required for non-public content) + :arg to_user_key: Public key of user private payload is being sent to, required for private payloads. :arg parent_user: (Optional) User object of the parent object, if there is one. This must be given for the Diaspora protocol if a parent object exists, so that a proper ``parent_author_signature`` can be generated. If given, the payload will be sent as this user. @@ -23,7 +23,7 @@ def handle_create_payload(entity, author_user, to_user=None, parent_user=None): if parent_user: outbound_entity.sign_with_parent(parent_user.private_key) send_as_user = parent_user if parent_user else author_user - data = protocol.build_send(entity=outbound_entity, from_user=send_as_user, to_user=to_user) + data = protocol.build_send(entity=outbound_entity, from_user=send_as_user, to_user_key=to_user_key) return data @@ -33,32 +33,62 @@ def handle_send(entity, author_user, recipients=None, parent_user=None): Using this we will build a list of payloads per protocol, after resolving any that need to be guessed or looked up over the network. After that, each recipient will get the generated protocol payload delivered. - NOTE! This will not (yet) support Diaspora limited messages - `handle_create_payload` above should be directly - called instead and payload sent with `federation.utils.network.send_document`. - Any given user arguments must have ``private_key`` and ``handle`` attributes. :arg entity: Entity object to send. Can be a base entity or a protocol specific one. :arg author_user: User authoring the object. - :arg recipients: A list of tuples to delivery to. Tuple contains (recipient handle or domain, protocol or None). - For example ``[("foo@example.com", "diaspora"), ("bar@example.com", None)]``. + :arg recipients: A list of recipients to delivery to. Each recipient is a tuple + containing at minimum the "id", optionally "public key" for private deliveries. + Instead of a tuple, for public deliveries the "id" as str is also ok. + If public key is provided, Diaspora protocol delivery will be made as an encrypted + private delivery. + For example + [ + ("diaspora://user@domain.tld/profile/zyx", ), + ("diaspora://user@domain2.tld/profile/xyz", None), + "diaspora://user@domain3.tld/profile/xyz", + ] :arg parent_user: (Optional) User object of the parent object, if there is one. This must be given for the Diaspora protocol if a parent object exists, so that a proper ``parent_author_signature`` can be generated. If given, the payload will be sent as this user. """ - payloads = {"diaspora": {"payload": None, "recipients": set()}} - # Generate payload per protocol and split recipients to protocols - for recipient, protocol in recipients: - # TODO currently we only support Diaspora protocol, so no need to guess, just generate the payload - if not payloads["diaspora"]["payload"]: - payloads["diaspora"]["payload"] = handle_create_payload(entity, author_user, parent_user=parent_user) - if "@" in recipient: - payloads["diaspora"]["recipients"].add(recipient.split("@")[1]) + payloads = [] + public_payloads = { + "diaspora": { + "payload": None, + "urls": set(), + }, + } + + # Generate payloads and collect urls + for recipient in recipients: + id = recipient[0] if isinstance(recipient, tuple) else recipient + public_key = recipient[1] if isinstance(recipient, tuple) and len(recipient) > 1 else None + if public_key: + # Private payload + payload = handle_create_payload(entity, author_user, to_user_key=public_key, parent_user=parent_user) + # TODO get_private_endpoint should be imported per protocol + url = get_private_endpoint(id) + payloads.append({ + "urls": {url}, "payload": payload, + }) else: - payloads["diaspora"]["recipients"].add(recipient) + if not public_payloads["diaspora"]["payload"]: + public_payloads["diaspora"]["payload"] = handle_create_payload( + entity, author_user, parent_user=parent_user, + ) + # TODO get_public_endpoint should be imported per protocol + url = get_public_endpoint(id) + public_payloads["diaspora"]["urls"].add(url) + + # Add public payload + if public_payloads["diaspora"]["payload"]: + payloads.append({ + "urls": public_payloads["diaspora"]["urls"], "payload": public_payloads["diaspora"]["payload"], + }) + # Do actual sending - for protocol, data in payloads.items(): - for recipient in data.get("recipients"): - # TODO protocol independant url generation by importing named helper under protocol - url = get_public_endpoint(recipient) - send_document(url, data.get("payload")) + for payload in payloads: + for url in payload["urls"]: + # TODO set content type per protocol above when collecting and use here + send_document(url, payload["payload"]) diff --git a/federation/protocols/diaspora/encrypted.py b/federation/protocols/diaspora/encrypted.py index ccf23a4..b7f7102 100644 --- a/federation/protocols/diaspora/encrypted.py +++ b/federation/protocols/diaspora/encrypted.py @@ -1,12 +1,34 @@ import json -from base64 import b64decode +from base64 import b64decode, b64encode from Crypto.Cipher import PKCS1_v1_5, AES +from Crypto.Random import get_random_bytes from lxml import etree +def pkcs7_pad(inp, block_size): + """ + Using the PKCS#7 padding scheme, pad to be a multiple of + bytes. Ruby's AES encryption pads with this scheme, but + pycrypto doesn't support it. + + Implementation copied from pyaspora: + https://github.com/mjnovice/pyaspora/blob/master/pyaspora/diaspora/protocol.py#L209 + """ + val = block_size - len(inp) % block_size + if val == 0: + return inp + (bytes([block_size]) * block_size) + else: + return inp + (bytes([val]) * val) + + def pkcs7_unpad(data): - """Remove the padding bytes that were added at point of encryption.""" + """ + Remove the padding bytes that were added at point of encryption. + + Implementation copied from pyaspora: + https://github.com/mjnovice/pyaspora/blob/master/pyaspora/diaspora/protocol.py#L209 + """ if isinstance(data, str): return data[0:-ord(data[-1])] else: @@ -28,3 +50,39 @@ class EncryptedPayload: encrypter = AES.new(key, AES.MODE_CBC, iv) content = encrypter.decrypt(encrypted_magic_envelope) return etree.fromstring(pkcs7_unpad(content)) + + @staticmethod + def get_aes_key_json(iv, key): + return json.dumps({ + "key": b64encode(key).decode("ascii"), + "iv": b64encode(iv).decode("ascii"), + }).encode("utf-8") + + @staticmethod + def get_iv_key_encrypter(): + iv = get_random_bytes(AES.block_size) + key = get_random_bytes(32) + encrypter = AES.new(key, AES.MODE_CBC, iv) + return iv, key, encrypter + + @staticmethod + def encrypt(payload, public_key): + """ + Encrypt a payload using an encrypted JSON wrapper. + + See: https://diaspora.github.io/diaspora_federation/federation/encryption.html + + :param payload: Payload document as a string. + :param public_key: Public key of recipient as an RSA object. + :return: Encrypted JSON wrapper as dict. + """ + iv, key, encrypter = EncryptedPayload.get_iv_key_encrypter() + aes_key_json = EncryptedPayload.get_aes_key_json(iv, key) + cipher = PKCS1_v1_5.new(public_key) + aes_key = b64encode(cipher.encrypt(aes_key_json)) + padded_payload = pkcs7_pad(payload.encode("utf-8"), AES.block_size) + encrypted_me = b64encode(encrypter.encrypt(padded_payload)) + return { + "aes_key": aes_key, + "encrypted_magic_envelope": encrypted_me, + } diff --git a/federation/protocols/diaspora/protocol.py b/federation/protocols/diaspora/protocol.py index 8ddf79d..62d877d 100644 --- a/federation/protocols/diaspora/protocol.py +++ b/federation/protocols/diaspora/protocol.py @@ -229,13 +229,22 @@ class Protocol(BaseProtocol): else: return data[0:-data[-1]] - def build_send(self, entity, from_user, to_user=None, *args, **kwargs): - """Build POST data for sending out to remotes.""" + def build_send(self, entity, from_user, to_user_key=None, *args, **kwargs): + """ + Build POST data for sending out to remotes. + + :param entity: The outbound ready entity for this protocol. + :param from_user: The user sending this payload. Must have ``private_key`` and ``handle`` properties. + :param to_user_key: (Optional) Public key of user we're sending a private payload to. + :returns: dict or string depending on if private or public payload. + """ if entity.outbound_doc is not None: # Use pregenerated outbound document xml = entity.outbound_doc else: xml = entity.to_xml() me = MagicEnvelope(etree.tostring(xml), private_key=from_user.private_key, author_handle=from_user.handle) - # TODO wrap if doing encrypted delivery - return me.render() + rendered = me.render() + if to_user_key: + return EncryptedPayload.encrypt(rendered, to_user_key) + return rendered diff --git a/federation/tests/protocols/diaspora/test_encrypted.py b/federation/tests/protocols/diaspora/test_encrypted.py index 5d10988..3ec02d7 100644 --- a/federation/tests/protocols/diaspora/test_encrypted.py +++ b/federation/tests/protocols/diaspora/test_encrypted.py @@ -1,8 +1,10 @@ from unittest.mock import patch, Mock from Crypto.Cipher import AES +from lxml import etree from federation.protocols.diaspora.encrypted import pkcs7_unpad, EncryptedPayload +from federation.tests.fixtures.keys import get_dummy_private_key def test_pkcs7_unpad(): @@ -31,3 +33,13 @@ class TestEncryptedPayload: mock_encrypter.assert_called_once_with("magically encrypted") assert doc.tag == "foo" assert doc.text == "bar" + + def test_encrypt(self): + private_key = get_dummy_private_key() + public_key = private_key.publickey() + encrypted = EncryptedPayload.encrypt("eggs", public_key) + assert "aes_key" in encrypted + assert "encrypted_magic_envelope" in encrypted + # See we can decrypt it too + decrypted = EncryptedPayload.decrypt(encrypted, private_key) + assert etree.tostring(decrypted).decode("utf-8") == "eggs" diff --git a/federation/tests/protocols/diaspora/test_protocol.py b/federation/tests/protocols/diaspora/test_protocol.py index a721ea9..744fbf6 100644 --- a/federation/tests/protocols/diaspora/test_protocol.py +++ b/federation/tests/protocols/diaspora/test_protocol.py @@ -193,6 +193,28 @@ class TestDiasporaProtocol(DiasporaTestBase): mock_render.assert_called_once_with() assert data == "rendered" + @patch("federation.protocols.diaspora.protocol.MagicEnvelope") + @patch("federation.protocols.diaspora.protocol.EncryptedPayload.encrypt", return_value="encrypted") + def test_build_send_does_right_calls__private_payload(self, mock_encrypt, mock_me): + mock_render = Mock(return_value="rendered") + mock_me_instance = Mock(render=mock_render) + mock_me.return_value = mock_me_instance + protocol = Protocol() + entity = DiasporaPost() + private_key = get_dummy_private_key() + outbound_entity = get_outbound_entity(entity, private_key) + data = protocol.build_send(outbound_entity, to_user_key="public key", from_user=Mock( + private_key=private_key, handle="johnny@localhost", + )) + mock_me.assert_called_once_with( + etree.tostring(entity.to_xml()), private_key=private_key, author_handle="johnny@localhost", + ) + mock_render.assert_called_once_with() + mock_encrypt.assert_called_once_with( + "rendered", "public key", + ) + assert data == "encrypted" + @patch("federation.protocols.diaspora.protocol.MagicEnvelope") def test_build_send_uses_outbound_doc(self, mock_me): protocol = self.init_protocol() diff --git a/federation/tests/test_outbound.py b/federation/tests/test_outbound.py index 48588a7..d2f7bfc 100644 --- a/federation/tests/test_outbound.py +++ b/federation/tests/test_outbound.py @@ -1,9 +1,10 @@ -from unittest.mock import Mock, patch, call +from unittest.mock import Mock, patch from Crypto.PublicKey import RSA from federation.entities.diaspora.entities import DiasporaPost, DiasporaComment from federation.outbound import handle_create_payload, handle_send +from federation.tests.fixtures.keys import get_dummy_private_key class TestHandleCreatePayloadBuildsAPayload: @@ -14,7 +15,7 @@ class TestHandleCreatePayloadBuildsAPayload: author_user = Mock() entity = DiasporaPost() handle_create_payload(entity, author_user) - mock_protocol.build_send.assert_called_once_with(entity=entity, from_user=author_user, to_user=None) + mock_protocol.build_send.assert_called_once_with(entity=entity, from_user=author_user, to_user_key=None) @patch("federation.outbound.get_outbound_entity") def test_handle_create_payload_calls_get_outbound_entity(self, mock_get_outbound_entity): @@ -44,25 +45,31 @@ class TestHandleCreatePayloadBuildsAPayload: mock_sign.assert_called_once_with(parent_user.private_key) -@patch("federation.outbound.handle_create_payload", return_value="payload") @patch("federation.outbound.send_document") class TestHandleSend: - def test_calls_handle_create_payload(self, mock_send, mock_create, diasporapost): - recipients = [("foo@127.0.0.1", "diaspora"), ("localhost", None)] - mock_author = Mock() - handle_send(diasporapost, mock_author, recipients) - mock_create.assert_called_once_with(diasporapost, mock_author, parent_user=None) - mock_create.reset_mock() - handle_send(diasporapost, mock_author, recipients, parent_user="parent_user") - mock_create.assert_called_once_with(diasporapost, mock_author, parent_user="parent_user") - - def test_calls_send_document(self, mock_send, mock_create, diasporapost): - recipients = [("foo@127.0.0.1", "diaspora"), ("localhost", None)] - mock_from_user = Mock() - handle_send(diasporapost, mock_from_user, recipients) - call_args_list = [ - call("https://127.0.0.1/receive/public", "payload"), - call("https://localhost/receive/public", "payload"), + def test_calls_handle_create_payload(self, mock_send, diasporapost): + key = get_dummy_private_key() + recipients = [ + ("diaspora://foo@127.0.0.1/profile/xyz", key.publickey()), + ("diaspora://foo@localhost/profile/abc", None), + "diaspora://foo@example.net/profile/zzz", + "diaspora://qwer@example.net/profile/qwerty", # Same host twice to ensure one delivery only per host + # for public payloads ] - assert call_args_list[0] in mock_send.call_args_list - assert call_args_list[1] in mock_send.call_args_list + mock_author = Mock(private_key=key, handle="foo@example.com") + handle_send(diasporapost, mock_author, recipients) + + # Ensure first call is a private payload + assert mock_send.call_args_list[0][0][0] == "https://127.0.0.1/receive/users/xyz" + encrypted = mock_send.call_args_list[0][0][1] + assert "aes_key" in encrypted + assert "encrypted_magic_envelope" in encrypted + + # Ensure public payloads and recipients, one per unique host + public_endpoints = { + mock_send.call_args_list[1][0][0], + mock_send.call_args_list[2][0][0], + } + assert public_endpoints == {"https://example.net/receive/public", "https://localhost/receive/public"} + assert mock_send.call_args_list[1][0][1].startswith("