Merge pull request #112 from jaywink/encrypted-json-payload

Enable delivery of new style private messages using Diaspora protocol
merge-requests/130/head
Jason Robinson 2018-02-13 23:19:31 +02:00 zatwierdzone przez GitHub
commit 456d8344d8
Nie znaleziono w bazie danych klucza dla tego podpisu
ID klucza GPG: 4AEE18F83AFDEB23
11 zmienionych plików z 239 dodań i 56 usunięć

Wyświetl plik

@ -2,10 +2,27 @@
## [unreleased]
### Added
* Enable generating encrypted JSON payloads with the Diaspora protocol which adds private message support. ([related issue](https://github.com/jaywink/federation/issues/82))
JSON encrypted payload encryption and decryption is handled by the Diaspora `EncryptedPayload` class.
### Changed
* Send outbound Diaspora payloads in new format. Remove possibility to generate legacy MagicEnvelope payloads. ([related issue](https://github.com/jaywink/federation/issues/82))
* Send outbound Diaspora payloads in new format. Remove possibility to generate legacy MagicEnvelope payloads. ([related issue](https://github.com/jaywink/federation/issues/82))
* **Backwards incompatible**. Refactor `handle_send` function
Now handle_send high level outbound helper function also allows delivering private payloads using the Diaspora protocol. ([related issue](https://github.com/jaywink/federation/issues/82))
The signature has changed. Parameter `recipients` should now be a list of recipients to delivery to. Each recipient should either be an `id` or a tuple of `(id, public key)`. If public key is provided, Diaspora protocol delivery will be made as an encrypted private delivery.
* **Backwards incompatible**. Change `handle_create_payload` function signature.
Parameter `to_user` is now `to_user_key` and thus instead of an object containing the `key` attribute it should now be an RSA public key object instance. This simplifies things since we only need the key from the user, nothing else.
## [0.15.0] - 2018-02-12
### Added

Wyświetl plik

@ -109,6 +109,7 @@ Diaspora
.. autofunction:: federation.utils.diaspora.fetch_public_key
.. autofunction:: federation.utils.diaspora.get_fetch_content_endpoint
.. autofunction:: federation.utils.diaspora.get_public_endpoint
.. autofunction:: federation.utils.diaspora.get_private_endpoint
.. autofunction:: federation.utils.diaspora.parse_diaspora_uri
.. autofunction:: federation.utils.diaspora.parse_profile_from_hcard
.. autofunction:: federation.utils.diaspora.retrieve_and_parse_content

Wyświetl plik

@ -1,17 +1,17 @@
from federation.entities.diaspora.mappers import get_outbound_entity
from federation.protocols.diaspora.protocol import Protocol
from federation.utils.diaspora import get_public_endpoint
from federation.utils.diaspora import get_public_endpoint, get_private_endpoint
from federation.utils.network import send_document
def handle_create_payload(entity, author_user, to_user=None, parent_user=None):
def handle_create_payload(entity, author_user, to_user_key=None, parent_user=None):
"""Create a payload with the correct protocol.
Any given user arguments must have ``private_key`` and ``handle`` attributes.
:arg entity: Entity object to send. Can be a base entity or a protocol specific one.
:arg author_user: User authoring the object.
:arg to_user: Profile entry to send to (required for non-public content)
:arg to_user_key: Public key of user private payload is being sent to, required for private payloads.
:arg parent_user: (Optional) User object of the parent object, if there is one. This must be given for the
Diaspora protocol if a parent object exists, so that a proper ``parent_author_signature`` can
be generated. If given, the payload will be sent as this user.
@ -23,7 +23,7 @@ def handle_create_payload(entity, author_user, to_user=None, parent_user=None):
if parent_user:
outbound_entity.sign_with_parent(parent_user.private_key)
send_as_user = parent_user if parent_user else author_user
data = protocol.build_send(entity=outbound_entity, from_user=send_as_user, to_user=to_user)
data = protocol.build_send(entity=outbound_entity, from_user=send_as_user, to_user_key=to_user_key)
return data
@ -33,32 +33,62 @@ def handle_send(entity, author_user, recipients=None, parent_user=None):
Using this we will build a list of payloads per protocol, after resolving any that need to be guessed or
looked up over the network. After that, each recipient will get the generated protocol payload delivered.
NOTE! This will not (yet) support Diaspora limited messages - `handle_create_payload` above should be directly
called instead and payload sent with `federation.utils.network.send_document`.
Any given user arguments must have ``private_key`` and ``handle`` attributes.
:arg entity: Entity object to send. Can be a base entity or a protocol specific one.
:arg author_user: User authoring the object.
:arg recipients: A list of tuples to delivery to. Tuple contains (recipient handle or domain, protocol or None).
For example ``[("foo@example.com", "diaspora"), ("bar@example.com", None)]``.
:arg recipients: A list of recipients to delivery to. Each recipient is a tuple
containing at minimum the "id", optionally "public key" for private deliveries.
Instead of a tuple, for public deliveries the "id" as str is also ok.
If public key is provided, Diaspora protocol delivery will be made as an encrypted
private delivery.
For example
[
("diaspora://user@domain.tld/profile/zyx", <RSAPublicKey object>),
("diaspora://user@domain2.tld/profile/xyz", None),
"diaspora://user@domain3.tld/profile/xyz",
]
:arg parent_user: (Optional) User object of the parent object, if there is one. This must be given for the
Diaspora protocol if a parent object exists, so that a proper ``parent_author_signature`` can
be generated. If given, the payload will be sent as this user.
"""
payloads = {"diaspora": {"payload": None, "recipients": set()}}
# Generate payload per protocol and split recipients to protocols
for recipient, protocol in recipients:
# TODO currently we only support Diaspora protocol, so no need to guess, just generate the payload
if not payloads["diaspora"]["payload"]:
payloads["diaspora"]["payload"] = handle_create_payload(entity, author_user, parent_user=parent_user)
if "@" in recipient:
payloads["diaspora"]["recipients"].add(recipient.split("@")[1])
payloads = []
public_payloads = {
"diaspora": {
"payload": None,
"urls": set(),
},
}
# Generate payloads and collect urls
for recipient in recipients:
id = recipient[0] if isinstance(recipient, tuple) else recipient
public_key = recipient[1] if isinstance(recipient, tuple) and len(recipient) > 1 else None
if public_key:
# Private payload
payload = handle_create_payload(entity, author_user, to_user_key=public_key, parent_user=parent_user)
# TODO get_private_endpoint should be imported per protocol
url = get_private_endpoint(id)
payloads.append({
"urls": {url}, "payload": payload,
})
else:
payloads["diaspora"]["recipients"].add(recipient)
if not public_payloads["diaspora"]["payload"]:
public_payloads["diaspora"]["payload"] = handle_create_payload(
entity, author_user, parent_user=parent_user,
)
# TODO get_public_endpoint should be imported per protocol
url = get_public_endpoint(id)
public_payloads["diaspora"]["urls"].add(url)
# Add public payload
if public_payloads["diaspora"]["payload"]:
payloads.append({
"urls": public_payloads["diaspora"]["urls"], "payload": public_payloads["diaspora"]["payload"],
})
# Do actual sending
for protocol, data in payloads.items():
for recipient in data.get("recipients"):
# TODO protocol independant url generation by importing named helper under protocol
url = get_public_endpoint(recipient)
send_document(url, data.get("payload"))
for payload in payloads:
for url in payload["urls"]:
# TODO set content type per protocol above when collecting and use here
send_document(url, payload["payload"])

Wyświetl plik

@ -1,12 +1,34 @@
import json
from base64 import b64decode
from base64 import b64decode, b64encode
from Crypto.Cipher import PKCS1_v1_5, AES
from Crypto.Random import get_random_bytes
from lxml import etree
def pkcs7_pad(inp, block_size):
"""
Using the PKCS#7 padding scheme, pad <inp> to be a multiple of
<block_size> bytes. Ruby's AES encryption pads with this scheme, but
pycrypto doesn't support it.
Implementation copied from pyaspora:
https://github.com/mjnovice/pyaspora/blob/master/pyaspora/diaspora/protocol.py#L209
"""
val = block_size - len(inp) % block_size
if val == 0:
return inp + (bytes([block_size]) * block_size)
else:
return inp + (bytes([val]) * val)
def pkcs7_unpad(data):
"""Remove the padding bytes that were added at point of encryption."""
"""
Remove the padding bytes that were added at point of encryption.
Implementation copied from pyaspora:
https://github.com/mjnovice/pyaspora/blob/master/pyaspora/diaspora/protocol.py#L209
"""
if isinstance(data, str):
return data[0:-ord(data[-1])]
else:
@ -28,3 +50,39 @@ class EncryptedPayload:
encrypter = AES.new(key, AES.MODE_CBC, iv)
content = encrypter.decrypt(encrypted_magic_envelope)
return etree.fromstring(pkcs7_unpad(content))
@staticmethod
def get_aes_key_json(iv, key):
return json.dumps({
"key": b64encode(key).decode("ascii"),
"iv": b64encode(iv).decode("ascii"),
}).encode("utf-8")
@staticmethod
def get_iv_key_encrypter():
iv = get_random_bytes(AES.block_size)
key = get_random_bytes(32)
encrypter = AES.new(key, AES.MODE_CBC, iv)
return iv, key, encrypter
@staticmethod
def encrypt(payload, public_key):
"""
Encrypt a payload using an encrypted JSON wrapper.
See: https://diaspora.github.io/diaspora_federation/federation/encryption.html
:param payload: Payload document as a string.
:param public_key: Public key of recipient as an RSA object.
:return: Encrypted JSON wrapper as dict.
"""
iv, key, encrypter = EncryptedPayload.get_iv_key_encrypter()
aes_key_json = EncryptedPayload.get_aes_key_json(iv, key)
cipher = PKCS1_v1_5.new(public_key)
aes_key = b64encode(cipher.encrypt(aes_key_json))
padded_payload = pkcs7_pad(payload.encode("utf-8"), AES.block_size)
encrypted_me = b64encode(encrypter.encrypt(padded_payload))
return {
"aes_key": aes_key,
"encrypted_magic_envelope": encrypted_me,
}

Wyświetl plik

@ -229,13 +229,22 @@ class Protocol(BaseProtocol):
else:
return data[0:-data[-1]]
def build_send(self, entity, from_user, to_user=None, *args, **kwargs):
"""Build POST data for sending out to remotes."""
def build_send(self, entity, from_user, to_user_key=None, *args, **kwargs):
"""
Build POST data for sending out to remotes.
:param entity: The outbound ready entity for this protocol.
:param from_user: The user sending this payload. Must have ``private_key`` and ``handle`` properties.
:param to_user_key: (Optional) Public key of user we're sending a private payload to.
:returns: dict or string depending on if private or public payload.
"""
if entity.outbound_doc is not None:
# Use pregenerated outbound document
xml = entity.outbound_doc
else:
xml = entity.to_xml()
me = MagicEnvelope(etree.tostring(xml), private_key=from_user.private_key, author_handle=from_user.handle)
# TODO wrap if doing encrypted delivery
return me.render()
rendered = me.render()
if to_user_key:
return EncryptedPayload.encrypt(rendered, to_user_key)
return rendered

Wyświetl plik

@ -1,8 +1,10 @@
from unittest.mock import patch, Mock
from Crypto.Cipher import AES
from lxml import etree
from federation.protocols.diaspora.encrypted import pkcs7_unpad, EncryptedPayload
from federation.tests.fixtures.keys import get_dummy_private_key
def test_pkcs7_unpad():
@ -31,3 +33,13 @@ class TestEncryptedPayload:
mock_encrypter.assert_called_once_with("magically encrypted")
assert doc.tag == "foo"
assert doc.text == "bar"
def test_encrypt(self):
private_key = get_dummy_private_key()
public_key = private_key.publickey()
encrypted = EncryptedPayload.encrypt("<spam>eggs</spam>", public_key)
assert "aes_key" in encrypted
assert "encrypted_magic_envelope" in encrypted
# See we can decrypt it too
decrypted = EncryptedPayload.decrypt(encrypted, private_key)
assert etree.tostring(decrypted).decode("utf-8") == "<spam>eggs</spam>"

Wyświetl plik

@ -193,6 +193,28 @@ class TestDiasporaProtocol(DiasporaTestBase):
mock_render.assert_called_once_with()
assert data == "rendered"
@patch("federation.protocols.diaspora.protocol.MagicEnvelope")
@patch("federation.protocols.diaspora.protocol.EncryptedPayload.encrypt", return_value="encrypted")
def test_build_send_does_right_calls__private_payload(self, mock_encrypt, mock_me):
mock_render = Mock(return_value="rendered")
mock_me_instance = Mock(render=mock_render)
mock_me.return_value = mock_me_instance
protocol = Protocol()
entity = DiasporaPost()
private_key = get_dummy_private_key()
outbound_entity = get_outbound_entity(entity, private_key)
data = protocol.build_send(outbound_entity, to_user_key="public key", from_user=Mock(
private_key=private_key, handle="johnny@localhost",
))
mock_me.assert_called_once_with(
etree.tostring(entity.to_xml()), private_key=private_key, author_handle="johnny@localhost",
)
mock_render.assert_called_once_with()
mock_encrypt.assert_called_once_with(
"rendered", "public key",
)
assert data == "encrypted"
@patch("federation.protocols.diaspora.protocol.MagicEnvelope")
def test_build_send_uses_outbound_doc(self, mock_me):
protocol = self.init_protocol()

Wyświetl plik

@ -1,9 +1,10 @@
from unittest.mock import Mock, patch, call
from unittest.mock import Mock, patch
from Crypto.PublicKey import RSA
from federation.entities.diaspora.entities import DiasporaPost, DiasporaComment
from federation.outbound import handle_create_payload, handle_send
from federation.tests.fixtures.keys import get_dummy_private_key
class TestHandleCreatePayloadBuildsAPayload:
@ -14,7 +15,7 @@ class TestHandleCreatePayloadBuildsAPayload:
author_user = Mock()
entity = DiasporaPost()
handle_create_payload(entity, author_user)
mock_protocol.build_send.assert_called_once_with(entity=entity, from_user=author_user, to_user=None)
mock_protocol.build_send.assert_called_once_with(entity=entity, from_user=author_user, to_user_key=None)
@patch("federation.outbound.get_outbound_entity")
def test_handle_create_payload_calls_get_outbound_entity(self, mock_get_outbound_entity):
@ -44,25 +45,31 @@ class TestHandleCreatePayloadBuildsAPayload:
mock_sign.assert_called_once_with(parent_user.private_key)
@patch("federation.outbound.handle_create_payload", return_value="payload")
@patch("federation.outbound.send_document")
class TestHandleSend:
def test_calls_handle_create_payload(self, mock_send, mock_create, diasporapost):
recipients = [("foo@127.0.0.1", "diaspora"), ("localhost", None)]
mock_author = Mock()
handle_send(diasporapost, mock_author, recipients)
mock_create.assert_called_once_with(diasporapost, mock_author, parent_user=None)
mock_create.reset_mock()
handle_send(diasporapost, mock_author, recipients, parent_user="parent_user")
mock_create.assert_called_once_with(diasporapost, mock_author, parent_user="parent_user")
def test_calls_send_document(self, mock_send, mock_create, diasporapost):
recipients = [("foo@127.0.0.1", "diaspora"), ("localhost", None)]
mock_from_user = Mock()
handle_send(diasporapost, mock_from_user, recipients)
call_args_list = [
call("https://127.0.0.1/receive/public", "payload"),
call("https://localhost/receive/public", "payload"),
def test_calls_handle_create_payload(self, mock_send, diasporapost):
key = get_dummy_private_key()
recipients = [
("diaspora://foo@127.0.0.1/profile/xyz", key.publickey()),
("diaspora://foo@localhost/profile/abc", None),
"diaspora://foo@example.net/profile/zzz",
"diaspora://qwer@example.net/profile/qwerty", # Same host twice to ensure one delivery only per host
# for public payloads
]
assert call_args_list[0] in mock_send.call_args_list
assert call_args_list[1] in mock_send.call_args_list
mock_author = Mock(private_key=key, handle="foo@example.com")
handle_send(diasporapost, mock_author, recipients)
# Ensure first call is a private payload
assert mock_send.call_args_list[0][0][0] == "https://127.0.0.1/receive/users/xyz"
encrypted = mock_send.call_args_list[0][0][1]
assert "aes_key" in encrypted
assert "encrypted_magic_envelope" in encrypted
# Ensure public payloads and recipients, one per unique host
public_endpoints = {
mock_send.call_args_list[1][0][0],
mock_send.call_args_list[2][0][0],
}
assert public_endpoints == {"https://example.net/receive/public", "https://localhost/receive/public"}
assert mock_send.call_args_list[1][0][1].startswith("<me:env xmlns:me=")
assert mock_send.call_args_list[2][0][1].startswith("<me:env xmlns:me=")

Wyświetl plik

@ -11,7 +11,7 @@ from federation.utils.diaspora import (
retrieve_diaspora_hcard, retrieve_diaspora_host_meta, _get_element_text_or_none,
_get_element_attr_or_none, parse_profile_from_hcard, retrieve_and_parse_profile, retrieve_and_parse_content,
get_fetch_content_endpoint, fetch_public_key, parse_diaspora_uri,
retrieve_and_parse_diaspora_webfinger, parse_diaspora_webfinger)
retrieve_and_parse_diaspora_webfinger, parse_diaspora_webfinger, get_public_endpoint, get_private_endpoint)
class TestParseDiasporaWebfinger:
@ -99,7 +99,6 @@ class TestRetrieveAndParseDiasporaWebfinger:
mock_retrieve.return_value = DiasporaHostMeta(
webfinger_host="https://localhost"
).xrd
# mock_fetch.return_value = "document", None, None
mock_xrd.return_value = "document"
result = retrieve_and_parse_diaspora_webfinger("bob@localhost")
calls = [
@ -110,7 +109,6 @@ class TestRetrieveAndParseDiasporaWebfinger:
call("https://localhost/webfinger?q=%s" % quote("bob@localhost")),
]
assert calls == mock_fetch.call_args_list
# mock_fetch.assert_called_with("https://localhost/webfinger?q=%s" % quote("bob@localhost"))
assert result == {'hcard_url': None}
@ -291,3 +289,19 @@ class TestRetrieveAndParseProfile:
mock_parse.return_value = mock_profile
retrieve_and_parse_profile("foo@bar")
assert mock_profile.validate.called
class TestGetPublicEndpoint:
def test_correct_endpoint(self):
endpoint = get_public_endpoint("diaspora://foobar@example.com/profile/123456")
assert endpoint == "https://example.com/receive/public"
class TestGetPrivateEndpoint:
def test_correct_endpoint(self):
endpoint = get_private_endpoint("diaspora://foobar@example.com/profile/123456")
assert endpoint == "https://example.com/receive/users/123456"
def test_raises_value_error_for_non_profile_id(self):
with pytest.raises(ValueError):
get_private_endpoint("diaspora://foobar@example.com/comment/123456")

Wyświetl plik

@ -235,6 +235,19 @@ def get_fetch_content_endpoint(domain, entity_type, guid):
return "https://%s/fetch/%s/%s" % (domain, entity_type, guid)
def get_public_endpoint(domain):
def get_public_endpoint(id):
"""Get remote endpoint for delivering public payloads."""
handle, _entity_type, _guid = parse_diaspora_uri(id)
_username, domain = handle.split("@")
return "https://%s/receive/public" % domain
def get_private_endpoint(id):
"""Get remote endpoint for delivering private payloads."""
handle, entity_type, guid = parse_diaspora_uri(id)
if entity_type != "profile":
raise ValueError(
"Invalid entity type %s to generate private remote endpoint for delivery. Must be 'profile'." % entity_type
)
_username, domain = handle.split("@")
return "https://%s/receive/users/%s" % (domain, guid)

Wyświetl plik

@ -80,7 +80,7 @@ def send_document(url, data, timeout=10, *args, **kwargs):
Additional ``*args`` and ``**kwargs`` will be passed on to ``requests.post``.
:arg url: Full url to send to, including protocol
:arg data: POST data to send (dict)
:arg data: Dictionary (will be form-encoded), bytes, or file-like object to send in the body
:arg timeout: Seconds to wait for response (defaults to 10)
:returns: Tuple of status code (int or None) and error (exception class instance or None)
"""