From fbaee66da8fb8886ac9c18c19210afb51a17a1f9 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Sun, 2 May 2021 00:41:39 +0300 Subject: [PATCH 1/8] Ensure missing profiles are created on Matrix side If we try to send a Post to the Matrix appservice, missing profiles need creating. Closes #127 --- federation/entities/matrix/entities.py | 18 +++++++++++++++--- federation/entities/utils.py | 22 +++++++++++++++++++++- 2 files changed, 36 insertions(+), 4 deletions(-) diff --git a/federation/entities/matrix/entities.py b/federation/entities/matrix/entities.py index 48673f4..6eacdb4 100644 --- a/federation/entities/matrix/entities.py +++ b/federation/entities/matrix/entities.py @@ -11,7 +11,7 @@ import requests from federation.entities.base import Post, Profile from federation.entities.matrix.enums import EventType from federation.entities.mixins import BaseEntity -from federation.entities.utils import get_base_attributes +from federation.entities.utils import get_base_attributes, get_profile from federation.utils.matrix import get_matrix_configuration, appservice_auth_header from federation.utils.network import fetch_document, fetch_file @@ -20,6 +20,7 @@ logger = logging.getLogger("federation") class MatrixEntityMixin(BaseEntity): _event_type: str = None + _payloads: List[Dict] = [] _profile_room_id = None _txn_id: str = None @@ -59,11 +60,10 @@ class MatrixEntityMixin(BaseEntity): if status == 200: data = json.loads(doc) self._profile_room_id = data["room_id"] - # TODO create? # noinspection PyMethodMayBeStatic def payloads(self) -> List[Dict]: - return [] + return self._payloads @property def profile_room_alias(self): @@ -84,6 +84,7 @@ class MatrixRoomMessage(Post, MatrixEntityMixin): _thread_room_id: str = None def create_thread_room(self): + # TODO don't do this immediately here, append to payloads for later pushing out headers = appservice_auth_header() # Create the thread room response = requests.post( @@ -113,6 +114,17 @@ class MatrixRoomMessage(Post, MatrixEntityMixin): response.raise_for_status() self._thread_room_event_id = response.json()["event_id"] + def get_profile_room_id(self): + super().get_profile_room_id() + if not self._profile_room_id: + from federation.entities.matrix.mappers import get_outbound_entity + # Need to also create the profile + profile = get_profile(self.actor_id) + profile_entity = get_outbound_entity(profile, None) + payloads = profile_entity.payloads() + if payloads: + self._payloads.extend(payloads) + def payloads(self) -> List[Dict]: payloads = super().payloads() payloads.append({ diff --git a/federation/entities/utils.py b/federation/entities/utils.py index 40a5c52..356c6e4 100644 --- a/federation/entities/utils.py +++ b/federation/entities/utils.py @@ -1,5 +1,8 @@ import inspect -from typing import Optional +from typing import Optional, TYPE_CHECKING + +if TYPE_CHECKING: + from federation.entities.base import Profile def get_base_attributes(entity): @@ -36,3 +39,20 @@ def get_name_for_profile(fid: str) -> Optional[str]: return profile.name except Exception: pass + + +def get_profile(fid): + # type: (str) -> Profile + """ + Get a profile via the configured profile getter. + + Currently only works with Django configuration. + """ + try: + from federation.utils.django import get_function_from_config + profile_func = get_function_from_config("get_profile_function") + if not profile_func: + return + return profile_func(fid=fid) + except Exception: + pass From 448c8cd780b1cba75ac73ebcd4bb19b9e8a8ed45 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Sun, 2 May 2021 01:15:46 +0300 Subject: [PATCH 2/8] Ensure Matrix register call happens earlier We need to register the user via the appservice before going on to the thread room creation later in MatrixRoomMessage.pre_send. --- federation/entities/matrix/entities.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/federation/entities/matrix/entities.py b/federation/entities/matrix/entities.py index 6eacdb4..d1ffcb8 100644 --- a/federation/entities/matrix/entities.py +++ b/federation/entities/matrix/entities.py @@ -84,7 +84,6 @@ class MatrixRoomMessage(Post, MatrixEntityMixin): _thread_room_id: str = None def create_thread_room(self): - # TODO don't do this immediately here, append to payloads for later pushing out headers = appservice_auth_header() # Create the thread room response = requests.post( @@ -210,6 +209,18 @@ class MatrixProfile(Profile, MatrixEntityMixin): _remote_profile_create_needed = False _remote_room_create_needed = False + def register_user(self): + headers = appservice_auth_header() + response = requests.post( + url=f"{super().get_endpoint()}/register", + json={ + "username": f"{self.localpart}", + "type": "m.login.application_service", + }, + headers=headers, + ) + response.raise_for_status() + @property def localpart(self) -> str: config = get_matrix_configuration() @@ -218,13 +229,7 @@ class MatrixProfile(Profile, MatrixEntityMixin): def payloads(self) -> List[Dict]: payloads = super().payloads() if self._remote_profile_create_needed: - payloads.append({ - "endpoint": f"{super().get_endpoint()}/register", - "payload": { - "username": f"{self.localpart}", - "type": "m.login.application_service", - }, - }) + self.register_user() if self._remote_room_create_needed: payloads.append({ "endpoint": f"{super().get_endpoint()}/createRoom", From 7973f3e87c219dee205cc382adede3c182bb39eb Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Sun, 2 May 2021 01:48:20 +0300 Subject: [PATCH 3/8] Ensure joined to profile and thread rooms Closes #126 --- federation/entities/matrix/entities.py | 31 +++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/federation/entities/matrix/entities.py b/federation/entities/matrix/entities.py index d1ffcb8..adf2199 100644 --- a/federation/entities/matrix/entities.py +++ b/federation/entities/matrix/entities.py @@ -73,6 +73,11 @@ class MatrixEntityMixin(BaseEntity): def profile_room_alias_url_safe(self): return f"{quote(self.profile_room_alias)}" + @property + def server_name(self) -> str: + config = get_matrix_configuration() + return config['homeserver_name'] + @property def txn_id(self) -> str: return self._txn_id @@ -89,7 +94,10 @@ class MatrixRoomMessage(Post, MatrixEntityMixin): response = requests.post( url=f"{super().get_endpoint()}/createRoom?user_id={self.mxid}", json={ - # TODO auto-invite recipients if private chat + # TODO auto-invite other recipients if private chat + "invite": [ + self.mxid, + ], "preset": "public_chat" if self.public else "private_chat", "name": f"Thread by {self.mxid}", "topic": self.url, @@ -99,6 +107,7 @@ class MatrixRoomMessage(Post, MatrixEntityMixin): response.raise_for_status() self._thread_room_id = response.json()["room_id"] # Send the thread message + # TODO move this to a payload response = requests.put( url=f"{super().get_endpoint()}/rooms/{self._thread_room_id}/send/{self.event_type}/" f"{str(uuid4())}?user_id={self.mxid}", @@ -126,6 +135,12 @@ class MatrixRoomMessage(Post, MatrixEntityMixin): def payloads(self) -> List[Dict]: payloads = super().payloads() + # Ensure we're joined to the profile room + # TODO remove this after a bit + payloads.append({ + "endpoint": f"{super().get_endpoint()}/join/%23{self.mxid}?user_id={self.mxid}", + "payload": {}, + }) payloads.append({ "endpoint": f"{super().get_endpoint()}/rooms/{self._profile_room_id}/send/{self.event_type}/" f"{self.txn_id}?user_id={self.mxid}", @@ -141,6 +156,12 @@ class MatrixRoomMessage(Post, MatrixEntityMixin): }, "method": "put", }) + # Join the thread room + payloads.append({ + "endpoint": f"{super().get_endpoint()}/join/{self._thread_room_id}?user_id={self.mxid}", + "payload": {}, + }) + return payloads def pre_send(self): @@ -223,8 +244,7 @@ class MatrixProfile(Profile, MatrixEntityMixin): @property def localpart(self) -> str: - config = get_matrix_configuration() - return self.mxid.replace("@", "").replace(f":{config['homeserver_name']}", "") + return self.mxid.replace("@", "").replace(f":{self.server_name}", "") def payloads(self) -> List[Dict]: payloads = super().payloads() @@ -243,6 +263,11 @@ class MatrixProfile(Profile, MatrixEntityMixin): "topic": f"Profile room of {self.url}", }, }) + # Ensure we're joined to the profile room + payloads.append({ + "endpoint": f"{super().get_endpoint()}/join/%23{self.mxid}?user_id={self.mxid}", + "payload": {}, + }) payloads.append({ "endpoint": f"{super().get_endpoint()}/profile/{self.mxid}/displayname?user_id={self.mxid}", "payload": { From a4228242dee5d4c42006bb080bdb69fa7bbe11a4 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Sun, 2 May 2021 01:49:07 +0300 Subject: [PATCH 4/8] Tag thread rooms as low priority So Element and other clients that respect tags don't clutter rooms should the appservice user use a client. --- federation/entities/matrix/entities.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/federation/entities/matrix/entities.py b/federation/entities/matrix/entities.py index adf2199..2ddba73 100644 --- a/federation/entities/matrix/entities.py +++ b/federation/entities/matrix/entities.py @@ -161,7 +161,15 @@ class MatrixRoomMessage(Post, MatrixEntityMixin): "endpoint": f"{super().get_endpoint()}/join/{self._thread_room_id}?user_id={self.mxid}", "payload": {}, }) - + # Tag the thread room as low priority + payloads.append({ + "endpoint": f"{super().get_endpoint()}/user/{self.mxid}/rooms/{self._thread_room_id}/tags/m.lowpriority" + f"?user_id={self.mxid}", + "payload": { + "order": 0, + }, + "method": "put", + }) return payloads def pre_send(self): From ca521d4a03f6f4daa21dd8a13e5e10be0becf826 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Sun, 2 May 2021 02:20:26 +0300 Subject: [PATCH 5/8] Try again for room join --- federation/entities/matrix/entities.py | 39 +++++++++++++++----------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/federation/entities/matrix/entities.py b/federation/entities/matrix/entities.py index 2ddba73..c32da9f 100644 --- a/federation/entities/matrix/entities.py +++ b/federation/entities/matrix/entities.py @@ -136,9 +136,9 @@ class MatrixRoomMessage(Post, MatrixEntityMixin): def payloads(self) -> List[Dict]: payloads = super().payloads() # Ensure we're joined to the profile room - # TODO remove this after a bit + # TODO remove this after a bit, once the auto-join on creation works payloads.append({ - "endpoint": f"{super().get_endpoint()}/join/%23{self.mxid}?user_id={self.mxid}", + "endpoint": f"{super().get_endpoint()}/rooms/{self._profile_room_id}/join?user_id={self.mxid}", "payload": {}, }) payloads.append({ @@ -158,7 +158,7 @@ class MatrixRoomMessage(Post, MatrixEntityMixin): }) # Join the thread room payloads.append({ - "endpoint": f"{super().get_endpoint()}/join/{self._thread_room_id}?user_id={self.mxid}", + "endpoint": f"{super().get_endpoint()}/rooms/{self._thread_room_id}/join?user_id={self.mxid}", "payload": {}, }) # Tag the thread room as low priority @@ -238,6 +238,24 @@ class MatrixProfile(Profile, MatrixEntityMixin): _remote_profile_create_needed = False _remote_room_create_needed = False + def create_profile_room(self): + headers = appservice_auth_header() + response = requests.post( + url=f"{super().get_endpoint()}/createRoom", + json={ + "invite": [ + self.mxid, + ], + "name": self.name, + "preset": "public_chat" if self.public else "private_chat", + "room_alias_name": f"@{self.localpart}", + "topic": f"Profile room of {self.url}", + }, + headers=headers, + ) + response.raise_for_status() + self._profile_room_id = response.json()["room_id"] + def register_user(self): headers = appservice_auth_header() response = requests.post( @@ -259,21 +277,10 @@ class MatrixProfile(Profile, MatrixEntityMixin): if self._remote_profile_create_needed: self.register_user() if self._remote_room_create_needed: - payloads.append({ - "endpoint": f"{super().get_endpoint()}/createRoom", - "payload": { - "invite": [ - self.mxid, - ], - "name": self.name, - "preset": "public_chat" if self.public else "private_chat", - "room_alias_name": f"@{self.localpart}", - "topic": f"Profile room of {self.url}", - }, - }) + self.create_profile_room() # Ensure we're joined to the profile room payloads.append({ - "endpoint": f"{super().get_endpoint()}/join/%23{self.mxid}?user_id={self.mxid}", + "endpoint": f"{super().get_endpoint()}/rooms/{self._profile_room_id}/join?user_id={self.mxid}", "payload": {}, }) payloads.append({ From a82f75822e97fabd870dbf049230712198035ba1 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Sun, 2 May 2021 02:46:54 +0300 Subject: [PATCH 6/8] Revert invite in thread room creation --- federation/entities/matrix/entities.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/federation/entities/matrix/entities.py b/federation/entities/matrix/entities.py index c32da9f..dea5434 100644 --- a/federation/entities/matrix/entities.py +++ b/federation/entities/matrix/entities.py @@ -95,15 +95,30 @@ class MatrixRoomMessage(Post, MatrixEntityMixin): url=f"{super().get_endpoint()}/createRoom?user_id={self.mxid}", json={ # TODO auto-invite other recipients if private chat - "invite": [ - self.mxid, - ], "preset": "public_chat" if self.public else "private_chat", "name": f"Thread by {self.mxid}", "topic": self.url, }, headers=headers, ) + """ + WAT + + ERROR 2021-05-02 02:25:28,299 outbound 14919 140545779672896 handle_send - failed to generate matrix payload for @forgelog:jasonrobinson.me, https://matrix.jasonrobinson.me: Traceback (most recent call last): + File "/home/jasonrobinson/venv/src/federation/federation/outbound.py", line 305, in handle_send + entity, author_user, protocol, parent_user=parent_user, payload_logger=payload_logger, + File "/home/jasonrobinson/venv/src/federation/federation/outbound.py", line 52, in handle_create_payload + outbound_entity = mappers.get_outbound_entity(entity, author_user.rsa_private_key) + File "/home/jasonrobinson/venv/src/federation/federation/entities/matrix/mappers.py", line 36, in get_outbound_entity + outbound.pre_send() + File "/home/jasonrobinson/venv/src/federation/federation/entities/matrix/entities.py", line 185, in pre_send + self.create_thread_room() + File "/home/jasonrobinson/venv/src/federation/federation/entities/matrix/entities.py", line 107, in create_thread_room + response.raise_for_status() + File "/home/jasonrobinson/venv/lib/python3.6/site-packages/requests/models.py", line 943, in raise_for_status + raise HTTPError(http_error_msg, response=self) + requests.exceptions.HTTPError: 403 Client Error: Forbidden for url: https://matrix.jasonrobinson.me/_matrix/client/r0/createRoom?user_id=@forgelog:jasonrobinson.me + """ response.raise_for_status() self._thread_room_id = response.json()["room_id"] # Send the thread message From 1d782f5f5a2881d0e84f7010651ff4b2bcc21fc5 Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Sun, 2 May 2021 02:47:09 +0300 Subject: [PATCH 7/8] Remove a comment --- federation/entities/matrix/entities.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/federation/entities/matrix/entities.py b/federation/entities/matrix/entities.py index dea5434..94729d9 100644 --- a/federation/entities/matrix/entities.py +++ b/federation/entities/matrix/entities.py @@ -101,24 +101,6 @@ class MatrixRoomMessage(Post, MatrixEntityMixin): }, headers=headers, ) - """ - WAT - - ERROR 2021-05-02 02:25:28,299 outbound 14919 140545779672896 handle_send - failed to generate matrix payload for @forgelog:jasonrobinson.me, https://matrix.jasonrobinson.me: Traceback (most recent call last): - File "/home/jasonrobinson/venv/src/federation/federation/outbound.py", line 305, in handle_send - entity, author_user, protocol, parent_user=parent_user, payload_logger=payload_logger, - File "/home/jasonrobinson/venv/src/federation/federation/outbound.py", line 52, in handle_create_payload - outbound_entity = mappers.get_outbound_entity(entity, author_user.rsa_private_key) - File "/home/jasonrobinson/venv/src/federation/federation/entities/matrix/mappers.py", line 36, in get_outbound_entity - outbound.pre_send() - File "/home/jasonrobinson/venv/src/federation/federation/entities/matrix/entities.py", line 185, in pre_send - self.create_thread_room() - File "/home/jasonrobinson/venv/src/federation/federation/entities/matrix/entities.py", line 107, in create_thread_room - response.raise_for_status() - File "/home/jasonrobinson/venv/lib/python3.6/site-packages/requests/models.py", line 943, in raise_for_status - raise HTTPError(http_error_msg, response=self) - requests.exceptions.HTTPError: 403 Client Error: Forbidden for url: https://matrix.jasonrobinson.me/_matrix/client/r0/createRoom?user_id=@forgelog:jasonrobinson.me - """ response.raise_for_status() self._thread_room_id = response.json()["room_id"] # Send the thread message From d44e2e3c8fb959a89f49888bab9885daeb0b1e8c Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Sun, 2 May 2021 22:27:40 +0300 Subject: [PATCH 8/8] Make user own the Matrix profile room Avoids needing to join it. --- federation/entities/matrix/entities.py | 22 +--------------------- 1 file changed, 1 insertion(+), 21 deletions(-) diff --git a/federation/entities/matrix/entities.py b/federation/entities/matrix/entities.py index 94729d9..17df35c 100644 --- a/federation/entities/matrix/entities.py +++ b/federation/entities/matrix/entities.py @@ -104,7 +104,6 @@ class MatrixRoomMessage(Post, MatrixEntityMixin): response.raise_for_status() self._thread_room_id = response.json()["room_id"] # Send the thread message - # TODO move this to a payload response = requests.put( url=f"{super().get_endpoint()}/rooms/{self._thread_room_id}/send/{self.event_type}/" f"{str(uuid4())}?user_id={self.mxid}", @@ -132,12 +131,6 @@ class MatrixRoomMessage(Post, MatrixEntityMixin): def payloads(self) -> List[Dict]: payloads = super().payloads() - # Ensure we're joined to the profile room - # TODO remove this after a bit, once the auto-join on creation works - payloads.append({ - "endpoint": f"{super().get_endpoint()}/rooms/{self._profile_room_id}/join?user_id={self.mxid}", - "payload": {}, - }) payloads.append({ "endpoint": f"{super().get_endpoint()}/rooms/{self._profile_room_id}/send/{self.event_type}/" f"{self.txn_id}?user_id={self.mxid}", @@ -153,11 +146,6 @@ class MatrixRoomMessage(Post, MatrixEntityMixin): }, "method": "put", }) - # Join the thread room - payloads.append({ - "endpoint": f"{super().get_endpoint()}/rooms/{self._thread_room_id}/join?user_id={self.mxid}", - "payload": {}, - }) # Tag the thread room as low priority payloads.append({ "endpoint": f"{super().get_endpoint()}/user/{self.mxid}/rooms/{self._thread_room_id}/tags/m.lowpriority" @@ -238,11 +226,8 @@ class MatrixProfile(Profile, MatrixEntityMixin): def create_profile_room(self): headers = appservice_auth_header() response = requests.post( - url=f"{super().get_endpoint()}/createRoom", + url=f"{super().get_endpoint()}/createRoom?user_id={self.mxid}", json={ - "invite": [ - self.mxid, - ], "name": self.name, "preset": "public_chat" if self.public else "private_chat", "room_alias_name": f"@{self.localpart}", @@ -275,11 +260,6 @@ class MatrixProfile(Profile, MatrixEntityMixin): self.register_user() if self._remote_room_create_needed: self.create_profile_room() - # Ensure we're joined to the profile room - payloads.append({ - "endpoint": f"{super().get_endpoint()}/rooms/{self._profile_room_id}/join?user_id={self.mxid}", - "payload": {}, - }) payloads.append({ "endpoint": f"{super().get_endpoint()}/profile/{self.mxid}/displayname?user_id={self.mxid}", "payload": {