From 923582caf5471971616a1735b6c0f5f8424a9c35 Mon Sep 17 00:00:00 2001 From: Andrew Ferrazzutti Date: Fri, 28 Oct 2022 17:10:51 -0400 Subject: [PATCH 001/218] Check AppService fields for None on stop This prevents getting an AttributeError when AppService.stop is called before AppService.start --- mautrix/appservice/appservice.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/mautrix/appservice/appservice.py b/mautrix/appservice/appservice.py index 4c9bf471..cceacf56 100644 --- a/mautrix/appservice/appservice.py +++ b/mautrix/appservice/appservice.py @@ -56,7 +56,9 @@ class AppService(AppServiceServerMixin): loop: asyncio.AbstractEventLoop log: TraceLogger app: web.Application - runner: web.AppRunner + runner: web.AppRunner | None + + _http_session: aiohttp.ClientSession | None def __init__( self, @@ -178,10 +180,12 @@ async def start(self, host: str = "127.0.0.1", port: int = 8080) -> None: async def stop(self) -> None: self.log.debug("Stopping appservice web server") - await self.runner.cleanup() + if self.runner: + await self.runner.cleanup() self._intent = None - await self._http_session.close() - self._http_session = None + if self._http_session: + await self._http_session.close() + self._http_session = None await self.state_store.close() async def _liveness_probe(self, _: web.Request) -> web.Response: From 837afc51c7d293ca2120d74fcf815e81c1c1aa95 Mon Sep 17 00:00:00 2001 From: Andrew Ferrazzutti Date: Fri, 28 Oct 2022 17:11:59 -0400 Subject: [PATCH 002/218] Add None to some AppService field typehints --- mautrix/appservice/appservice.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mautrix/appservice/appservice.py b/mautrix/appservice/appservice.py index cceacf56..551eab5c 100644 --- a/mautrix/appservice/appservice.py +++ b/mautrix/appservice/appservice.py @@ -36,8 +36,8 @@ class AppService(AppServiceServerMixin): domain: str id: str verify_ssl: bool - tls_cert: str - tls_key: str + tls_cert: str | None + tls_key: str | None as_token: str hs_token: str bot_mxid: UserID @@ -59,6 +59,7 @@ class AppService(AppServiceServerMixin): runner: web.AppRunner | None _http_session: aiohttp.ClientSession | None + _intent: IntentAPI | None def __init__( self, From d7475a29b9c9f6d5cfa21cc9e96ab44ae99df410 Mon Sep 17 00:00:00 2001 From: Sumner Evans Date: Mon, 7 Nov 2022 15:46:29 -0700 Subject: [PATCH 003/218] intent/batch send: add option to set com.beeper.mark_read_by Signed-off-by: Sumner Evans --- mautrix/appservice/api/intent.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mautrix/appservice/api/intent.py b/mautrix/appservice/api/intent.py index 2d91f8db..e5d17103 100644 --- a/mautrix/appservice/api/intent.py +++ b/mautrix/appservice/api/intent.py @@ -501,6 +501,7 @@ async def batch_send( events: Iterable[BatchSendEvent], state_events_at_start: Iterable[BatchSendStateEvent] = (), beeper_new_messages: bool = False, + beeper_mark_read_by: UserID | None = None, ) -> BatchSendResponse: """ Send a batch of historical events into a room. See `MSC2716`_ for more info. @@ -530,6 +531,8 @@ async def batch_send( query["batch_id"] = batch_id if beeper_new_messages: query["com.beeper.new_messages"] = "true" + if beeper_mark_read_by: + query["com.beeper.mark_read_by"] = beeper_mark_read_by resp = await self.api.request( Method.POST, path, From 873d8d4929c4bb274a91b089e5f94340a61e1b1d Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 8 Nov 2022 18:17:29 +0200 Subject: [PATCH 004/218] Bump version to 0.18.7 --- CHANGELOG.md | 2 ++ mautrix/__init__.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e0f40acb..c88a0a9a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,5 @@ +## v0.18.7 (2022-11-08) + ## v0.18.6 (2022-10-24) * *(util.formatter)* Added conversion method for `
` tag and defaulted to diff --git a/mautrix/__init__.py b/mautrix/__init__.py index bb37c6b4..6a515f62 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.18.6" +__version__ = "0.18.7" __author__ = "Tulir Asokan " __all__ = [ "api", From b2ad957aab76f4abe9e92afbd6e4321b7e0ae657 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 11 Nov 2022 19:14:19 +0200 Subject: [PATCH 005/218] Give a logger to PgCryptoStore --- mautrix/crypto/store/asyncpg/store.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mautrix/crypto/store/asyncpg/store.py b/mautrix/crypto/store/asyncpg/store.py index cde6ad71..8609e12d 100644 --- a/mautrix/crypto/store/asyncpg/store.py +++ b/mautrix/crypto/store/asyncpg/store.py @@ -70,6 +70,7 @@ def __init__(self, account_id: str, pickle_key: str, db: Database) -> None: self.db = db self.account_id = account_id self.pickle_key = pickle_key + self.log = db.log self._sync_token = None self._device_id = DeviceID("") From 8684e2444b1612ead93dd14fe08c5948efda86bb Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 18 Nov 2022 17:11:26 +0200 Subject: [PATCH 006/218] Update state store after creating room --- mautrix/client/api/rooms.py | 4 ++-- mautrix/client/store_updater.py | 22 ++++++++++++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/mautrix/client/api/rooms.py b/mautrix/client/api/rooms.py index 61ea16d7..9f935430 100644 --- a/mautrix/client/api/rooms.py +++ b/mautrix/client/api/rooms.py @@ -5,7 +5,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import annotations -from typing import Any, Awaitable, Callable, Iterable +from typing import Any, Awaitable, Callable import asyncio from multidict import CIMultiDict @@ -64,7 +64,7 @@ async def create_room( topic: str | None = None, is_direct: bool = False, invitees: list[UserID] | None = None, - initial_state: Iterable[StateEvent | StrippedStateEvent | dict[str, JSON]] | None = None, + initial_state: list[StateEvent | StrippedStateEvent | dict[str, JSON]] | None = None, room_version: str = None, creation_content: RoomCreateStateEventContent | dict[str, JSON] | None = None, power_level_override: PowerLevelStateEventContent | dict[str, JSON] | None = None, diff --git a/mautrix/client/store_updater.py b/mautrix/client/store_updater.py index 929b378d..a5530bde 100644 --- a/mautrix/client/store_updater.py +++ b/mautrix/client/store_updater.py @@ -149,6 +149,28 @@ async def get_state(self, room_id: RoomID) -> list[StateEvent]: ) return state + async def create_room(self, *args, **kwargs) -> RoomID: + room_id = await super().create_room(*args, **kwargs) + if self.state_store: + invitee_membership = Membership.INVITE + if kwargs.get("beeper_auto_join_invites"): + invitee_membership = Membership.JOIN + for user_id in kwargs.get("invitees", []): + await self.state_store.set_membership(room_id, user_id, invitee_membership) + for evt in kwargs.get("initial_state", []): + await self.state_store.update_state( + StateEvent( + type=EventType.find(evt["type"], t_class=EventType.Class.STATE), + room_id=room_id, + event_id=EventID("$fake-create-id"), + sender=self.mxid, + state_key=evt.get("state_key", ""), + timestamp=0, + content=evt["content"], + ) + ) + return room_id + async def send_state_event( self, room_id: RoomID, From effbf5f2db91cab7390813e692c4a8580d66e06c Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 18 Nov 2022 18:34:33 +0200 Subject: [PATCH 007/218] Allow passing event ID in batch sends --- mautrix/types/event/batch.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/mautrix/types/event/batch.py b/mautrix/types/event/batch.py index df45b742..48e2d661 100644 --- a/mautrix/types/event/batch.py +++ b/mautrix/types/event/batch.py @@ -3,26 +3,28 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from typing import Any +from typing import Any, Optional from attr import dataclass import attr -from ..primitive import UserID +from ..primitive import UserID, EventID from ..util import SerializableAttrs from .base import BaseEvent -@dataclass +@dataclass(kw_only=True) class BatchSendEvent(BaseEvent, SerializableAttrs): """Base event class for events sent via a batch send request.""" sender: UserID timestamp: int = attr.ib(metadata={"json": "origin_server_ts"}) content: Any + # N.B. Overriding event IDs is not allowed in standard room versions + event_id: Optional[EventID] = None -@dataclass +@dataclass(kw_only=True) class BatchSendStateEvent(BatchSendEvent, SerializableAttrs): """ State events to be used as initial state events on batch send events. These never need to be From 666cc7270435c6282ef24538d14b095fd1f66549 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 18 Nov 2022 18:56:22 +0200 Subject: [PATCH 008/218] Fix import order --- mautrix/types/event/batch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/types/event/batch.py b/mautrix/types/event/batch.py index 48e2d661..b08ad21e 100644 --- a/mautrix/types/event/batch.py +++ b/mautrix/types/event/batch.py @@ -8,7 +8,7 @@ from attr import dataclass import attr -from ..primitive import UserID, EventID +from ..primitive import EventID, UserID from ..util import SerializableAttrs from .base import BaseEvent From af063d4b7c63f06a0560018af194dc612494cd95 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 18 Nov 2022 19:23:26 +0200 Subject: [PATCH 009/218] Bump version to 0.18.8 --- CHANGELOG.md | 8 ++++++++ mautrix/__init__.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c88a0a9a..dc547c82 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,11 @@ +## v0.18.8 (2022-11-18) + +* *(crypto.store.asyncpg)* Fixed bug causing `put_group_session` to fail when + trying to log unique key errors. +* *(client)* Added wrapper for `create_room` to update the state store with + initial state and invites (applies to anything extending `StoreUpdatingAPI`, + such as the high-level `Client` and appservice `IntentAPI` classes). + ## v0.18.7 (2022-11-08) ## v0.18.6 (2022-10-24) diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 6a515f62..b66f6369 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.18.7" +__version__ = "0.18.8" __author__ = "Tulir Asokan " __all__ = [ "api", From 4d79e1c71de12ccd493be8df4093650d9c89a492 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 30 Nov 2022 15:48:38 +0200 Subject: [PATCH 010/218] Enable foreign keys and WAL mode by default on SQLite --- mautrix/util/async_db/aiosqlite.py | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/mautrix/util/async_db/aiosqlite.py b/mautrix/util/async_db/aiosqlite.py index 444aaef4..9b7fa16a 100644 --- a/mautrix/util/async_db/aiosqlite.py +++ b/mautrix/util/async_db/aiosqlite.py @@ -110,7 +110,29 @@ def __init__( self._db_args.pop("max_size", None) self._stopped = False self._conns = 0 - self._init_commands = self._db_args.pop("init_commands", []) + self._init_commands = self._add_missing_pragmas(self._db_args.pop("init_commands", [])) + + @staticmethod + def _add_missing_pragmas(init_commands: list[str]) -> list[str]: + has_foreign_keys = False + has_journal_mode = False + has_busy_timeout = False + for cmd in init_commands: + if "PRAGMA" not in cmd: + continue + if "foreign_keys" in cmd: + has_foreign_keys = True + elif "journal_mode" in cmd: + has_journal_mode = True + elif "busy_timeout" in cmd: + has_busy_timeout = True + if not has_foreign_keys: + init_commands.append("PRAGMA foreign_keys = ON") + if not has_journal_mode: + init_commands.append("PRAGMA journal_mode = WAL") + if not has_busy_timeout: + init_commands.append("PRAGMA busy_timeout = 5000") + return init_commands async def start(self) -> None: if self._conns: @@ -118,12 +140,13 @@ async def start(self) -> None: elif self._stopped: raise RuntimeError("database pool can't be restarted") self.log.debug(f"Connecting to {self.url}") + self.log.debug(f"Database connection init commands: {self._init_commands}") for _ in range(self._pool.maxsize): conn = await TxnConnection(self._path, **self._db_args) if self._init_commands: cur = await conn.cursor() for command in self._init_commands: - self.log.debug("Executing command: %s", command) + self.log.trace("Executing init command: %s", command) await cur.execute(command) await conn.commit() conn.row_factory = sqlite3.Row From 116729a875db8d14ff1b36cdaaef31523a0596c8 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 6 Dec 2022 11:22:56 +0200 Subject: [PATCH 011/218] Add additional forbidden example value for homeserver address --- mautrix/bridge/config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mautrix/bridge/config.py b/mautrix/bridge/config.py index 012512a6..16d8f6a3 100644 --- a/mautrix/bridge/config.py +++ b/mautrix/bridge/config.py @@ -76,6 +76,7 @@ def _new_token() -> str: def forbidden_defaults(self) -> list[ForbiddenDefault]: return [ ForbiddenDefault("homeserver.address", "https://example.com"), + ForbiddenDefault("homeserver.address", "https://matrix.example.com"), ForbiddenDefault("homeserver.domain", "example.com"), ] + ( [ From fba7541c847f3cc807021fb4267272733e42225e Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 6 Dec 2022 13:01:51 +0200 Subject: [PATCH 012/218] Add more workarounds for Conduit doing dumb stuff Fixes mautrix/telegram#873 --- mautrix/types/event/state.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/mautrix/types/event/state.py b/mautrix/types/event/state.py index 1ef18e3b..c6b351c6 100644 --- a/mautrix/types/event/state.py +++ b/mautrix/types/event/state.py @@ -258,7 +258,9 @@ def deserialize(cls, data: JSON) -> "StrippedStateEvent": try: event_type = EventType.find(data.get("type", None)) data.get("content", {})["__mautrix_event_type"] = event_type - data.get("unsigned", {}).get("prev_content", {})["__mautrix_event_type"] = event_type + (data.get("unsigned") or {}).get("prev_content", {})[ + "__mautrix_event_type" + ] = event_type except ValueError: pass return super().deserialize(data) @@ -305,7 +307,10 @@ def deserialize(cls, data: JSON) -> "StateEvent": try: event_type = EventType.find(data.get("type"), t_class=EventType.Class.STATE) data.get("content", {})["__mautrix_event_type"] = event_type - if "prev_content" in data and "prev_content" not in data.get("unsigned", {}): + if "prev_content" in data and "prev_content" not in (data.get("unsigned") or {}): + # This if is a workaround for Conduit being extremely dumb + if data.get("unsigned", {}) is None: + data["unsigned"] = {} data.setdefault("unsigned", {})["prev_content"] = data["prev_content"] data.get("unsigned", {}).get("prev_content", {})["__mautrix_event_type"] = event_type except ValueError: From 478577ebbbbd005c9cda5cb022c551dce7769bac Mon Sep 17 00:00:00 2001 From: Alejandro Herrera <50601186+bramenn@users.noreply.github.com> Date: Tue, 13 Dec 2022 14:26:48 -0500 Subject: [PATCH 013/218] Allow raw dicts in `MemoryStateStore.set_power_levels` (#127) --- mautrix/client/state_store/memory.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mautrix/client/state_store/memory.py b/mautrix/client/state_store/memory.py index 5d2d5b48..8f2edac5 100644 --- a/mautrix/client/state_store/memory.py +++ b/mautrix/client/state_store/memory.py @@ -170,8 +170,10 @@ async def get_power_levels(self, room_id: RoomID) -> PowerLevelStateEventContent return self.power_levels.get(room_id) async def set_power_levels( - self, room_id: RoomID, content: PowerLevelStateEventContent + self, room_id: RoomID, content: PowerLevelStateEventContent | dict[str, Any] ) -> None: + if not isinstance(content, PowerLevelStateEventContent): + content = PowerLevelStateEventContent.deserialize(content) self.power_levels[room_id] = content async def has_encryption_info_cached(self, room_id: RoomID) -> bool: From 11a092712530d3901af4cdf26d2eb6ee0b0f895a Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 14 Dec 2022 01:15:11 +0200 Subject: [PATCH 014/218] Bump version to 0.18.9 --- CHANGELOG.md | 14 ++++++++++++++ mautrix/__init__.py | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dc547c82..af2f5fc9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,17 @@ +## v0.18.9 (2022-12-14) + +* *(util.async_db)* Changed aiosqlite connector to force-enable foreign keys, + WAL mode and busy_timeout. + * The values can be changed by manually specifying the same PRAGMAs in the + `init_commands` db arg, e.g. `- PRAGMA foreign_keys = OFF`. +* *(types)* Added workaround to `StateEvent.deserialize` to handle Conduit's + broken `unsigned` fields. +* *(client.state_store)* Fixed `set_power_level` to allow raw dicts the same + way as `set_encryption_info` does (thanks to [@bramenn] in [#127]). + +[@bramenn]: https://github.com/bramenn +[#127]: https://github.com/mautrix/python/pull/127 + ## v0.18.8 (2022-11-18) * *(crypto.store.asyncpg)* Fixed bug causing `put_group_session` to fail when diff --git a/mautrix/__init__.py b/mautrix/__init__.py index b66f6369..ae628d89 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.18.8" +__version__ = "0.18.9" __author__ = "Tulir Asokan " __all__ = [ "api", From f6b10699c16b90bf763614014a94d35e24a1efd7 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 21 Dec 2022 22:40:20 +0200 Subject: [PATCH 015/218] Fix decoding json values in env vars --- mautrix/bridge/config.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/mautrix/bridge/config.py b/mautrix/bridge/config.py index 16d8f6a3..f48b20aa 100644 --- a/mautrix/bridge/config.py +++ b/mautrix/bridge/config.py @@ -46,7 +46,7 @@ def __init__( continue key = key.removeprefix(env_prefix) if value.startswith("json::"): - value = json.loads(value) + value = json.loads(value.removeprefix("json::")) self.env[key] = value def __getitem__(self, item: str) -> Any: @@ -57,8 +57,6 @@ def __getitem__(self, item: str) -> Any: except KeyError: pass else: - if val.startswith("json::"): - val = json.loads(val.removeprefix("json::")) return val return super().__getitem__(item) From 9d734319d3674185345f857e1ae0423b2b8f4d87 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 21 Dec 2022 22:43:32 +0200 Subject: [PATCH 016/218] Simplify try except block --- mautrix/bridge/config.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/mautrix/bridge/config.py b/mautrix/bridge/config.py index f48b20aa..4289b178 100644 --- a/mautrix/bridge/config.py +++ b/mautrix/bridge/config.py @@ -53,11 +53,9 @@ def __getitem__(self, item: str) -> Any: if self.env: try: sanitized_item = item.replace(".", "_").replace("[", "").replace("]", "").upper() - val = self.env[sanitized_item] + return self.env[sanitized_item] except KeyError: pass - else: - return val return super().__getitem__(item) def save(self) -> None: From e6fd197b2fd7ccecd3e338ec5979f4f6bb798f19 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 27 Dec 2022 22:19:33 +0200 Subject: [PATCH 017/218] Use hungryserv room yeeting endpoint --- mautrix/appservice/api/intent.py | 6 ++++++ mautrix/bridge/portal.py | 10 ++++++++++ 2 files changed, 16 insertions(+) diff --git a/mautrix/appservice/api/intent.py b/mautrix/appservice/api/intent.py index e5d17103..f16441e4 100644 --- a/mautrix/appservice/api/intent.py +++ b/mautrix/appservice/api/intent.py @@ -544,6 +544,12 @@ async def batch_send( ) return BatchSendResponse.deserialize(resp) + async def beeper_delete_room(self, room_id: RoomID) -> None: + versions = await self.versions() + if not versions.supports("com.beeper.room_yeeting"): + raise RuntimeError("Homeserver does not support yeeting rooms") + await self.api.request(Method.POST, Path.unstable["com.beeper.yeet"].rooms[room_id].delete) + # endregion # region Ensure functions diff --git a/mautrix/bridge/portal.py b/mautrix/bridge/portal.py index 0d4b110f..7ade73da 100644 --- a/mautrix/bridge/portal.py +++ b/mautrix/bridge/portal.py @@ -458,6 +458,16 @@ async def cleanup_room( message: str = "Cleaning room", puppets_only: bool = False, ) -> None: + if not puppets_only and cls.bridge.homeserver_software.is_hungry: + try: + await intent.beeper_delete_room(room_id) + return + except Exception: + cls.log.warning( + f"Failed to delete {room_id} using hungryserv yeet endpoint, " + f"falling back to normal method", + exc_info=True, + ) try: members = await intent.get_room_members(room_id) except MatrixError: From 883b7fb42017bbf3624f779137745023ca4ed809 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 27 Dec 2022 22:24:44 +0200 Subject: [PATCH 018/218] Remove double retry loop when accepting invites as bridge bot --- mautrix/bridge/matrix.py | 22 +++++----------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/mautrix/bridge/matrix.py b/mautrix/bridge/matrix.py index 6cd6b1c2..144264aa 100644 --- a/mautrix/bridge/matrix.py +++ b/mautrix/bridge/matrix.py @@ -472,23 +472,11 @@ async def send_permission_error(self, room_id: RoomID) -> None: ) async def accept_bot_invite(self, room_id: RoomID, inviter: br.BaseUser) -> None: - tries = 0 - while tries < 5: - try: - await self.az.intent.join_room(room_id) - break - except (IntentError, MatrixError): - tries += 1 - wait_for_seconds = (tries + 1) * 10 - if tries < 5: - self.log.exception( - f"Failed to join room {room_id} with bridge bot, " - f"retrying in {wait_for_seconds} seconds..." - ) - await asyncio.sleep(wait_for_seconds) - else: - self.log.exception(f"Failed to join room {room_id}, giving up.") - return + try: + await self.az.intent.join_room(room_id) + except Exception: + self.log.exception(f"Failed to join room {room_id} as bridge bot") + return if not await self.allow_command(inviter): await self.send_permission_error(room_id) From 26de4b2e5ca24d28ca7e55371d34cd5c27a3382d Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 2 Jan 2023 13:46:34 +0200 Subject: [PATCH 019/218] Remove appservice typing state store --- mautrix/appservice/api/intent.py | 12 ++++++++---- mautrix/appservice/state_store/memory.py | 21 --------------------- 2 files changed, 8 insertions(+), 25 deletions(-) diff --git a/mautrix/appservice/api/intent.py b/mautrix/appservice/api/intent.py index f16441e4..92966313 100644 --- a/mautrix/appservice/api/intent.py +++ b/mautrix/appservice/api/intent.py @@ -407,13 +407,17 @@ async def set_typing( room_id: RoomID, is_typing: bool = True, timeout: int = 5000, - ignore_cache: bool = False, ) -> None: + """ + Args: + room_id: The ID of the room in which the user is typing. + is_typing: Whether the user is typing. + .. deprecated:: 0.18.10 + Use ``timeout=0`` instead of setting this flag. + timeout: The length of time in seconds to mark this user as typing. + """ await self.ensure_joined(room_id) - if not ignore_cache and is_typing == self.state_store.is_typing(room_id, self.mxid): - return await super().set_typing(room_id, timeout if is_typing else 0) - self.state_store.set_typing(room_id, self.mxid, is_typing, timeout) async def error_and_leave( self, room_id: RoomID, text: str | None = None, html: str | None = None diff --git a/mautrix/appservice/state_store/memory.py b/mautrix/appservice/state_store/memory.py index f7af8a36..a9aba4ea 100644 --- a/mautrix/appservice/state_store/memory.py +++ b/mautrix/appservice/state_store/memory.py @@ -13,7 +13,6 @@ class ASStateStore(ClientStateStore, ABC): _presence: Dict[UserID, str] - _typing: Dict[Tuple[RoomID, UserID], int] _read: Dict[Tuple[RoomID, UserID], EventID] _registered: Dict[UserID, bool] @@ -21,7 +20,6 @@ def __init__(self) -> None: self._registered = {} # Non-persistent storage self._presence = {} - self._typing = {} self._read = {} async def is_registered(self, user_id: UserID) -> bool: @@ -69,22 +67,3 @@ def get_read(self, room_id: RoomID, user_id: UserID) -> Optional[EventID]: return self._read[(room_id, user_id)] except KeyError: return None - - def set_typing( - self, room_id: RoomID, user_id: UserID, is_typing: bool, timeout: int = 0 - ) -> None: - if is_typing: - ts = int(round(time.time() * 1000)) - self._typing[(room_id, user_id)] = ts + timeout - else: - try: - del self._typing[(room_id, user_id)] - except KeyError: - pass - - def is_typing(self, room_id: RoomID, user_id: UserID) -> bool: - ts = int(round(time.time() * 1000)) - try: - return self._typing[(room_id, user_id)] > ts - except KeyError: - return False From b717598ccdc850895db001e998aff28c47752194 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 2 Jan 2023 13:46:43 +0200 Subject: [PATCH 020/218] Update changelog --- CHANGELOG.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index af2f5fc9..70c15c8f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,12 @@ +## unreleased + +* *(bridge)* Removed accidentally nested reply loop when accepting invites as + the bridge bot. +* *(bridge)* Fixed decoding JSON values in config override env vars. +* *(appservice)* Removed typing status from state store. + * Additionally, the `is_typing` boolean in `set_typing` is now deprecated, + and `timeout=0` should be used instead to match the `ClientAPI` behavior. + ## v0.18.9 (2022-12-14) * *(util.async_db)* Changed aiosqlite connector to force-enable foreign keys, From 11593cc8e3a18b7d851dbd535a768b7a9ded07cf Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 2 Jan 2023 20:16:11 +0200 Subject: [PATCH 021/218] Fix manhole python compiler --- mautrix/util/manhole.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/mautrix/util/manhole.py b/mautrix/util/manhole.py index 727ae36d..c6505822 100644 --- a/mautrix/util/manhole.py +++ b/mautrix/util/manhole.py @@ -76,7 +76,13 @@ class StatefulCommandCompiler(codeop.CommandCompiler): def __init__(self) -> None: super().__init__() self.compiler = functools.partial( - compile, optimize=1, flags=ast.PyCF_ONLY_AST | codeop.PyCF_DONT_IMPLY_DEDENT + compile, + optimize=1, + flags=( + ast.PyCF_ONLY_AST + | codeop.PyCF_DONT_IMPLY_DEDENT + | codeop.PyCF_ALLOW_INCOMPLETE_INPUT + ), ) self.buf = BytesIO() From 2b004c642127f2a1b8d8b714acdfeddbdc691c51 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 3 Jan 2023 21:20:51 +0200 Subject: [PATCH 022/218] Ignore M_NOT_FOUND in hungryserv yeet endpoint --- mautrix/bridge/portal.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mautrix/bridge/portal.py b/mautrix/bridge/portal.py index 7ade73da..f51a0ae8 100644 --- a/mautrix/bridge/portal.py +++ b/mautrix/bridge/portal.py @@ -462,6 +462,9 @@ async def cleanup_room( try: await intent.beeper_delete_room(room_id) return + except MNotFound as err: + cls.log.debug(f"Hungryserv yeet returned {err}, assuming the room is already gone") + return except Exception: cls.log.warning( f"Failed to delete {room_id} using hungryserv yeet endpoint, " From ef0856c70e2dd6b2e929f46c760fc87d8097d730 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 4 Jan 2023 17:44:11 +0200 Subject: [PATCH 023/218] Remove legacy fields in Beeper MSS events --- mautrix/types/event/beeper.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/mautrix/types/event/beeper.py b/mautrix/types/event/beeper.py index dc588800..23a9f5cd 100644 --- a/mautrix/types/event/beeper.py +++ b/mautrix/types/event/beeper.py @@ -49,19 +49,8 @@ class BeeperMessageStatusEventContent(SerializableAttrs): error: Optional[str] = None message: Optional[str] = None - success: Optional[bool] = None - still_working: Optional[bool] = None - can_retry: Optional[bool] = None - is_certain: Optional[bool] = None - last_retry: Optional[EventID] = None - def fill_legacy_booleans(self) -> None: - self.success = self.status == MessageStatus.SUCCESS - if not self.success: - self.still_working = self.status == MessageStatus.PENDING - self.can_retry = self.status in (MessageStatus.PENDING, MessageStatus.RETRIABLE) - @dataclass class BeeperMessageStatusEvent(BaseRoomEvent, SerializableAttrs): From beade51174334d5c3ef4fd5f023ae05b772cb8ee Mon Sep 17 00:00:00 2001 From: Scott Weber Date: Mon, 9 Jan 2023 17:01:23 -0500 Subject: [PATCH 024/218] Add probe_path and probe_bytes functions (#129) * Add probe_path and probe_bytes functions * Run black and isort * Update mautrix/util/ffmpeg.py Co-authored-by: Tulir Asokan Co-authored-by: Tulir Asokan --- mautrix/util/ffmpeg.py | 102 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 97 insertions(+), 5 deletions(-) diff --git a/mautrix/util/ffmpeg.py b/mautrix/util/ffmpeg.py index f158ae52..7d195c43 100644 --- a/mautrix/util/ffmpeg.py +++ b/mautrix/util/ffmpeg.py @@ -5,9 +5,11 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import annotations -from typing import Iterable +from typing import Any, Iterable from pathlib import Path import asyncio +import json +import logging import mimetypes import os import shutil @@ -34,7 +36,89 @@ def __init__(self) -> None: ffmpeg_path = _abswhich("ffmpeg") -ffmpeg_default_params = ("-hide_banner", "-loglevel", "warning") +ffmpeg_default_params = ("-hide_banner", "-loglevel", "warning", "-y") + +ffprobe_path = _abswhich("ffprobe") +ffprobe_default_params = ( + "-loglevel", + "quiet", + "-print_format", + "json", + "-show_optional_fields", + "1", + "-show_format", + "-show_streams", +) + + +async def probe_path( + input_file: os.PathLike[str] | str, + logger: logging.Logger | None = None, +) -> Any: + """ + Probes a media file on the disk using ffprobe. + + Args: + input_file: The full path to the file. + + Returns: + A Python object containing the parsed JSON response from ffprobe + + Raises: + ConverterError: if ffprobe returns a non-zero exit code. + """ + if ffprobe_path is None: + raise NotInstalledError() + + input_file = Path(input_file) + proc = await asyncio.create_subprocess_exec( + ffprobe_path, + *ffprobe_default_params, + str(input_file), + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + stdin=asyncio.subprocess.PIPE, + ) + stdout, stderr = await proc.communicate() + if proc.returncode != 0: + err_text = stderr.decode("utf-8") if stderr else f"unknown ({proc.returncode})" + raise ConverterError(f"ffprobe error: {err_text}") + elif stderr and logger: + logger.warn(f"ffprobe warning: {stderr.decode('utf-8')}") + return json.loads(stdout) + + +async def probe_bytes( + data: bytes, + input_mime: str | None = None, + logger: logging.Logger | None = None, +) -> Any: + """ + Probe media file data using ffprobe. + + Args: + data: The bytes of the file to probe. + input_mime: The mime type of the input data. If not specified, will be guessed using magic. + + Returns: + A Python object containing the parsed JSON response from ffprobe + + Raises: + ConverterError: if ffprobe returns a non-zero exit code. + """ + if ffprobe_path is None: + raise NotInstalledError() + + if input_mime is None: + if magic is None: + raise ValueError("input_mime was not specified and magic is not installed") + input_mime = magic.mimetype(data) + input_extension = mimetypes.guess_extension(input_mime) + with tempfile.TemporaryDirectory(prefix="mautrix_ffmpeg_") as tmpdir: + input_file = Path(tmpdir) / f"data{input_extension}" + with open(input_file, "wb") as file: + file.write(data) + return await probe_path(input_file=input_file, logger=logger) async def convert_path( @@ -44,6 +128,7 @@ async def convert_path( output_args: Iterable[str] | None = None, remove_input: bool = False, output_path_override: os.PathLike[str] | str | None = None, + logger: logging.Logger | None = None, ) -> Path | bytes: """ Convert a media file on the disk using ffmpeg. @@ -76,6 +161,10 @@ async def convert_path( else: input_file = Path(input_file) output_file = input_file.parent / f"{input_file.stem}{output_extension}" + if input_file == output_file: + output_file = Path(output_file) + output_file = output_file.parent / f"{output_file.stem}-new{output_extension}" + proc = await asyncio.create_subprocess_exec( ffmpeg_path, *ffmpeg_default_params, @@ -92,9 +181,8 @@ async def convert_path( if proc.returncode != 0: err_text = stderr.decode("utf-8") if stderr else f"unknown ({proc.returncode})" raise ConverterError(f"ffmpeg error: {err_text}") - elif stderr: - # TODO log warnings? - pass + elif stderr and logger: + logger.warn(f"ffmpeg warning: {stderr.decode('utf-8')}") if remove_input and isinstance(input_file, Path): input_file.unlink(missing_ok=True) return stdout if output_file == "-" else output_file @@ -106,6 +194,7 @@ async def convert_bytes( input_args: Iterable[str] | None = None, output_args: Iterable[str] | None = None, input_mime: str | None = None, + logger: logging.Logger | None = None, ) -> bytes: """ Convert media file data using ffmpeg. @@ -140,6 +229,7 @@ async def convert_bytes( output_extension=output_extension, input_args=input_args, output_args=output_args, + logger=logger, ) with open(output_file, "rb") as file: return file.read() @@ -152,4 +242,6 @@ async def convert_bytes( "NotInstalledError", "convert_bytes", "convert_path", + "probe_bytes", + "probe_path", ] From 3a34fdcfa3855f38b95123aa83b2e652e6569d62 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 10 Jan 2023 15:22:12 +0200 Subject: [PATCH 025/218] Remove is_typing parameter in IntentAPI.set_typing --- mautrix/appservice/api/intent.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/mautrix/appservice/api/intent.py b/mautrix/appservice/api/intent.py index 92966313..8a486eca 100644 --- a/mautrix/appservice/api/intent.py +++ b/mautrix/appservice/api/intent.py @@ -405,19 +405,10 @@ async def get_room_member_info( async def set_typing( self, room_id: RoomID, - is_typing: bool = True, - timeout: int = 5000, + timeout: int = 0, ) -> None: - """ - Args: - room_id: The ID of the room in which the user is typing. - is_typing: Whether the user is typing. - .. deprecated:: 0.18.10 - Use ``timeout=0`` instead of setting this flag. - timeout: The length of time in seconds to mark this user as typing. - """ await self.ensure_joined(room_id) - await super().set_typing(room_id, timeout if is_typing else 0) + await super().set_typing(room_id, timeout) async def error_and_leave( self, room_id: RoomID, text: str | None = None, html: str | None = None From b0701627bca11dbbf37c552e8eda62f4b3c85918 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 10 Jan 2023 15:23:59 +0200 Subject: [PATCH 026/218] Add Dendrite support to hacky already joined error check --- mautrix/appservice/api/intent.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mautrix/appservice/api/intent.py b/mautrix/appservice/api/intent.py index 8a486eca..8b5ca16f 100644 --- a/mautrix/appservice/api/intent.py +++ b/mautrix/appservice/api/intent.py @@ -248,7 +248,9 @@ async def invite_user( await self.state_store.joined(room_id, user_id) except MatrixRequestError as e: # TODO remove this once MSC3848 is released and minimum spec version is bumped - if e.errcode == "M_FORBIDDEN" and "is already in the room" in e.message: + if e.errcode == "M_FORBIDDEN" and ( + "already in the room" in e.message or "is already joined to room" in e.message + ): await self.state_store.joined(room_id, user_id) else: raise From bee00cffca58d7ae566c2d216f6c21498d262a16 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 10 Jan 2023 15:25:30 +0200 Subject: [PATCH 027/218] Bump version to 0.19.0 --- CHANGELOG.md | 10 ++++++---- mautrix/__init__.py | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 70c15c8f..9027b641 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,11 +1,13 @@ -## unreleased +## v0.19.0 (2023-01-10) +* **Breaking change *(appservice)*** Removed typing status from state store. +* **Breaking change *(appservice)*** Removed `is_typing` parameter from + `IntentAPI.set_typing` to make the signature match `ClientAPI.set_typing`. + `timeout=0` is equivalent to the old `is_typing=False`. +* **Breaking change *(types)*** Removed legacy fields in Beeper MSS events. * *(bridge)* Removed accidentally nested reply loop when accepting invites as the bridge bot. * *(bridge)* Fixed decoding JSON values in config override env vars. -* *(appservice)* Removed typing status from state store. - * Additionally, the `is_typing` boolean in `set_typing` is now deprecated, - and `timeout=0` should be used instead to match the `ClientAPI` behavior. ## v0.18.9 (2022-12-14) diff --git a/mautrix/__init__.py b/mautrix/__init__.py index ae628d89..0b02a1fc 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.18.9" +__version__ = "0.19.0" __author__ = "Tulir Asokan " __all__ = [ "api", From a1f5a5bc5b07a25564e817789927218dcad6cdd3 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 11 Jan 2023 12:29:57 +0200 Subject: [PATCH 028/218] Add byte iter memory optimization to async media uploads --- mautrix/api.py | 10 ++---- .../client/api/modules/media_repository.py | 8 ++++- mautrix/util/__init__.py | 31 ++++++++++--------- mautrix/util/async_iter_bytes.py | 28 +++++++++++++++++ 4 files changed, 54 insertions(+), 23 deletions(-) create mode 100644 mautrix/util/async_iter_bytes.py diff --git a/mautrix/api.py b/mautrix/api.py index a0561d08..a1ccd07e 100644 --- a/mautrix/api.py +++ b/mautrix/api.py @@ -22,6 +22,7 @@ from mautrix import __optional_imports__, __version__ as mautrix_version from mautrix.errors import MatrixConnectionError, MatrixRequestError, make_request_error +from mautrix.util.async_iter_bytes import AsyncBody, async_iter_bytes from mautrix.util.logging import TraceLogger from mautrix.util.opt_prometheus import Counter @@ -155,7 +156,6 @@ def replace(self, find: str, replace: str) -> PathBuilder: """ _req_id = 0 -AsyncBody = AsyncGenerator[Union[bytes, bytearray, memoryview], None] def _next_global_req_id() -> int: @@ -164,12 +164,6 @@ def _next_global_req_id() -> int: return _req_id -async def _async_iter_bytes(data: bytearray | bytes, chunk_size: int = 1024**2) -> AsyncBody: - with memoryview(data) as mv: - for i in range(0, len(data), chunk_size): - yield mv[i : i + chunk_size] - - class HTTPAPI: """HTTPAPI is a simple asyncio Matrix API request sender.""" @@ -395,7 +389,7 @@ async def request( method, log_url, content, orig_content, query_params, headers, req_id, sensitive ) API_CALLS.labels(method=metrics_method).inc() - req_content = _async_iter_bytes(content) if do_fake_iter else content + req_content = async_iter_bytes(content) if do_fake_iter else content start = time.monotonic() try: resp_data, resp = await self._send( diff --git a/mautrix/client/api/modules/media_repository.py b/mautrix/client/api/modules/media_repository.py index b1de6751..2b76e881 100644 --- a/mautrix/client/api/modules/media_repository.py +++ b/mautrix/client/api/modules/media_repository.py @@ -20,6 +20,7 @@ MXOpenGraph, SerializerError, ) +from mautrix.util.async_iter_bytes import async_iter_bytes from mautrix.util.opt_prometheus import Histogram from ..base import BaseClientAPI @@ -286,15 +287,20 @@ async def _upload_to_url( headers: dict[str, str], data: bytes | bytearray | AsyncIterable[bytes], post_upload_query: dict[str, str], + min_iter_size: int = 25 * 1024 * 1024, ) -> None: retry_count = self.api.default_retry_count backoff = 4 + do_fake_iter = data and hasattr(data, "__len__") and len(data) > min_iter_size + if do_fake_iter: + headers["Content-Length"] = str(len(data)) while True: self.log.debug("Uploading media to external URL %s", upload_url) upload_response = None try: + req_data = async_iter_bytes(data) if do_fake_iter else data upload_response = await self.api.session.put( - upload_url, data=data, headers=headers + upload_url, data=req_data, headers=headers ) upload_response.raise_for_status() except Exception as e: diff --git a/mautrix/util/__init__.py b/mautrix/util/__init__.py index 6a7827d1..0212dd7e 100644 --- a/mautrix/util/__init__.py +++ b/mautrix/util/__init__.py @@ -1,22 +1,25 @@ __all__ = [ + "async_db", + "config", + "db", "formatter", "logging", - "config", - "signed_token", - "simple_template", - "manhole", - "markdown", - "simple_lock", + "async_getter_lock", + "async_iter_bytes", + "bridge_state", + "color_log", + "ffmpeg", "file_store", - "program", - "async_db", - "db", - "opt_prometheus", + "format_duration", "magic", - "bridge_state", + "manhole", + "markdown", "message_send_checkpoint", - "variation_selector", - "format_duration", - "ffmpeg", + "opt_prometheus", + "program", + "signed_token", + "simple_lock", + "simple_template", "utf16_surrogate", + "variation_selector", ] diff --git a/mautrix/util/async_iter_bytes.py b/mautrix/util/async_iter_bytes.py new file mode 100644 index 00000000..25e3806d --- /dev/null +++ b/mautrix/util/async_iter_bytes.py @@ -0,0 +1,28 @@ +# Copyright (c) 2023 Tulir Asokan +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +from typing import AsyncGenerator, Union + +AsyncBody = AsyncGenerator[Union[bytes, bytearray, memoryview], None] + + +async def async_iter_bytes(data: bytearray | bytes, chunk_size: int = 1024**2) -> AsyncBody: + """ + Return memory views into a byte array in chunks. This is used to prevent aiohttp from copying + the entire request body. + + Args: + data: The underlying data to iterate through. + chunk_size: How big each returned chunk should be. + + Returns: + An async generator that yields the given data in chunks. + """ + with memoryview(data) as mv: + for i in range(0, len(data), chunk_size): + yield mv[i : i + chunk_size] + + +__all__ = ["AsyncBody", "async_iter_bytes"] From d3486f22c0a6d3180519c261c5e0767d98404abb Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 11 Jan 2023 12:49:08 +0200 Subject: [PATCH 029/218] Add missing import --- mautrix/util/async_iter_bytes.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mautrix/util/async_iter_bytes.py b/mautrix/util/async_iter_bytes.py index 25e3806d..088baed4 100644 --- a/mautrix/util/async_iter_bytes.py +++ b/mautrix/util/async_iter_bytes.py @@ -3,6 +3,8 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. +from __future__ import annotations + from typing import AsyncGenerator, Union AsyncBody = AsyncGenerator[Union[bytes, bytearray, memoryview], None] From 2a71f0176701627765cd5a659f092d17b1798908 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 11 Jan 2023 12:50:49 +0200 Subject: [PATCH 030/218] Update CI and linter versions --- .github/workflows/python-package.yml | 6 +++--- .pre-commit-config.yaml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 81c07712..7c2264b4 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -9,7 +9,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.8", "3.9", "3.10"] + python-version: ["3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v3 @@ -50,14 +50,14 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-python@v3 with: - python-version: "3.10" + python-version: "3.11" - uses: isort/isort-action@master with: sortPaths: "./mautrix" - uses: psf/black@stable with: src: "./mautrix" - version: "22.3.0" + version: "22.12.0" - name: pre-commit run: | pip install pre-commit diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 56919be3..69af4a5a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.1.0 + rev: v4.3.0 hooks: - id: trailing-whitespace exclude_types: [markdown] @@ -8,13 +8,13 @@ repos: - id: check-yaml - id: check-added-large-files - repo: https://github.com/psf/black - rev: 22.3.0 + rev: 22.12.0 hooks: - id: black language_version: python3 files: ^mautrix/.*\.pyi?$ - repo: https://github.com/PyCQA/isort - rev: 5.10.1 + rev: 5.11.4 hooks: - id: isort files: ^mautrix/.*\.pyi?$ From ac82892bc06f102ff8bf3262182c9c722e1d8891 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 11 Jan 2023 15:09:27 +0200 Subject: [PATCH 031/218] Mark Python 3.11 as supported in setup.py --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f7596154..4b3c9f78 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ ], extras_require={ "detect_mimetype": ["python-magic>=0.4.15,<0.5"], - "lint": ["black==22.1.0", "isort"], + "lint": ["black~=22.1", "isort"], "test": ["pytest", "pytest-asyncio", *test_dependencies], "encryption": encryption_dependencies, }, @@ -45,6 +45,7 @@ "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", ], package_data={ From 51c4e4b77a9cba43b3854ccb534fd4a53a463197 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 11 Jan 2023 15:12:21 +0200 Subject: [PATCH 032/218] Bump version to 0.19.1 --- CHANGELOG.md | 11 +++++++++++ mautrix/__init__.py | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9027b641..bd69c304 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,14 @@ +## v0.19.1 (2023-01-11) + +* Marked Python 3.11 as supported. Python 3.8 support will likely be dropped in + the coming months. +* *(client.api)* Added request payload memory optimization to MSC3870 URL uploads. + * aiohttp will duplicate the entire request body if it's raw bytes, which + wastes a lot of memory. The optimization is passing an iterator instead of + raw bytes, so aiohttp won't accidentally duplicate the whole thing. + * The main `HTTPAPI` has had the optimization for a while, but uploading to + URL calls aiohttp manually. + ## v0.19.0 (2023-01-10) * **Breaking change *(appservice)*** Removed typing status from state store. diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 0b02a1fc..224a35ca 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.19.0" +__version__ = "0.19.1" __author__ = "Tulir Asokan " __all__ = [ "api", From 933af513fc78aa06a4e5fa5243a461b36200ae04 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 11 Jan 2023 16:54:09 +0200 Subject: [PATCH 033/218] Add utility for reading aiohttp response into bytearray --- mautrix/api.py | 2 +- .../client/api/modules/media_repository.py | 2 +- mautrix/util/__init__.py | 4 +- mautrix/util/async_body.py | 95 +++++++++++++++++++ mautrix/util/async_iter_bytes.py | 30 ------ 5 files changed, 100 insertions(+), 33 deletions(-) create mode 100644 mautrix/util/async_body.py delete mode 100644 mautrix/util/async_iter_bytes.py diff --git a/mautrix/api.py b/mautrix/api.py index a1ccd07e..f6c9f475 100644 --- a/mautrix/api.py +++ b/mautrix/api.py @@ -22,7 +22,7 @@ from mautrix import __optional_imports__, __version__ as mautrix_version from mautrix.errors import MatrixConnectionError, MatrixRequestError, make_request_error -from mautrix.util.async_iter_bytes import AsyncBody, async_iter_bytes +from mautrix.util.async_body import AsyncBody, async_iter_bytes from mautrix.util.logging import TraceLogger from mautrix.util.opt_prometheus import Counter diff --git a/mautrix/client/api/modules/media_repository.py b/mautrix/client/api/modules/media_repository.py index 2b76e881..f91ba456 100644 --- a/mautrix/client/api/modules/media_repository.py +++ b/mautrix/client/api/modules/media_repository.py @@ -20,7 +20,7 @@ MXOpenGraph, SerializerError, ) -from mautrix.util.async_iter_bytes import async_iter_bytes +from mautrix.util.async_body import async_iter_bytes from mautrix.util.opt_prometheus import Histogram from ..base import BaseClientAPI diff --git a/mautrix/util/__init__.py b/mautrix/util/__init__.py index 0212dd7e..e731dfa4 100644 --- a/mautrix/util/__init__.py +++ b/mautrix/util/__init__.py @@ -1,11 +1,13 @@ __all__ = [ + # Directory modules "async_db", "config", "db", "formatter", "logging", + # File modules + "async_body", "async_getter_lock", - "async_iter_bytes", "bridge_state", "color_log", "ffmpeg", diff --git a/mautrix/util/async_body.py b/mautrix/util/async_body.py new file mode 100644 index 00000000..4db4d1e5 --- /dev/null +++ b/mautrix/util/async_body.py @@ -0,0 +1,95 @@ +# Copyright (c) 2023 Tulir Asokan +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +from __future__ import annotations + +from typing import AsyncGenerator, Union +import logging + +import aiohttp + +AsyncBody = AsyncGenerator[Union[bytes, bytearray, memoryview], None] + + +async def async_iter_bytes(data: bytearray | bytes, chunk_size: int = 1024**2) -> AsyncBody: + """ + Return memory views into a byte array in chunks. This is used to prevent aiohttp from copying + the entire request body. + + Args: + data: The underlying data to iterate through. + chunk_size: How big each returned chunk should be. + + Returns: + An async generator that yields the given data in chunks. + """ + with memoryview(data) as mv: + for i in range(0, len(data), chunk_size): + yield mv[i : i + chunk_size] + + +class FileTooLargeError(Exception): + def __init__(self, max_size: int) -> None: + super().__init__(f"File size larger than maximum ({max_size / 1024 / 1024} MiB)") + + +_default_dl_log = logging.getLogger("mau.util.download") + + +async def read_response_chunks( + resp: aiohttp.ClientResponse, max_size: int, log: logging.Logger = _default_dl_log +) -> bytearray: + """ + Read the body from an aiohttp response in chunks into a mutable bytearray. + + Args: + resp: The aiohttp response object to read the body from. + max_size: The maximum size to read. FileTooLargeError will be raised if the Content-Length + is higher than this, or if the body exceeds this size during reading. + log: A logger for logging download status. + + Returns: + The body data as a byte array. + + Raises: + FileTooLargeError: if the body is larger than the provided max_size. + """ + content_length = int(resp.headers.get("Content-Length", "0")) + if 0 < max_size < content_length: + raise FileTooLargeError(max_size) + size_str = "unknown length" if content_length == 0 else f"{content_length} bytes" + log.info(f"Reading file download response with {size_str} (max: {max_size})") + data = bytearray(content_length) + mv = memoryview(data) if content_length > 0 else None + read_size = 0 + max_size += 1 + while True: + block = await resp.content.readany() + if not block: + break + max_size -= len(block) + if max_size <= 0: + raise FileTooLargeError(max_size) + if len(data) >= read_size + len(block): + mv[read_size : read_size + len(block)] = block + elif len(data) > read_size: + log.warning("File being downloaded is bigger than expected") + mv[read_size:] = block[: len(data) - read_size] + mv.release() + mv = None + data.extend(block[len(data) - read_size :]) + else: + if mv is not None: + mv.release() + mv = None + data.extend(block) + read_size += len(block) + if mv is not None: + mv.release() + log.info(f"Successfully read {read_size} bytes of file download response") + return data + + +__all__ = ["AsyncBody", "FileTooLargeError", "async_iter_bytes", "async_read_bytes"] diff --git a/mautrix/util/async_iter_bytes.py b/mautrix/util/async_iter_bytes.py deleted file mode 100644 index 088baed4..00000000 --- a/mautrix/util/async_iter_bytes.py +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright (c) 2023 Tulir Asokan -# -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import annotations - -from typing import AsyncGenerator, Union - -AsyncBody = AsyncGenerator[Union[bytes, bytearray, memoryview], None] - - -async def async_iter_bytes(data: bytearray | bytes, chunk_size: int = 1024**2) -> AsyncBody: - """ - Return memory views into a byte array in chunks. This is used to prevent aiohttp from copying - the entire request body. - - Args: - data: The underlying data to iterate through. - chunk_size: How big each returned chunk should be. - - Returns: - An async generator that yields the given data in chunks. - """ - with memoryview(data) as mv: - for i in range(0, len(data), chunk_size): - yield mv[i : i + chunk_size] - - -__all__ = ["AsyncBody", "async_iter_bytes"] From e0baf7c0e9b7ee5d633ed9ebb066e5853ad5ed52 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 11 Jan 2023 17:10:36 +0200 Subject: [PATCH 034/218] Bump latest version in asyncpg crypto store v6 is only for existing databases so it can be skipped for new ones --- mautrix/crypto/store/asyncpg/upgrade.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/crypto/store/asyncpg/upgrade.py b/mautrix/crypto/store/asyncpg/upgrade.py index f7f50a74..7a55c10d 100644 --- a/mautrix/crypto/store/asyncpg/upgrade.py +++ b/mautrix/crypto/store/asyncpg/upgrade.py @@ -16,7 +16,7 @@ ) -@upgrade_table.register(description="Latest revision", upgrades_to=5) +@upgrade_table.register(description="Latest revision", upgrades_to=6) async def upgrade_blank_to_v4(conn: Connection) -> None: await conn.execute( """CREATE TABLE IF NOT EXISTS crypto_account ( From 62b515e718edccf2838bd879fa71fb75cb8746c3 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 14 Jan 2023 14:16:39 +0200 Subject: [PATCH 035/218] Fix external URL upload retry loop --- mautrix/client/api/modules/media_repository.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mautrix/client/api/modules/media_repository.py b/mautrix/client/api/modules/media_repository.py index f91ba456..e516fbd0 100644 --- a/mautrix/client/api/modules/media_repository.py +++ b/mautrix/client/api/modules/media_repository.py @@ -304,7 +304,7 @@ async def _upload_to_url( ) upload_response.raise_for_status() except Exception as e: - if retry_count == 0: + if retry_count <= 0: raise make_request_error( http_status=upload_response.status if upload_response else -1, text=(await upload_response.text()) if upload_response else "", @@ -317,7 +317,7 @@ async def _upload_to_url( ) await asyncio.sleep(backoff) backoff *= 2 - retry_count = -1 + retry_count -= 1 else: break From 95f61b9dcc4c64c9aab59a5017e88bd53324a76b Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 14 Jan 2023 14:18:01 +0200 Subject: [PATCH 036/218] Reduce base backoff in external URL uploads --- mautrix/client/api/modules/media_repository.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/client/api/modules/media_repository.py b/mautrix/client/api/modules/media_repository.py index e516fbd0..f247285a 100644 --- a/mautrix/client/api/modules/media_repository.py +++ b/mautrix/client/api/modules/media_repository.py @@ -290,7 +290,7 @@ async def _upload_to_url( min_iter_size: int = 25 * 1024 * 1024, ) -> None: retry_count = self.api.default_retry_count - backoff = 4 + backoff = 2 do_fake_iter = data and hasattr(data, "__len__") and len(data) > min_iter_size if do_fake_iter: headers["Content-Length"] = str(len(data)) From e808bea4774ff5035ac57a18d30ee1b4016662c5 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 14 Jan 2023 14:20:09 +0200 Subject: [PATCH 037/218] Update changelog --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bd69c304..a11d8f56 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## v0.19.2 (unreleased) + +* *(util.async_body)* Added utility for reading aiohttp response into a bytearray + (so that the output is mutable, e.g. for decrypting or encrypting media). +* *(client.api)* Fixed retry loop for MSC3870 URL uploads not exiting properly + after too many errors. + ## v0.19.1 (2023-01-11) * Marked Python 3.11 as supported. Python 3.8 support will likely be dropped in From f4b1dfbf2f7a4fdbe441424284b68d1cca57b404 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 14 Jan 2023 14:25:48 +0200 Subject: [PATCH 038/218] Bump version to 0.19.2 --- CHANGELOG.md | 2 +- mautrix/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a11d8f56..84b027db 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -## v0.19.2 (unreleased) +## v0.19.2 (2023-01-14) * *(util.async_body)* Added utility for reading aiohttp response into a bytearray (so that the output is mutable, e.g. for decrypting or encrypting media). diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 224a35ca..67d8ba2e 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.19.1" +__version__ = "0.19.2" __author__ = "Tulir Asokan " __all__ = [ "api", From 6d261cc5c004f7fee7d2ed5c5b526b61b4f5b0ff Mon Sep 17 00:00:00 2001 From: Sumner Evans Date: Thu, 19 Jan 2023 11:32:07 -0700 Subject: [PATCH 039/218] checkpoints: add delivery failed Signed-off-by: Sumner Evans --- mautrix/util/message_send_checkpoint.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mautrix/util/message_send_checkpoint.py b/mautrix/util/message_send_checkpoint.py index 61eb691d..5dd023b5 100644 --- a/mautrix/util/message_send_checkpoint.py +++ b/mautrix/util/message_send_checkpoint.py @@ -29,6 +29,7 @@ class MessageSendCheckpointStatus(SerializableEnum): PERM_FAILURE = "PERM_FAILURE" UNSUPPORTED = "UNSUPPORTED" TIMEOUT = "TIMEOUT" + DELIVERY_FAILED = "DELIVERY_FAILED" class MessageSendCheckpointReportedBy(SerializableEnum): From dd3e11860471e55cd97a5174cd792c04001dd65a Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 24 Jan 2023 16:46:50 +0200 Subject: [PATCH 040/218] Bump maximum delay for receiving decryption keys --- mautrix/bridge/matrix.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mautrix/bridge/matrix.py b/mautrix/bridge/matrix.py index 144264aa..ffd2eb9a 100644 --- a/mautrix/bridge/matrix.py +++ b/mautrix/bridge/matrix.py @@ -583,7 +583,6 @@ async def _send_crypto_status_error( error=msg, message=err.human_message if err else None, ) - status_content.fill_legacy_booleans() await self.az.intent.send_message_event( evt.room_id, EventType.BEEPER_MESSAGE_STATUS, status_content ) @@ -784,7 +783,7 @@ async def handle_encrypted(self, evt: EncryptedEvent) -> None: try: decrypted = await self.e2ee.decrypt(evt, wait_session_timeout=3) except SessionNotFound as e: - await self._handle_encrypted_wait(evt, e, wait=6) + await self._handle_encrypted_wait(evt, e, wait=22) except DecryptionError as e: self.log.warning(f"Failed to decrypt {evt.event_id}: {e}") self.log.trace("%s decryption traceback:", evt.event_id, exc_info=True) From 0c5debaebc678c05c52090ffb2f3d66b19141918 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 26 Jan 2023 00:36:04 +0200 Subject: [PATCH 041/218] Deduplicate bridge states more --- mautrix/util/bridge_state.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mautrix/util/bridge_state.py b/mautrix/util/bridge_state.py index b7c346f0..dcb42977 100644 --- a/mautrix/util/bridge_state.py +++ b/mautrix/util/bridge_state.py @@ -62,8 +62,8 @@ class BridgeStateEvent(SerializableEnum): class BridgeState(SerializableAttrs): human_readable_errors: ClassVar[Dict[Optional[str], str]] = {} default_source: ClassVar[str] = "bridge" - default_error_ttl: ClassVar[int] = 60 - default_ok_ttl: ClassVar[int] = 240 + default_error_ttl: ClassVar[int] = 3600 + default_ok_ttl: ClassVar[int] = 21600 state_event: BridgeStateEvent user_id: Optional[UserID] = None @@ -106,8 +106,8 @@ def should_deduplicate(self, prev_state: Optional["BridgeState"]) -> bool: ): # If there's no previous state or the state was different, send this one. return False - # If there's more than ⅘ of the previous pong's time-to-live left, drop this one - return prev_state.timestamp + (prev_state.ttl / 5) > self.timestamp + # If the previous state is recent, drop this one + return prev_state.timestamp + prev_state.ttl > self.timestamp async def send(self, url: str, token: str, log: logging.Logger, log_sent: bool = True) -> bool: if not url: From fa9f9fd294eab84ac654b114078186563f9633cc Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 27 Jan 2023 15:07:37 +0200 Subject: [PATCH 042/218] Bump version to 0.19.3 --- CHANGELOG.md | 4 ++++ mautrix/__init__.py | 2 +- optional-requirements.txt | 2 +- setup.py | 2 +- 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 84b027db..1a94abdf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## v0.19.3 (2023-01-27) + +* *(bridge)* Bumped default timeouts for decrypting incoming messages. + ## v0.19.2 (2023-01-14) * *(util.async_body)* Added utility for reading aiohttp response into a bytearray diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 67d8ba2e..4a89ae2f 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.19.2" +__version__ = "0.19.3" __author__ = "Tulir Asokan " __all__ = [ "api", diff --git a/optional-requirements.txt b/optional-requirements.txt index 6cfdcece..a5660f4a 100644 --- a/optional-requirements.txt +++ b/optional-requirements.txt @@ -1,6 +1,6 @@ python-magic ruamel.yaml -SQLAlchemy +SQLAlchemy<2 commonmark lxml asyncpg diff --git a/setup.py b/setup.py index 4b3c9f78..cabdf15a 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ from mautrix import __version__ encryption_dependencies = ["python-olm", "unpaddedbase64", "pycryptodome"] -test_dependencies = ["aiosqlite", "sqlalchemy", "asyncpg", *encryption_dependencies] +test_dependencies = ["aiosqlite", "sqlalchemy<2", "asyncpg", *encryption_dependencies] setuptools.setup( name="mautrix", From 6b3b17c5d9a4203eed2a847c21c4b9c908817d5c Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 31 Jan 2023 16:32:48 +0200 Subject: [PATCH 043/218] Use target event thread parent if event is already in thread Previously set_thread_parent didn't work correctly if you passed an event that's already in a thread and not the root. --- mautrix/types/event/message.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/mautrix/types/event/message.py b/mautrix/types/event/message.py index eab4ba95..47c4e2e6 100644 --- a/mautrix/types/event/message.py +++ b/mautrix/types/event/message.py @@ -112,6 +112,12 @@ def set_thread_parent( self.relates_to.event_id = ( thread_parent if isinstance(thread_parent, str) else thread_parent.event_id ) + if isinstance(thread_parent, MessageEvent) and isinstance( + thread_parent.content, BaseMessageEventContentFuncs + ): + self.relates_to.event_id = ( + thread_parent.content.get_thread_parent() or self.relates_to.event_id + ) if not disable_reply_fallback: self.set_reply(last_event_in_thread or thread_parent, **kwargs) self.relates_to.is_falling_back = True From 98140aa4b3168bbe64e3dc3278e6a11057e94536 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 11 Feb 2023 22:32:18 +0200 Subject: [PATCH 044/218] Update linters --- .github/workflows/python-package.yml | 2 +- .pre-commit-config.yaml | 6 +++--- dev-requirements.txt | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 7c2264b4..96b55773 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -57,7 +57,7 @@ jobs: - uses: psf/black@stable with: src: "./mautrix" - version: "22.12.0" + version: "23.1.0" - name: pre-commit run: | pip install pre-commit diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 69af4a5a..77beb7ac 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.3.0 + rev: v4.4.0 hooks: - id: trailing-whitespace exclude_types: [markdown] @@ -8,13 +8,13 @@ repos: - id: check-yaml - id: check-added-large-files - repo: https://github.com/psf/black - rev: 22.12.0 + rev: 23.1.0 hooks: - id: black language_version: python3 files: ^mautrix/.*\.pyi?$ - repo: https://github.com/PyCQA/isort - rev: 5.11.4 + rev: 5.12.0 hooks: - id: isort files: ^mautrix/.*\.pyi?$ diff --git a/dev-requirements.txt b/dev-requirements.txt index e513c0df..5cd14c23 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,3 +1,3 @@ pre-commit>=2.10.1,<3 isort>=5.10.1,<6 -black>=22.3,<23 +black>=23,<24 From 4c7503d2533de00ab37d20a9c2ba3bb736fdddfc Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 11 Feb 2023 22:36:11 +0200 Subject: [PATCH 045/218] Add wrapper for creating background tasks The wrapper ensures that the reference to the task is not lost before it completes, and also logs uncaught errors. --- mautrix/appservice/as_handler.py | 3 +- mautrix/bridge/custom_puppet.py | 3 +- mautrix/bridge/matrix.py | 8 +-- mautrix/bridge/portal.py | 7 +-- mautrix/bridge/user.py | 3 +- .../client/api/modules/media_repository.py | 3 +- mautrix/crypto/decrypt_olm.py | 7 +-- mautrix/crypto/machine.py | 3 +- mautrix/util/__init__.py | 1 + mautrix/util/background_task.py | 53 +++++++++++++++++++ 10 files changed, 76 insertions(+), 15 deletions(-) create mode 100644 mautrix/util/background_task.py diff --git a/mautrix/appservice/as_handler.py b/mautrix/appservice/as_handler.py index 88715100..78cd4307 100644 --- a/mautrix/appservice/as_handler.py +++ b/mautrix/appservice/as_handler.py @@ -27,6 +27,7 @@ SerializerError, UserID, ) +from mautrix.util import background_task HandlerFunc = Callable[[Event], Awaitable] @@ -314,7 +315,7 @@ async def try_handle(handler_func: HandlerFunc): for handler in self.event_handlers: # TODO add option to handle events synchronously - asyncio.create_task(try_handle(handler)) + background_task.create(try_handle(handler)) def matrix_event_handler(self, func: HandlerFunc) -> HandlerFunc: self.event_handlers.append(func) diff --git a/mautrix/bridge/custom_puppet.py b/mautrix/bridge/custom_puppet.py index 3d5c7f9e..e2759513 100644 --- a/mautrix/bridge/custom_puppet.py +++ b/mautrix/bridge/custom_puppet.py @@ -41,6 +41,7 @@ SyncToken, UserID, ) +from mautrix.util import background_task from .. import bridge as br @@ -409,7 +410,7 @@ def _handle_sync(self, sync_resp: dict) -> None: # Deserialize and handle all events for event in chain(ephemeral_events, presence_events): - asyncio.create_task(self.mx.try_handle_sync_event(Event.deserialize(event))) + background_task.create(self.mx.try_handle_sync_event(Event.deserialize(event))) async def _try_sync(self) -> None: try: diff --git a/mautrix/bridge/matrix.py b/mautrix/bridge/matrix.py index ffd2eb9a..1ba2ac7c 100644 --- a/mautrix/bridge/matrix.py +++ b/mautrix/bridge/matrix.py @@ -56,7 +56,7 @@ Version, VersionsResponse, ) -from mautrix.util import markdown +from mautrix.util import background_task, markdown from mautrix.util.logging import TraceLogger from mautrix.util.message_send_checkpoint import ( CHECKPOINT_TYPES, @@ -382,7 +382,7 @@ async def handle_puppet_nonportal_invite( members = await intent.get_room_members(room_id) except MatrixError: self.log.exception(f"Failed to get state after joining {room_id} as {intent.mxid}") - asyncio.create_task(intent.leave_room(room_id, reason="Internal error")) + background_task.create(intent.leave_room(room_id, reason="Internal error")) return if create_evt.type == RoomType.SPACE: await self.handle_puppet_space_invite(room_id, puppet, invited_by, evt) @@ -798,7 +798,7 @@ async def _handle_encrypted_wait( f"Couldn't find session {err.session_id} trying to decrypt {evt.event_id}," " waiting even longer" ) - asyncio.create_task( + background_task.create( self.e2ee.crypto.request_room_key( evt.room_id, evt.content.sender_key, @@ -875,7 +875,7 @@ def _send_message_checkpoint( info=str(err) if err else None, retry_num=retry_num, ) - asyncio.create_task(checkpoint.send(endpoint, self.az.as_token, self.log)) + background_task.create(checkpoint.send(endpoint, self.az.as_token, self.log)) allowed_event_classes: tuple[type, ...] = ( MessageEvent, diff --git a/mautrix/bridge/portal.py b/mautrix/bridge/portal.py index f51a0ae8..1a6a1343 100644 --- a/mautrix/bridge/portal.py +++ b/mautrix/bridge/portal.py @@ -30,6 +30,7 @@ TextMessageEventContent, UserID, ) +from mautrix.util import background_task from mautrix.util.logging import TraceLogger from mautrix.util.simple_lock import SimpleLock @@ -402,7 +403,7 @@ async def restart_scheduled_disappearing(cls) -> None: for msg in msgs: portal = await cls.bridge.get_portal(msg.room_id) if portal and portal.mxid: - asyncio.create_task(portal._disappear_event(msg)) + background_task.create(portal._disappear_event(msg)) else: await msg.delete() @@ -418,7 +419,7 @@ async def schedule_disappearing(self) -> None: for msg in msgs: msg.start_timer() await msg.update() - asyncio.create_task(self._disappear_event(msg)) + background_task.create(self._disappear_event(msg)) async def _send_message( self, @@ -431,7 +432,7 @@ async def _send_message( event_type, content = await self.matrix.e2ee.encrypt(self.mxid, event_type, content) event_id = await intent.send_message_event(self.mxid, event_type, content, **kwargs) if intent.api.is_real_user: - asyncio.create_task(intent.mark_read(self.mxid, event_id)) + background_task.create(intent.mark_read(self.mxid, event_id)) return event_id @property diff --git a/mautrix/bridge/user.py b/mautrix/bridge/user.py index bea6bdf7..66860c04 100644 --- a/mautrix/bridge/user.py +++ b/mautrix/bridge/user.py @@ -16,6 +16,7 @@ from mautrix.appservice import AppService from mautrix.errors import MNotFound from mautrix.types import EventID, EventType, Membership, MessageType, RoomID, UserID +from mautrix.util import background_task from mautrix.util.bridge_state import BridgeState, BridgeStateEvent from mautrix.util.logging import TraceLogger from mautrix.util.message_send_checkpoint import ( @@ -244,7 +245,7 @@ def send_remote_checkpoint( """ if not self.bridge.config["homeserver.message_send_checkpoint_endpoint"]: return WrappedTask(task=None) - task = asyncio.create_task( + task = background_task.create( MessageSendCheckpoint( event_id=event_id, room_id=room_id, diff --git a/mautrix/client/api/modules/media_repository.py b/mautrix/client/api/modules/media_repository.py index f247285a..edf5e701 100644 --- a/mautrix/client/api/modules/media_repository.py +++ b/mautrix/client/api/modules/media_repository.py @@ -20,6 +20,7 @@ MXOpenGraph, SerializerError, ) +from mautrix.util import background_task from mautrix.util.async_body import async_iter_bytes from mautrix.util.opt_prometheus import Histogram @@ -157,7 +158,7 @@ async def _try_upload(): except Exception as e: self.log.error(f"Failed to upload {mxc}: {type(e).__name__}: {e}") - asyncio.create_task(_try_upload()) + background_task.create(_try_upload()) return mxc else: with self._observe_upload_time(size): diff --git a/mautrix/crypto/decrypt_olm.py b/mautrix/crypto/decrypt_olm.py index 8182b160..6eef76f5 100644 --- a/mautrix/crypto/decrypt_olm.py +++ b/mautrix/crypto/decrypt_olm.py @@ -19,6 +19,7 @@ ToDeviceEvent, UserID, ) +from mautrix.util import background_task from .base import BaseOlmMachine from .sessions import Session @@ -74,19 +75,19 @@ async def _decrypt_olm_ciphertext( f"Found matching session yet decryption failed for sender {sender}" f" with key {sender_key}" ) - asyncio.create_task(self._unwedge_session(sender, sender_key)) + background_task.create(self._unwedge_session(sender, sender_key)) raise if not plaintext: if message.type != OlmMsgType.PREKEY: - asyncio.create_task(self._unwedge_session(sender, sender_key)) + background_task.create(self._unwedge_session(sender, sender_key)) raise DecryptionError("Decryption failed for normal message") self.log.trace(f"Trying to create inbound session for {sender}/{sender_key}") try: session = await self._create_inbound_session(sender_key, message.body) except olm.OlmSessionError as e: - asyncio.create_task(self._unwedge_session(sender, sender_key)) + background_task.create(self._unwedge_session(sender, sender_key)) raise DecryptionError("Failed to create new session from prekey message") from e self.log.debug( f"Created inbound session {session.id} for {sender} (sender key: {sender_key})" diff --git a/mautrix/crypto/machine.py b/mautrix/crypto/machine.py index a1fd4e16..3e845b44 100644 --- a/mautrix/crypto/machine.py +++ b/mautrix/crypto/machine.py @@ -24,6 +24,7 @@ TrustState, UserID, ) +from mautrix.util import background_task from mautrix.util.logging import TraceLogger from .account import OlmAccount @@ -109,7 +110,7 @@ async def handle_as_otk_counts( self.log.warning(f"Got OTK count for unknown device {user_id}/{device_id}") async def handle_as_device_lists(self, device_lists: DeviceLists) -> None: - asyncio.create_task(self.handle_device_lists(device_lists)) + background_task.create(self.handle_device_lists(device_lists)) async def handle_as_to_device_event(self, evt: ASToDeviceEvent) -> None: if evt.to_user_id != self.client.mxid or evt.to_device_id != self.client.device_id: diff --git a/mautrix/util/__init__.py b/mautrix/util/__init__.py index e731dfa4..fd349bef 100644 --- a/mautrix/util/__init__.py +++ b/mautrix/util/__init__.py @@ -8,6 +8,7 @@ # File modules "async_body", "async_getter_lock", + "background_task", "bridge_state", "color_log", "ffmpeg", diff --git a/mautrix/util/background_task.py b/mautrix/util/background_task.py new file mode 100644 index 00000000..e22e74f1 --- /dev/null +++ b/mautrix/util/background_task.py @@ -0,0 +1,53 @@ +# Copyright (c) 2023 Tulir Asokan +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +from __future__ import annotations + +from typing import Coroutine +import asyncio +import logging + +_tasks = set() +log = logging.getLogger("mau.background_task") + + +async def catch(coro: Coroutine, caller: str) -> None: + try: + await coro + except Exception: + log.exception(f"Uncaught error in background task (created in {caller})") + + +# Logger.findCaller finds the 3rd stack frame, so add an intermediate function +# to get the caller of create(). +def _find_caller() -> tuple[str, int, str, None]: + return log.findCaller() + + +def create(coro: Coroutine, *, name: str | None = None, catch_errors: bool = True) -> asyncio.Task: + """ + Create a background asyncio task safely, ensuring a reference is kept until the task completes. + It also catches and logs uncaught errors (unless disabled via the parameter). + + Args: + coro: The coroutine to wrap in a task and execute. + name: An optional name for the created task. + catch_errors: Should the task be wrapped in a try-except block to log any uncaught errors? + + Returns: + An asyncio Task object wrapping the given coroutine. + """ + if catch_errors: + try: + file_name, line_number, function_name, _ = _find_caller() + caller = f"{function_name} at {file_name}:{line_number}" + except ValueError: + caller = "unknown function" + task = asyncio.create_task(catch(coro, caller), name=name) + else: + task = asyncio.create_task(coro, name=name) + _tasks.add(task) + task.add_done_callback(_tasks.discard) + return task From 94a258c3b379485c0a4a299fa0eb2dae9694350f Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 12 Feb 2023 12:17:47 +0200 Subject: [PATCH 046/218] Bump version to 0.19.4 --- CHANGELOG.md | 9 +++++++++ mautrix/__init__.py | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a94abdf..70304b94 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,12 @@ +## v0.19.4 (2023-02-12) + +* *(types)* Changed `set_thread_parent` to inherit the existing thread parent + if a `MessageEvent` is passed, as starting threads from a message in a thread + is not allowed. +* *(util.background_task)* Added new utility for creating background tasks + safely, by ensuring that the task is not garbage collected before finishing + and logging uncaught exceptions immediately. + ## v0.19.3 (2023-01-27) * *(bridge)* Bumped default timeouts for decrypting incoming messages. diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 4a89ae2f..02a72acd 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.19.3" +__version__ = "0.19.4" __author__ = "Tulir Asokan " __all__ = [ "api", From 1410a3a9e1545ecd8c79254fefec7fbe2b8d7a23 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 15 Feb 2023 21:54:52 +0200 Subject: [PATCH 047/218] Fix usages of log.warn --- mautrix/util/ffmpeg.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mautrix/util/ffmpeg.py b/mautrix/util/ffmpeg.py index 7d195c43..41cebf32 100644 --- a/mautrix/util/ffmpeg.py +++ b/mautrix/util/ffmpeg.py @@ -84,7 +84,7 @@ async def probe_path( err_text = stderr.decode("utf-8") if stderr else f"unknown ({proc.returncode})" raise ConverterError(f"ffprobe error: {err_text}") elif stderr and logger: - logger.warn(f"ffprobe warning: {stderr.decode('utf-8')}") + logger.warning(f"ffprobe warning: {stderr.decode('utf-8')}") return json.loads(stdout) @@ -182,7 +182,7 @@ async def convert_path( err_text = stderr.decode("utf-8") if stderr else f"unknown ({proc.returncode})" raise ConverterError(f"ffmpeg error: {err_text}") elif stderr and logger: - logger.warn(f"ffmpeg warning: {stderr.decode('utf-8')}") + logger.warning(f"ffmpeg warning: {stderr.decode('utf-8')}") if remove_input and isinstance(input_file, Path): input_file.unlink(missing_ok=True) return stdout if output_file == "-" else output_file From 41608a8ba9c7ffe636bd20e5d1ccbbb5c6ae2385 Mon Sep 17 00:00:00 2001 From: Malte E Date: Mon, 20 Feb 2023 15:01:09 +0100 Subject: [PATCH 048/218] block parallel invites to the same room --- mautrix/bridge/matrix.py | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/mautrix/bridge/matrix.py b/mautrix/bridge/matrix.py index 1ba2ac7c..08382abc 100644 --- a/mautrix/bridge/matrix.py +++ b/mautrix/bridge/matrix.py @@ -5,6 +5,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import annotations +from collections import defaultdict import asyncio import logging import sys @@ -143,6 +144,7 @@ class BaseMatrixHandler: media_config: MediaRepoConfig versions: VersionsResponse minimum_spec_version: Version = SpecVersions.V11 + room_locks: dict[str, asyncio.Lock] user_id_prefix: str user_id_suffix: str @@ -159,6 +161,7 @@ def __init__( self.media_config = MediaRepoConfig(upload_size=50 * 1024 * 1024) self.versions = VersionsResponse.deserialize({"versions": ["v1.3"]}) self.az.matrix_event_handler(self.int_handle_event) + self.room_locks = defaultdict(asyncio.Lock) self.e2ee = None self.require_e2ee = False @@ -400,19 +403,20 @@ async def handle_puppet_invite( await intent.leave_room(room_id, reason="You're not allowed to invite this ghost.") return - portal = await self.bridge.get_portal(room_id) - if portal: - try: - await portal.handle_matrix_invite(invited_by, puppet) - except br.RejectMatrixInvite as e: - await intent.leave_room(room_id, reason=e.message) - except br.IgnoreMatrixInvite: - pass + async with self.room_locks[room_id]: + portal = await self.bridge.get_portal(room_id) + if portal: + try: + await portal.handle_matrix_invite(invited_by, puppet) + except br.RejectMatrixInvite as e: + await intent.leave_room(room_id, reason=e.message) + except br.IgnoreMatrixInvite: + pass + else: + await intent.join_room(room_id) + return else: - await intent.join_room(room_id) - return - else: - await self.handle_puppet_nonportal_invite(room_id, puppet, invited_by, evt) + await self.handle_puppet_nonportal_invite(room_id, puppet, invited_by, evt) async def handle_invite( self, room_id: RoomID, user_id: UserID, invited_by: br.BaseUser, evt: StateEvent From 06adf2f6d71b80b072eb709ab149d7dfb15e4064 Mon Sep 17 00:00:00 2001 From: Nick Mills-Barrett Date: Thu, 2 Mar 2023 19:30:27 +0400 Subject: [PATCH 049/218] Add proxy util from IG/FB bridges (#134) * Add proxy util from IG/FB bridges * Add comment about blocking request --- mautrix/util/proxy.py | 113 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100644 mautrix/util/proxy.py diff --git a/mautrix/util/proxy.py b/mautrix/util/proxy.py new file mode 100644 index 00000000..c73fd81f --- /dev/null +++ b/mautrix/util/proxy.py @@ -0,0 +1,113 @@ +from __future__ import annotations + +from typing import Awaitable, Callable, TypeVar +import asyncio +import json +import logging +import urllib.request + +from aiohttp import ClientConnectionError +from yarl import URL + +from mautrix.util.logging import TraceLogger + +try: + from aiohttp_socks import ProxyConnectionError, ProxyError, ProxyTimeoutError +except ImportError: + + class ProxyError(Exception): + pass + + ProxyConnectionError = ProxyTimeoutError = ProxyError + +RETRYABLE_PROXY_EXCEPTIONS = ( + ProxyError, + ProxyTimeoutError, + ProxyConnectionError, + ClientConnectionError, + ConnectionError, + asyncio.TimeoutError, +) + + +class ProxyHandler: + current_proxy_url: str | None = None + log = logging.getLogger("mau.proxy") + + def __init__(self, api_url: str | None) -> None: + self.api_url = api_url + + def get_proxy_url_from_api(self, reason: str | None = None) -> str | None: + assert self.api_url is not None + + api_url = str(URL(self.api_url).update_query({"reason": reason} if reason else {})) + + # NOTE: using urllib.request to intentionally block the whole bridge until the proxy change applied + request = urllib.request.Request(api_url, method="GET") + self.log.debug("Requesting proxy from: %s", api_url) + + try: + with urllib.request.urlopen(request) as f: + response = json.loads(f.read().decode()) + except Exception: + self.log.exception("Failed to retrieve proxy from API") + else: + return response["proxy_url"] + + return None + + def update_proxy_url(self, reason: str | None = None) -> bool: + old_proxy = self.current_proxy_url + new_proxy = None + + if self.api_url is not None: + new_proxy = self.get_proxy_url_from_api(reason) + else: + new_proxy = urllib.request.getproxies().get("http") + + if old_proxy != new_proxy: + self.log.debug("Set new proxy URL: %s", new_proxy) + self.current_proxy_url = new_proxy + return True + + self.log.debug("Got same proxy URL: %s", new_proxy) + return False + + def get_proxy_url(self) -> str | None: + if not self.current_proxy_url: + self.update_proxy_url() + + return self.current_proxy_url + + +T = TypeVar("T") + + +async def proxy_with_retry( + name: str, + func: Callable[[], Awaitable[T]], + logger: TraceLogger, + proxy_handler: ProxyHandler, + on_proxy_change: Callable[[], Awaitable[None]], + max_retries: int = 10, +) -> T: + errors = 0 + + while True: + try: + return await func() + except RETRYABLE_PROXY_EXCEPTIONS as e: + errors += 1 + if errors > max_retries: + raise + wait = min(errors * 10, 60) + logger.warning( + "%s while trying to %s, retrying in %d seconds", + e.__class__.__name__, + name, + wait, + ) + if errors > 1 and proxy_handler.update_proxy_url( + f"{e.__class__.__name__} while trying to {name}" + ): + await on_proxy_change() From 090f33361fd7925331c6a3f71a70317ae647d394 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 4 Mar 2023 14:57:49 +0200 Subject: [PATCH 050/218] Add default value for max upload size. Closes #133 --- mautrix/types/media.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/types/media.py b/mautrix/types/media.py index 72aa6c4f..0b277382 100644 --- a/mautrix/types/media.py +++ b/mautrix/types/media.py @@ -20,7 +20,7 @@ class MediaRepoConfig(SerializableAttrs): https://spec.matrix.org/v1.2/client-server-api/#get_matrixmediav3config """ - upload_size: int = field(json="m.upload.size") + upload_size: int = field(default=50 * 1024 * 1024, json="m.upload.size") @dataclass From eee6d1b599c79d0964815293074fbf2287e09fd6 Mon Sep 17 00:00:00 2001 From: Nick Mills-Barrett Date: Tue, 7 Mar 2023 15:37:21 +0400 Subject: [PATCH 051/218] More `proxy_with_retry` helper arguments (#136) * Add `min_wait_seconds` argument to `proxy_with_retry` helper * Add `retryable_exceptions` argument to `proxy_with_retry` helper This allows non-aiohttp calls to use the helper to retry proxy related requests. * Stricter type for `retryable_exceptions` Co-authored-by: Tulir Asokan --------- Co-authored-by: Tulir Asokan --- mautrix/util/proxy.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mautrix/util/proxy.py b/mautrix/util/proxy.py index c73fd81f..57364c2c 100644 --- a/mautrix/util/proxy.py +++ b/mautrix/util/proxy.py @@ -90,17 +90,19 @@ async def proxy_with_retry( proxy_handler: ProxyHandler, on_proxy_change: Callable[[], Awaitable[None]], max_retries: int = 10, + min_wait_seconds: int = 60, + retryable_exceptions: tuple[Exception] = RETRYABLE_PROXY_EXCEPTIONS, ) -> T: errors = 0 while True: try: return await func() - except RETRYABLE_PROXY_EXCEPTIONS as e: + except retryable_exceptions as e: errors += 1 if errors > max_retries: raise - wait = min(errors * 10, 60) + wait = min(errors * 10, min_wait_seconds) logger.warning( "%s while trying to %s, retrying in %d seconds", e.__class__.__name__, From 741995e17ada6c18c539059e132aec0861d942ec Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 7 Mar 2023 14:15:43 +0200 Subject: [PATCH 052/218] Bump version to 0.19.5 --- CHANGELOG.md | 10 ++++++++++ mautrix/__init__.py | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 70304b94..f1e0a13c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,13 @@ +## v0.19.5 (2023-03-07) + +* *(util.proxy)* Added utility for dynamic proxies (from mautrix-instagram/facebook). +* *(types)* Added default value for `upload_size` in `MediaRepoConfig` as the + field is optional in the spec. +* *(bridge)* Changed ghost invite handling to only process one per room at a time + (thanks to [@maltee1] in [#132]). + +[#132]: https://github.com/mautrix/python/pull/132 + ## v0.19.4 (2023-02-12) * *(types)* Changed `set_thread_parent` to inherit the existing thread parent diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 02a72acd..2bf008d5 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.19.4" +__version__ = "0.19.5" __author__ = "Tulir Asokan " __all__ = [ "api", From 03f54e2be035c5ec711ac7a72ee7130cc0b9fa68 Mon Sep 17 00:00:00 2001 From: Nick Mills-Barrett Date: Sat, 11 Mar 2023 18:28:38 +0400 Subject: [PATCH 053/218] Fix wait second handling in `proxy_with_retry` (#137) * Fix wait second handling in `proxy_with_retry` Adds new argument `max_wait_seconds` which actually was the broken behaviour of the previous `min_wait_seconds` which is now correctly handled. * Add `multiply_wait_seconds` argument --- mautrix/util/proxy.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/mautrix/util/proxy.py b/mautrix/util/proxy.py index 57364c2c..5edd6531 100644 --- a/mautrix/util/proxy.py +++ b/mautrix/util/proxy.py @@ -90,7 +90,9 @@ async def proxy_with_retry( proxy_handler: ProxyHandler, on_proxy_change: Callable[[], Awaitable[None]], max_retries: int = 10, - min_wait_seconds: int = 60, + min_wait_seconds: int = 0, + max_wait_seconds: int = 60, + multiply_wait_seconds: int = 10, retryable_exceptions: tuple[Exception] = RETRYABLE_PROXY_EXCEPTIONS, ) -> T: errors = 0 @@ -102,7 +104,9 @@ async def proxy_with_retry( errors += 1 if errors > max_retries: raise - wait = min(errors * 10, min_wait_seconds) + wait = errors * multiply_wait_seconds + wait = max(wait, min_wait_seconds) + wait = min(wait, max_wait_seconds) logger.warning( "%s while trying to %s, retrying in %d seconds", e.__class__.__name__, From 41b288a9dae85f42e8a62c53690a0ee1b3bca01b Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 13 Mar 2023 15:32:58 +0200 Subject: [PATCH 054/218] Log event ID and source in member change session invalidations --- mautrix/crypto/machine.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mautrix/crypto/machine.py b/mautrix/crypto/machine.py index 3e845b44..ed820c79 100644 --- a/mautrix/crypto/machine.py +++ b/mautrix/crypto/machine.py @@ -171,9 +171,11 @@ async def handle_member_event(self, evt: StateEvent) -> None: } if prev == cur or ignored_changes.get(prev) == cur: return + src = getattr(evt, "source", None) self.log.debug( f"Got membership state event in {evt.room_id} changing {evt.state_key} from " - f"{prev} to {cur}, invalidating group session" + f"{prev} to {cur} (event ID: {evt.event_id}, sync source: {src}), " + "invalidating group session" ) await self.crypto_store.remove_outbound_group_session(evt.room_id) From 72fb0f61f2a8cf8f062ddb8bacaa98ef4cbd0aa5 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 13 Mar 2023 15:50:28 +0200 Subject: [PATCH 055/218] Check cache before invalidating group sessions on member event --- mautrix/client/state_store/abstract.py | 3 +++ mautrix/crypto/machine.py | 13 +++++++++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/mautrix/client/state_store/abstract.py b/mautrix/client/state_store/abstract.py index e241d8f1..3d37087d 100644 --- a/mautrix/client/state_store/abstract.py +++ b/mautrix/client/state_store/abstract.py @@ -143,6 +143,9 @@ async def update_state(self, evt: StateEvent) -> None: if evt.type == EventType.ROOM_POWER_LEVELS: await self.set_power_levels(evt.room_id, evt.content) elif evt.type == EventType.ROOM_MEMBER: + evt.unsigned["mautrix_prev_membership"] = await self.get_member( + evt.room_id, UserID(evt.state_key) + ) await self.set_member(evt.room_id, UserID(evt.state_key), evt.content) elif evt.type == EventType.ROOM_ENCRYPTION: await self.set_encryption_info(evt.room_id, evt.content) diff --git a/mautrix/crypto/machine.py b/mautrix/crypto/machine.py index ed820c79..fe0ef3fa 100644 --- a/mautrix/crypto/machine.py +++ b/mautrix/crypto/machine.py @@ -18,6 +18,7 @@ DeviceOTKCount, EncryptionAlgorithm, EventType, + Member, Membership, StateEvent, ToDeviceEvent, @@ -172,10 +173,18 @@ async def handle_member_event(self, evt: StateEvent) -> None: if prev == cur or ignored_changes.get(prev) == cur: return src = getattr(evt, "source", None) + prev_cache = evt.unsigned.get("mautrix_prev_membership") + if isinstance(prev_cache, Member) and prev_cache.membership == cur: + self.log.debug( + f"Got duplicate membership state event in {evt.room_id} changing {evt.state_key} " + f"from {prev} to {cur}, cached state was {prev_cache} (event ID: {evt.event_id}, " + f"sync source: {src})" + ) + return self.log.debug( f"Got membership state event in {evt.room_id} changing {evt.state_key} from " - f"{prev} to {cur} (event ID: {evt.event_id}, sync source: {src}), " - "invalidating group session" + f"{prev} to {cur} (event ID: {evt.event_id}, sync source: {src}, " + f"cached: {prev_cache.membership}), invalidating group session" ) await self.crypto_store.remove_outbound_group_session(evt.room_id) From 177e5685660e45480907c8e62a633253ce880738 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 13 Mar 2023 15:50:36 +0200 Subject: [PATCH 056/218] Add extra check to cross signing key parser --- mautrix/types/crypto.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mautrix/types/crypto.py b/mautrix/types/crypto.py index 821599de..3bf7e96b 100644 --- a/mautrix/types/crypto.py +++ b/mautrix/types/crypto.py @@ -71,6 +71,8 @@ def first_ed25519_key(self) -> Optional[SigningKey]: return self.first_key_with_algorithm(EncryptionKeyAlgorithm.ED25519) def first_key_with_algorithm(self, alg: EncryptionKeyAlgorithm) -> Optional[SigningKey]: + if not self.keys: + return None try: return next(key for key_id, key in self.keys.items() if key_id.algorithm == alg) except StopIteration: From 718b7bb959b320707953dbb53f33fc5c0e1dfd55 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 13 Mar 2023 15:56:25 +0200 Subject: [PATCH 057/218] Add missing check --- mautrix/crypto/machine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/crypto/machine.py b/mautrix/crypto/machine.py index fe0ef3fa..f9884f7b 100644 --- a/mautrix/crypto/machine.py +++ b/mautrix/crypto/machine.py @@ -184,7 +184,7 @@ async def handle_member_event(self, evt: StateEvent) -> None: self.log.debug( f"Got membership state event in {evt.room_id} changing {evt.state_key} from " f"{prev} to {cur} (event ID: {evt.event_id}, sync source: {src}, " - f"cached: {prev_cache.membership}), invalidating group session" + f"cached: {prev_cache.membership if prev_cache else None}), invalidating group session" ) await self.crypto_store.remove_outbound_group_session(evt.room_id) From 04072cb2f4a00e6e1b02e16389720a94aad71f0f Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 13 Mar 2023 15:59:26 +0200 Subject: [PATCH 058/218] Bump version to 0.19.6 --- CHANGELOG.md | 7 +++++++ mautrix/__init__.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f1e0a13c..087ffefc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## v0.19.6 (2023-03-13) + +* *(crypto)* Added cache checks to prevent invalidating group session when the + server sends a duplicate member event in /sync. +* *(util.proxy)* Fixed `min_wait_seconds` behavior and added `max_wait_seconds` + and `multiply_wait_seconds` to `proxy_with_retry`. + ## v0.19.5 (2023-03-07) * *(util.proxy)* Added utility for dynamic proxies (from mautrix-instagram/facebook). diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 2bf008d5..3fd06e98 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.19.5" +__version__ = "0.19.6" __author__ = "Tulir Asokan " __all__ = [ "api", From 22d9125136ec2a3f577771c7c1f85f63a59da2ab Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 22 Mar 2023 22:07:27 +0200 Subject: [PATCH 059/218] Resolve trust when checking if key sharing is allowed --- mautrix/bridge/e2ee.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mautrix/bridge/e2ee.py b/mautrix/bridge/e2ee.py index 516e54bd..804792c1 100644 --- a/mautrix/bridge/e2ee.py +++ b/mautrix/bridge/e2ee.py @@ -132,9 +132,9 @@ async def allow_key_share(self, device: DeviceIdentity, request: RequestedKeyInf f"Rejecting key request from blacklisted device " f"{device.user_id}/{device.device_id}", code=RoomKeyWithheldCode.BLACKLISTED, - reason="You have been blacklisted by this device", + reason="Your device has been blacklisted by the bridge", ) - elif device.trust >= self.crypto.share_keys_min_trust: + elif await self.crypto.resolve_trust(device) >= self.crypto.share_keys_min_trust: portal = await self.bridge.get_portal(request.room_id) if portal is None: raise RejectKeyShare( @@ -161,7 +161,7 @@ async def allow_key_share(self, device: DeviceIdentity, request: RequestedKeyInf f"Rejecting key request from unverified device " f"{device.user_id}/{device.device_id}", code=RoomKeyWithheldCode.UNVERIFIED, - reason="You have not been verified by this device", + reason="Your device is not trusted by the bridge", ) def _ignore_user(self, user_id: str) -> bool: From 423d42a065e79d7c9607412e302f6821c6692354 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 22 Mar 2023 22:16:17 +0200 Subject: [PATCH 060/218] Also resolve trust in non-bridge key share check --- mautrix/crypto/key_share.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/crypto/key_share.py b/mautrix/crypto/key_share.py index 58525bbf..8d6b6e73 100644 --- a/mautrix/crypto/key_share.py +++ b/mautrix/crypto/key_share.py @@ -76,7 +76,7 @@ async def default_allow_key_share( code=RoomKeyWithheldCode.BLACKLISTED, reason="You have been blacklisted by this device", ) - elif device.trust >= self.share_keys_min_trust: + elif await self.resolve_trust(device) >= self.share_keys_min_trust: self.log.debug(f"Accepting key request from trusted device {device.device_id}") return True else: From 0b674872c228f70b41d6443f665515caee828463 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 22 Mar 2023 22:17:16 +0200 Subject: [PATCH 061/218] Bump version to 0.19.7 --- CHANGELOG.md | 5 +++++ mautrix/__init__.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 087ffefc..24192de4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +## v0.19.7 (2023-03-22) + +* *(bridge, crypto)* Fixed key sharing trust checker not resolving cross-signing + signatures when minimum trust level is set to cross-signed. + ## v0.19.6 (2023-03-13) * *(crypto)* Added cache checks to prevent invalidating group session when the diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 3fd06e98..53ed1518 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.19.6" +__version__ = "0.19.7" __author__ = "Tulir Asokan " __all__ = [ "api", From 4d4441b910da3f06b7ce8eadc6c859d60b977863 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 23 Mar 2023 16:27:22 +0200 Subject: [PATCH 062/218] Sync crypto store schema with mautrix-go --- mautrix/crypto/store/asyncpg/store.py | 13 +- mautrix/crypto/store/asyncpg/upgrade.py | 189 +++++++++++++++++++++--- mautrix/errors/__init__.py | 1 + mautrix/errors/crypto.py | 6 + 4 files changed, 188 insertions(+), 21 deletions(-) diff --git a/mautrix/crypto/store/asyncpg/store.py b/mautrix/crypto/store/asyncpg/store.py index 8609e12d..51985caf 100644 --- a/mautrix/crypto/store/asyncpg/store.py +++ b/mautrix/crypto/store/asyncpg/store.py @@ -12,6 +12,7 @@ from mautrix.client.state_store import SyncStore from mautrix.client.state_store.asyncpg import PgStateStore +from mautrix.errors import GroupSessionWithheldError from mautrix.types import ( CrossSigner, CrossSigningUsage, @@ -117,7 +118,7 @@ async def put_account(self, account: OlmAccount) -> None: await self.db.execute( q, self.account_id, - self._device_id, + self._device_id or "", account.shared, self._sync_token or "", pickle, @@ -236,6 +237,10 @@ async def put_group_session( INSERT INTO crypto_megolm_inbound_session ( session_id, sender_key, signing_key, room_id, session, forwarding_chains, account_id ) VALUES ($1, $2, $3, $4, $5, $6, $7) + ON CONFLICT (session_id, account_id) DO UPDATE + SET withheld_code=NULL, withheld_reason=NULL, sender_key=excluded.sender_key, + signing_key=excluded.signing_key, room_id=excluded.room_id, session=excluded.session, + forwarding_chains=excluded.forwarding_chains """ try: await self.db.execute( @@ -255,13 +260,15 @@ async def get_group_session( self, room_id: RoomID, session_id: SessionID ) -> InboundGroupSession | None: q = """ - SELECT sender_key, signing_key, session, forwarding_chains + SELECT sender_key, signing_key, session, forwarding_chains, withheld_code FROM crypto_megolm_inbound_session WHERE room_id=$1 AND session_id=$2 AND account_id=$3 """ row = await self.db.fetchrow(q, room_id, session_id, self.account_id) if row is None: return None + if row["withheld_code"] is not None: + raise GroupSessionWithheldError(session_id, row["withheld_code"]) forwarding_chain = row["forwarding_chains"].split(",") if row["forwarding_chains"] else [] return InboundGroupSession.from_pickle( row["session"], @@ -275,7 +282,7 @@ async def get_group_session( async def has_group_session(self, room_id: RoomID, session_id: SessionID) -> bool: q = """ SELECT COUNT(session) FROM crypto_megolm_inbound_session - WHERE room_id=$1 AND session_id=$2 AND account_id=$3 + WHERE room_id=$1 AND session_id=$2 AND account_id=$3 AND session IS NOT NULL """ count = await self.db.fetchval(q, room_id, session_id, self.account_id) return count > 0 diff --git a/mautrix/crypto/store/asyncpg/upgrade.py b/mautrix/crypto/store/asyncpg/upgrade.py index 7a55c10d..b0f2299c 100644 --- a/mautrix/crypto/store/asyncpg/upgrade.py +++ b/mautrix/crypto/store/asyncpg/upgrade.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022 Tulir Asokan +# Copyright (c) 2023 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this @@ -16,15 +16,15 @@ ) -@upgrade_table.register(description="Latest revision", upgrades_to=6) -async def upgrade_blank_to_v4(conn: Connection) -> None: +@upgrade_table.register(description="Latest revision", upgrades_to=8) +async def upgrade_blank_to_latest(conn: Connection) -> None: await conn.execute( """CREATE TABLE IF NOT EXISTS crypto_account ( - account_id TEXT PRIMARY KEY, - device_id TEXT, - shared BOOLEAN NOT NULL, - sync_token TEXT NOT NULL, - account bytea NOT NULL + account_id TEXT PRIMARY KEY, + device_id TEXT NOT NULL, + shared BOOLEAN NOT NULL, + sync_token TEXT NOT NULL, + account bytea NOT NULL )""" ) await conn.execute( @@ -68,16 +68,19 @@ async def upgrade_blank_to_v4(conn: Connection) -> None: ) await conn.execute( """CREATE TABLE IF NOT EXISTS crypto_megolm_inbound_session ( - account_id TEXT, - session_id CHAR(43), - sender_key CHAR(43) NOT NULL, - signing_key CHAR(43) NOT NULL, - room_id TEXT NOT NULL, - session bytea NOT NULL, - forwarding_chains TEXT NOT NULL, + account_id TEXT, + session_id CHAR(43), + sender_key CHAR(43) NOT NULL, + signing_key CHAR(43), + room_id TEXT NOT NULL, + session bytea, + forwarding_chains TEXT, + withheld_code TEXT, + withheld_reason TEXT, PRIMARY KEY (account_id, session_id) )""" ) + # TODO chnge max_age to BIGINT await conn.execute( """CREATE TABLE IF NOT EXISTS crypto_megolm_outbound_session ( account_id TEXT, @@ -97,8 +100,10 @@ async def upgrade_blank_to_v4(conn: Connection) -> None: """CREATE TABLE crypto_cross_signing_keys ( user_id TEXT, usage TEXT, - key CHAR(43), - first_seen_key CHAR(43), + key CHAR(43) NOT NULL, + + first_seen_key CHAR(43) NOT NULL, + PRIMARY KEY (user_id, usage) )""" ) @@ -108,7 +113,7 @@ async def upgrade_blank_to_v4(conn: Connection) -> None: signed_key TEXT, signer_user_id TEXT, signer_key TEXT, - signature TEXT, + signature CHAR(88) NOT NULL, PRIMARY KEY (signed_user_id, signed_key, signer_user_id, signer_key) )""" ) @@ -250,3 +255,151 @@ async def upgrade_v6(conn: Connection) -> None: await conn.execute("UPDATE crypto_device SET trust=300 WHERE trust=1") # verified await conn.execute("UPDATE crypto_device SET trust=-100 WHERE trust=2") # blacklisted await conn.execute("UPDATE crypto_device SET trust=0 WHERE trust=3") # ignored -> unset + + +@upgrade_table.register( + description="Synchronize schema with mautrix-go", upgrades_to=8, transaction=False +) +async def upgrade_v8(conn: Connection, scheme: Scheme) -> None: + if scheme == Scheme.POSTGRES: + async with conn.transaction(): + await upgrade_v8_postgres(conn) + else: + await upgrade_v8_sqlite(conn) + + +async def upgrade_v8_postgres(conn: Connection) -> None: + await conn.execute("UPDATE crypto_account SET device_id='' WHERE device_id IS NULL") + await conn.execute("ALTER TABLE crypto_account ALTER COLUMN device_id SET NOT NULL") + + await conn.execute( + "ALTER TABLE crypto_megolm_inbound_session ALTER COLUMN signing_key DROP NOT NULL" + ) + await conn.execute( + "ALTER TABLE crypto_megolm_inbound_session ALTER COLUMN session DROP NOT NULL" + ) + await conn.execute( + "ALTER TABLE crypto_megolm_inbound_session ALTER COLUMN forwarding_chains DROP NOT NULL" + ) + await conn.execute("ALTER TABLE crypto_megolm_inbound_session ADD COLUMN withheld_code TEXT") + await conn.execute("ALTER TABLE crypto_megolm_inbound_session ADD COLUMN withheld_reason TEXT") + + await conn.execute("DELETE FROM crypto_cross_signing_keys WHERE key IS NULL") + await conn.execute( + "UPDATE crypto_cross_signing_keys SET first_seen_key=key WHERE first_seen_key IS NULL" + ) + await conn.execute("ALTER TABLE crypto_cross_signing_keys ALTER COLUMN key SET NOT NULL") + await conn.execute( + "ALTER TABLE crypto_cross_signing_keys ALTER COLUMN first_seen_key SET NOT NULL" + ) + + await conn.execute("DELETE FROM crypto_cross_signing_signatures WHERE signature IS NULL") + await conn.execute( + "ALTER TABLE crypto_cross_signing_signatures ALTER COLUMN signature SET NOT NULL" + ) + + +async def upgrade_v8_sqlite(conn: Connection) -> None: + await conn.execute("PRAGMA foreign_keys = OFF") + async with conn.transaction(): + await conn.execute( + """CREATE TABLE new_crypto_account ( + account_id TEXT PRIMARY KEY, + device_id TEXT NOT NULL, + shared BOOLEAN NOT NULL, + sync_token TEXT NOT NULL, + account bytea NOT NULL + )""" + ) + await conn.execute( + """ + INSERT INTO new_crypto_account (account_id, device_id, shared, sync_token, account) + SELECT account_id, COALESCE(device_id, ''), shared, sync_token, account + FROM crypto_account + """ + ) + await conn.execute("DROP TABLE crypto_account") + await conn.execute("ALTER TABLE new_crypto_account RENAME TO crypto_account") + + await conn.execute( + """CREATE TABLE new_crypto_megolm_inbound_session ( + account_id TEXT, + session_id CHAR(43), + sender_key CHAR(43) NOT NULL, + signing_key CHAR(43), + room_id TEXT NOT NULL, + session bytea, + forwarding_chains TEXT, + withheld_code TEXT, + withheld_reason TEXT, + PRIMARY KEY (account_id, session_id) + )""" + ) + await conn.execute( + """ + INSERT INTO new_crypto_megolm_inbound_session ( + account_id, session_id, sender_key, signing_key, room_id, session, + forwarding_chains + ) + SELECT account_id, session_id, sender_key, signing_key, room_id, session, + forwarding_chains + FROM crypto_megolm_inbound_session + """ + ) + await conn.execute("DROP TABLE crypto_megolm_inbound_session") + await conn.execute( + "ALTER TABLE new_crypto_megolm_inbound_session RENAME TO crypto_megolm_inbound_session" + ) + + await conn.execute( + """CREATE TABLE new_crypto_cross_signing_keys ( + user_id TEXT, + usage TEXT, + key CHAR(43) NOT NULL, + + first_seen_key CHAR(43) NOT NULL, + + PRIMARY KEY (user_id, usage) + )""" + ) + await conn.execute( + """ + INSERT INTO new_crypto_cross_signing_keys (user_id, usage, key, first_seen_key) + SELECT user_id, usage, key, COALESCE(first_seen_key, key) + FROM crypto_cross_signing_keys + WHERE key IS NOT NULL + """ + ) + await conn.execute("DROP TABLE crypto_cross_signing_keys") + await conn.execute( + "ALTER TABLE new_crypto_cross_signing_keys RENAME TO crypto_cross_signing_keys" + ) + + await conn.execute( + """CREATE TABLE new_crypto_cross_signing_signatures ( + signed_user_id TEXT, + signed_key TEXT, + signer_user_id TEXT, + signer_key TEXT, + signature CHAR(88) NOT NULL, + PRIMARY KEY (signed_user_id, signed_key, signer_user_id, signer_key) + )""" + ) + await conn.execute( + """ + INSERT INTO new_crypto_cross_signing_signatures ( + signed_user_id, signed_key, signer_user_id, signer_key, signature + ) + SELECT signed_user_id, signed_key, signer_user_id, signer_key, signature + FROM crypto_cross_signing_signatures + WHERE signature IS NOT NULL + """ + ) + await conn.execute("DROP TABLE crypto_cross_signing_signatures") + await conn.execute( + "ALTER TABLE new_crypto_cross_signing_signatures " + "RENAME TO crypto_cross_signing_signatures" + ) + + await conn.execute("PRAGMA foreign_key_check") + await conn.execute("PRAGMA foreign_keys = ON") diff --git a/mautrix/errors/__init__.py b/mautrix/errors/__init__.py index fdec6e3a..d3646a37 100644 --- a/mautrix/errors/__init__.py +++ b/mautrix/errors/__init__.py @@ -6,6 +6,7 @@ DeviceValidationError, DuplicateMessageIndex, EncryptionError, + GroupSessionWithheldError, MatchingSessionDecryptionError, MismatchingRoomError, SessionNotFound, diff --git a/mautrix/errors/crypto.py b/mautrix/errors/crypto.py index 97592b05..4a65048c 100644 --- a/mautrix/errors/crypto.py +++ b/mautrix/errors/crypto.py @@ -36,6 +36,12 @@ class MatchingSessionDecryptionError(DecryptionError): pass +class GroupSessionWithheldError(DecryptionError): + def __init__(self, session_id: SessionID, withheld_code: str) -> None: + super().__init__(f"Session ID {session_id} was withheld ({withheld_code})") + self.withheld_code = withheld_code + + class SessionNotFound(DecryptionError): def __init__(self, session_id: SessionID, sender_key: IdentityKey | None = None) -> None: super().__init__( From 27732d4b7a42c53af72af2413ab2ee1b09c001fd Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 23 Mar 2023 16:35:29 +0200 Subject: [PATCH 063/218] Change max_age to BIGINT milliseconds --- mautrix/crypto/store/asyncpg/store.py | 9 ++------- mautrix/crypto/store/asyncpg/upgrade.py | 12 +++++++++--- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/mautrix/crypto/store/asyncpg/store.py b/mautrix/crypto/store/asyncpg/store.py index 51985caf..cb15be77 100644 --- a/mautrix/crypto/store/asyncpg/store.py +++ b/mautrix/crypto/store/asyncpg/store.py @@ -289,9 +289,7 @@ async def has_group_session(self, room_id: RoomID, session_id: SessionID) -> boo async def add_outbound_group_session(self, session: OutboundGroupSession) -> None: pickle = session.pickle(self.pickle_key) - max_age = session.max_age - if self.db.scheme == Scheme.SQLITE: - max_age = max_age.total_seconds() + max_age = int(session.max_age.total_seconds() * 1000) q = """ INSERT INTO crypto_megolm_outbound_session ( room_id, session_id, session, shared, max_messages, message_count, @@ -341,9 +339,6 @@ async def get_outbound_group_session(self, room_id: RoomID) -> OutboundGroupSess row = await self.db.fetchrow(q, room_id, self.account_id) if row is None: return None - max_age = row["max_age"] - if self.db.scheme == Scheme.SQLITE: - max_age = timedelta(seconds=max_age) return OutboundGroupSession.from_pickle( row["session"], passphrase=self.pickle_key, @@ -351,7 +346,7 @@ async def get_outbound_group_session(self, room_id: RoomID) -> OutboundGroupSess shared=row["shared"], max_messages=row["max_messages"], message_count=row["message_count"], - max_age=max_age, + max_age=timedelta(milliseconds=row["max_age"]), use_time=row["last_used"], creation_time=row["created_at"], ) diff --git a/mautrix/crypto/store/asyncpg/upgrade.py b/mautrix/crypto/store/asyncpg/upgrade.py index b0f2299c..9b8ce779 100644 --- a/mautrix/crypto/store/asyncpg/upgrade.py +++ b/mautrix/crypto/store/asyncpg/upgrade.py @@ -80,7 +80,6 @@ async def upgrade_blank_to_latest(conn: Connection) -> None: PRIMARY KEY (account_id, session_id) )""" ) - # TODO chnge max_age to BIGINT await conn.execute( """CREATE TABLE IF NOT EXISTS crypto_megolm_outbound_session ( account_id TEXT, @@ -90,7 +89,7 @@ async def upgrade_blank_to_latest(conn: Connection) -> None: shared BOOLEAN NOT NULL, max_messages INTEGER NOT NULL, message_count INTEGER NOT NULL, - max_age INTERVAL NOT NULL, + max_age BIGINT NOT NULL, created_at timestamp NOT NULL, last_used timestamp NOT NULL, PRIMARY KEY (account_id, room_id) @@ -167,7 +166,7 @@ async def upgrade_v2(conn: Connection, scheme: Scheme) -> None: shared BOOLEAN NOT NULL, max_messages INTEGER NOT NULL, message_count INTEGER NOT NULL, - max_age INTERVAL NOT NULL, + max_age BIGINT NOT NULL, created_at timestamp NOT NULL, last_used timestamp NOT NULL, PRIMARY KEY (account_id, room_id) @@ -298,6 +297,11 @@ async def upgrade_v8_postgres(conn: Connection) -> None: "ALTER TABLE crypto_cross_signing_signatures ALTER COLUMN signature SET NOT NULL" ) + await conn.execute( + "ALTER TABLE crypto_megolm_outbound_session ALTER COLUMN max_age TYPE BIGINT " + "USING (EXTRACT(EPOCH from max_age)*1000)::int" + ) + async def upgrade_v8_sqlite(conn: Connection) -> None: await conn.execute("PRAGMA foreign_keys = OFF") @@ -351,6 +355,8 @@ async def upgrade_v8_sqlite(conn: Connection) -> None: "ALTER TABLE new_crypto_megolm_inbound_session RENAME TO crypto_megolm_inbound_session" ) + await conn.execute("UPDATE crypto_megolm_outbound_session SET max_age=max_age*1000") + await conn.execute( """CREATE TABLE new_crypto_cross_signing_keys ( user_id TEXT, From 342728365d41cad4541e4b83970130e158c581e3 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 23 Mar 2023 16:40:03 +0200 Subject: [PATCH 064/218] Bump schema version to 9 --- mautrix/crypto/store/asyncpg/upgrade.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/mautrix/crypto/store/asyncpg/upgrade.py b/mautrix/crypto/store/asyncpg/upgrade.py index 9b8ce779..1f1b7e7e 100644 --- a/mautrix/crypto/store/asyncpg/upgrade.py +++ b/mautrix/crypto/store/asyncpg/upgrade.py @@ -257,17 +257,17 @@ async def upgrade_v6(conn: Connection) -> None: @upgrade_table.register( - description="Synchronize schema with mautrix-go", upgrades_to=8, transaction=False + description="Synchronize schema with mautrix-go", upgrades_to=9, transaction=False ) -async def upgrade_v8(conn: Connection, scheme: Scheme) -> None: +async def upgrade_v9(conn: Connection, scheme: Scheme) -> None: if scheme == Scheme.POSTGRES: async with conn.transaction(): - await upgrade_v8_postgres(conn) + await upgrade_v9_postgres(conn) else: - await upgrade_v8_sqlite(conn) + await upgrade_v9_sqlite(conn) -async def upgrade_v8_postgres(conn: Connection) -> None: +async def upgrade_v9_postgres(conn: Connection) -> None: await conn.execute("UPDATE crypto_account SET device_id='' WHERE device_id IS NULL") await conn.execute("ALTER TABLE crypto_account ALTER COLUMN device_id SET NOT NULL") @@ -303,7 +303,7 @@ async def upgrade_v8_postgres(conn: Connection) -> None: ) -async def upgrade_v8_sqlite(conn: Connection) -> None: +async def upgrade_v9_sqlite(conn: Connection) -> None: await conn.execute("PRAGMA foreign_keys = OFF") async with conn.transaction(): await conn.execute( From 9213dd0cdad42ed761aa315e52ae327b695564a3 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 23 Mar 2023 16:47:37 +0200 Subject: [PATCH 065/218] Fix upgrade index --- mautrix/crypto/store/asyncpg/upgrade.py | 11 +++++++++++ mautrix/util/async_db/upgrade.py | 4 ++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/mautrix/crypto/store/asyncpg/upgrade.py b/mautrix/crypto/store/asyncpg/upgrade.py index 1f1b7e7e..8e8da45a 100644 --- a/mautrix/crypto/store/asyncpg/upgrade.py +++ b/mautrix/crypto/store/asyncpg/upgrade.py @@ -267,6 +267,17 @@ async def upgrade_v9(conn: Connection, scheme: Scheme) -> None: await upgrade_v9_sqlite(conn) +# These two are never used because the previous one jumps from 6 to 9. +@upgrade_table.register +async def upgrade_noop_7_to_8(_: Connection) -> None: + pass + + +@upgrade_table.register +async def upgrade_noop_8_to_9(_: Connection) -> None: + pass + + async def upgrade_v9_postgres(conn: Connection) -> None: await conn.execute("UPDATE crypto_account SET device_id='' WHERE device_id IS NULL") await conn.execute("ALTER TABLE crypto_account ALTER COLUMN device_id SET NOT NULL") diff --git a/mautrix/util/async_db/upgrade.py b/mautrix/util/async_db/upgrade.py index d69dac56..9e593ece 100644 --- a/mautrix/util/async_db/upgrade.py +++ b/mautrix/util/async_db/upgrade.py @@ -21,7 +21,7 @@ UpgradeWithoutScheme = Callable[[LoggingConnection], Awaitable[Optional[int]]] -async def noop_upgrade(_: LoggingConnection) -> None: +async def noop_upgrade(_: LoggingConnection, _2: Scheme) -> None: pass @@ -178,6 +178,6 @@ def _find_upgrade_table(fn: Upgrade) -> UpgradeTable: def register_upgrade(index: int = -1, description: str = "") -> Callable[[Upgrade], Upgrade]: def actually_register(fn: Upgrade) -> Upgrade: - return _find_upgrade_table(fn).register(index, description, fn) + return _find_upgrade_table(fn).register(fn, index=index, description=description) return actually_register From 97f5e87b3c2e9aa3efcd25ecdf68624743098425 Mon Sep 17 00:00:00 2001 From: Sumner Evans Date: Wed, 5 Apr 2023 10:48:29 -0600 Subject: [PATCH 066/218] client/user_data: add beeper_update_profile method Signed-off-by: Sumner Evans --- mautrix/client/api/user_data.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/mautrix/client/api/user_data.py b/mautrix/client/api/user_data.py index f37cb769..4c3d437a 100644 --- a/mautrix/client/api/user_data.py +++ b/mautrix/client/api/user_data.py @@ -5,6 +5,8 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import annotations +from typing import Any + from mautrix.api import Method, Path from mautrix.errors import MatrixResponseError, MNotFound from mautrix.types import ContentURI, Member, SerializerError, User, UserID, UserSearchResults @@ -170,3 +172,16 @@ async def get_profile(self, user_id: UserID) -> Member: raise MatrixResponseError("Invalid member in response") from e # endregion + + # region Beeper Custom Fields API + + async def beeper_update_profile(self, custom_fields: dict[str, Any]) -> None: + """ + Set custom fields on the user's profile. Only works on Hungryserv. + + Args: + custom_fields: A dictionary of fields to set in the custom content of the profile. + """ + await self.api.request(Method.PATCH, Path.v3.profile[self.mxid], custom_fields) + + # endregion From ddf796ffc10b5573b3aee15560c238fad01ec6d3 Mon Sep 17 00:00:00 2001 From: Sumner Evans Date: Wed, 5 Apr 2023 14:31:58 -0600 Subject: [PATCH 067/218] bridge/init_as_bot: set Beeper contact info fields Signed-off-by: Sumner Evans --- mautrix/bridge/bridge.py | 10 ++++++++++ mautrix/bridge/matrix.py | 11 +++++++++++ 2 files changed, 21 insertions(+) diff --git a/mautrix/bridge/bridge.py b/mautrix/bridge/bridge.py index 7005d775..396a04a1 100644 --- a/mautrix/bridge/bridge.py +++ b/mautrix/bridge/bridge.py @@ -315,6 +315,16 @@ def is_bridge_ghost(self, user_id: UserID) -> bool: async def count_logged_in_users(self) -> int: return 0 + @staticmethod + @abstractmethod + def get_beeper_service_name() -> str: + pass + + @staticmethod + @abstractmethod + def get_beeper_network_name() -> str: + pass + async def manhole_global_namespace(self, user_id: UserID) -> dict[str, Any]: own_user = await self.get_user(user_id, create=False) try: diff --git a/mautrix/bridge/matrix.py b/mautrix/bridge/matrix.py index 08382abc..aba84567 100644 --- a/mautrix/bridge/matrix.py +++ b/mautrix/bridge/matrix.py @@ -272,6 +272,17 @@ async def init_as_bot(self) -> None: except Exception: self.log.exception("Failed to set bot avatar") + if self.bridge.homeserver_software.is_hungry: + self.log.debug("Setting contact info on the appservice bot") + await self.az.intent.beeper_update_profile( + { + "com.beeper.bridge.service": self.bridge.get_beeper_service_name(), + "com.beeper.bridge.network": self.bridge.get_beeper_network_name(), + "com.beeper.bridge.is_bridge_bot": True, + "com.beeper.bridge.is_bot": True, + } + ) + async def init_encryption(self) -> None: if self.e2ee: await self.e2ee.start() From 0d55eb589e8fa85307f48e13e67305f479f1d13e Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 6 Apr 2023 11:11:41 +0300 Subject: [PATCH 068/218] Disable reply fallback in set_thread_parent --- mautrix/types/event/message.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/mautrix/types/event/message.py b/mautrix/types/event/message.py index 47c4e2e6..021e91f9 100644 --- a/mautrix/types/event/message.py +++ b/mautrix/types/event/message.py @@ -119,7 +119,7 @@ def set_thread_parent( thread_parent.content.get_thread_parent() or self.relates_to.event_id ) if not disable_reply_fallback: - self.set_reply(last_event_in_thread or thread_parent, **kwargs) + self.set_reply(last_event_in_thread or thread_parent, disable_fallback=True, **kwargs) self.relates_to.is_falling_back = True def set_edit(self, edits: Union[EventID, "MessageEvent"]) -> None: @@ -315,12 +315,16 @@ class TextMessageEventContent(BaseMessageEventContent, SerializableAttrs): formatted_body: str = None def set_reply( - self, reply_to: Union["MessageEvent", EventID], *, displayname: Optional[str] = None + self, + reply_to: Union["MessageEvent", EventID], + *, + displayname: Optional[str] = None, + disable_fallback: bool = False, ) -> None: super().set_reply(reply_to) if isinstance(reply_to, str): return - if isinstance(reply_to, MessageEvent): + if isinstance(reply_to, MessageEvent) and not disable_fallback: self.ensure_has_html() if isinstance(reply_to.content, TextMessageEventContent): reply_to.content.trim_reply_fallback() From 251b8fcb9acee184f6d4fabe955072ab8f0fea8f Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 6 Apr 2023 11:22:38 +0300 Subject: [PATCH 069/218] Fix latest schema version --- mautrix/crypto/store/asyncpg/upgrade.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/crypto/store/asyncpg/upgrade.py b/mautrix/crypto/store/asyncpg/upgrade.py index 8e8da45a..789552a1 100644 --- a/mautrix/crypto/store/asyncpg/upgrade.py +++ b/mautrix/crypto/store/asyncpg/upgrade.py @@ -16,7 +16,7 @@ ) -@upgrade_table.register(description="Latest revision", upgrades_to=8) +@upgrade_table.register(description="Latest revision", upgrades_to=9) async def upgrade_blank_to_latest(conn: Connection) -> None: await conn.execute( """CREATE TABLE IF NOT EXISTS crypto_account ( From 9ff58f9af97d6f31abfc42fb4f9069de1d07e3e0 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 6 Apr 2023 16:51:44 +0300 Subject: [PATCH 070/218] Replace service/network name with properties --- mautrix/bridge/bridge.py | 12 ++---------- mautrix/bridge/matrix.py | 6 +++--- 2 files changed, 5 insertions(+), 13 deletions(-) diff --git a/mautrix/bridge/bridge.py b/mautrix/bridge/bridge.py index 396a04a1..d8241038 100644 --- a/mautrix/bridge/bridge.py +++ b/mautrix/bridge/bridge.py @@ -59,6 +59,8 @@ class Bridge(Program, ABC): markdown_version: str manhole: br.commands.manhole.ManholeState | None homeserver_software: HomeserverSoftware + beeper_network_name: str | None = None + beeper_service_name: str | None = None def __init__( self, @@ -315,16 +317,6 @@ def is_bridge_ghost(self, user_id: UserID) -> bool: async def count_logged_in_users(self) -> int: return 0 - @staticmethod - @abstractmethod - def get_beeper_service_name() -> str: - pass - - @staticmethod - @abstractmethod - def get_beeper_network_name() -> str: - pass - async def manhole_global_namespace(self, user_id: UserID) -> dict[str, Any]: own_user = await self.get_user(user_id, create=False) try: diff --git a/mautrix/bridge/matrix.py b/mautrix/bridge/matrix.py index aba84567..6ce3bfde 100644 --- a/mautrix/bridge/matrix.py +++ b/mautrix/bridge/matrix.py @@ -272,12 +272,12 @@ async def init_as_bot(self) -> None: except Exception: self.log.exception("Failed to set bot avatar") - if self.bridge.homeserver_software.is_hungry: + if self.bridge.homeserver_software.is_hungry and self.bridge.beeper_network_name: self.log.debug("Setting contact info on the appservice bot") await self.az.intent.beeper_update_profile( { - "com.beeper.bridge.service": self.bridge.get_beeper_service_name(), - "com.beeper.bridge.network": self.bridge.get_beeper_network_name(), + "com.beeper.bridge.service": self.bridge.beeper_service_name, + "com.beeper.bridge.network": self.bridge.beeper_network_name, "com.beeper.bridge.is_bridge_bot": True, "com.beeper.bridge.is_bot": True, } From 754ec520f3e6b13ec22624ce4c8aa84e87c03d1f Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 6 Apr 2023 16:54:58 +0300 Subject: [PATCH 071/218] Bump version to 0.19.8 --- CHANGELOG.md | 5 +++++ mautrix/__init__.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 24192de4..862da25a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +## v0.19.8 (2023-04-06) + +* *(crypto)* Updated crypto store schema to match mautrix-go. +* *(types)* Fixed `set_thread_parent` adding reply fallbacks to the message body. + ## v0.19.7 (2023-03-22) * *(bridge, crypto)* Fixed key sharing trust checker not resolving cross-signing diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 53ed1518..40300b56 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.19.7" +__version__ = "0.19.8" __author__ = "Tulir Asokan " __all__ = [ "api", From 4860ddf76c77b4624ce200c73fb13675688ef615 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 8 Apr 2023 22:37:13 +0300 Subject: [PATCH 072/218] Fix crypto store migration when there are long-lived megolm sessions --- mautrix/crypto/store/asyncpg/upgrade.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/crypto/store/asyncpg/upgrade.py b/mautrix/crypto/store/asyncpg/upgrade.py index 789552a1..914da23d 100644 --- a/mautrix/crypto/store/asyncpg/upgrade.py +++ b/mautrix/crypto/store/asyncpg/upgrade.py @@ -310,7 +310,7 @@ async def upgrade_v9_postgres(conn: Connection) -> None: await conn.execute( "ALTER TABLE crypto_megolm_outbound_session ALTER COLUMN max_age TYPE BIGINT " - "USING (EXTRACT(EPOCH from max_age)*1000)::int" + "USING (EXTRACT(EPOCH from max_age)*1000)::bigint" ) From 82a9929e089b908c89ca9116e4163303311938d8 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 12 Apr 2023 16:51:32 +0300 Subject: [PATCH 073/218] Bump version to 0.19.9 --- CHANGELOG.md | 5 +++++ mautrix/__init__.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 862da25a..e7cabdd6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +## v0.19.9 (2023-04-12) + +* *(crypto)* Fixed bug in crypto store migration when using outbound sessions + with max age higher than usual. + ## v0.19.8 (2023-04-06) * *(crypto)* Updated crypto store schema to match mautrix-go. diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 40300b56..de48a11a 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.19.8" +__version__ = "0.19.9" __author__ = "Tulir Asokan " __all__ = [ "api", From efd1eaaf6dffb58c9595b77589c2249171314a66 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 12 Apr 2023 20:09:23 +0300 Subject: [PATCH 074/218] Add options to automatically ratchet/delete megolm sessions (#144) --- mautrix/bridge/config.py | 7 ++ mautrix/bridge/e2ee.py | 59 +++++++++++- mautrix/bridge/portal.py | 2 +- mautrix/client/state_store/asyncpg/upgrade.py | 19 +++- mautrix/crypto/__init__.py | 2 +- mautrix/crypto/base.py | 8 ++ mautrix/crypto/decrypt_megolm.py | 81 +++++++++++++--- mautrix/crypto/device_lists.py | 10 ++ mautrix/crypto/encrypt_megolm.py | 63 ++++++++----- mautrix/crypto/key_request.py | 17 +++- mautrix/crypto/machine.py | 53 +++++++++++ mautrix/crypto/sessions.py | 67 +++++++++++++- mautrix/crypto/store/abstract.py | 40 ++++++++ mautrix/crypto/store/asyncpg/store.py | 92 ++++++++++++++++++- mautrix/crypto/store/asyncpg/upgrade.py | 23 ++++- mautrix/crypto/store/memory.py | 24 +++++ mautrix/types/event/beeper.py | 9 +- mautrix/types/event/to_device.py | 16 +++- mautrix/types/event/type.py | 1 + mautrix/types/event/type.pyi | 1 + 20 files changed, 539 insertions(+), 55 deletions(-) diff --git a/mautrix/bridge/config.py b/mautrix/bridge/config.py index 4289b178..f3e7720c 100644 --- a/mautrix/bridge/config.py +++ b/mautrix/bridge/config.py @@ -138,6 +138,13 @@ def do_update(self, helper: ConfigUpdateHelper) -> None: copy("bridge.encryption.default") copy("bridge.encryption.require") copy("bridge.encryption.appservice") + copy("bridge.encryption.delete_keys.delete_outbound_on_ack") + copy("bridge.encryption.delete_keys.dont_store_outbound") + copy("bridge.encryption.delete_keys.ratchet_on_decrypt") + copy("bridge.encryption.delete_keys.delete_fully_used_on_decrypt") + copy("bridge.encryption.delete_keys.delete_prev_on_new_session") + copy("bridge.encryption.delete_keys.delete_on_device_delete") + copy("bridge.encryption.delete_keys.periodically_delete_expired") copy("bridge.encryption.verification_levels.receive") copy("bridge.encryption.verification_levels.send") copy("bridge.encryption.verification_levels.share") diff --git a/mautrix/bridge/e2ee.py b/mautrix/bridge/e2ee.py index 804792c1..95b419f7 100644 --- a/mautrix/bridge/e2ee.py +++ b/mautrix/bridge/e2ee.py @@ -13,7 +13,7 @@ from mautrix.appservice import AppService from mautrix.client import Client, InternalEventType, SyncStore from mautrix.crypto import CryptoStore, OlmMachine, PgCryptoStore, RejectKeyShare, StateStore -from mautrix.errors import EncryptionError, SessionNotFound +from mautrix.errors import EncryptionError, MForbidden, MNotFound, SessionNotFound from mautrix.types import ( JSON, DeviceIdentity, @@ -34,6 +34,7 @@ StateFilter, TrustState, ) +from mautrix.util import background_task from mautrix.util.async_db import Database from mautrix.util.logging import TraceLogger @@ -61,6 +62,7 @@ class EncryptionManager: min_send_trust: TrustState key_sharing_enabled: bool appservice_mode: bool + periodically_delete_expired_keys: bool bridge: br.Bridge az: AppService @@ -68,6 +70,7 @@ class EncryptionManager: _id_suffix: str _share_session_events: dict[RoomID, asyncio.Event] + _key_delete_task: asyncio.Task | None def __init__( self, @@ -100,6 +103,7 @@ def __init__( sync_store=self.crypto_store, log=self.log.getChild("client"), default_retry_count=default_http_retry_count, + state_store=self.bridge.state_store, ) self.crypto = OlmMachine(self.client, self.crypto_store, self.state_store) self.client.add_event_handler(InternalEventType.SYNC_STOPPED, self._exit_on_sync_fail) @@ -115,6 +119,16 @@ def __init__( self.az.device_list_handler = self.crypto.handle_as_device_lists self.az.to_device_handler = self.crypto.handle_as_to_device_event + delete_cfg = bridge.config["bridge.encryption.delete_keys"] + self.crypto.delete_outbound_keys_on_ack = delete_cfg["delete_outbound_on_ack"] + self.crypto.dont_store_outbound_keys = delete_cfg["dont_store_outbound"] + self.crypto.delete_previous_keys_on_receive = delete_cfg["delete_prev_on_new_session"] + self.crypto.ratchet_keys_on_decrypt = delete_cfg["ratchet_on_decrypt"] + self.crypto.delete_fully_used_keys_on_decrypt = delete_cfg["delete_fully_used_on_decrypt"] + self.crypto.delete_keys_on_device_delete = delete_cfg["delete_on_device_delete"] + self.periodically_delete_expired_keys = delete_cfg["periodically_delete_expired"] + self._key_delete_task = None + async def _exit_on_sync_fail(self, data) -> None: if data["error"]: self.log.critical("Exiting due to crypto sync error") @@ -267,6 +281,37 @@ async def start(self) -> None: else: _ = self.client.start(self._filter) self.log.info("End-to-bridge encryption support is enabled (sync mode)") + if self.periodically_delete_expired_keys: + self._key_delete_task = background_task.create(self._periodically_delete_keys()) + background_task.create(self._resync_encryption_info()) + + async def _resync_encryption_info(self) -> None: + rows = await self.crypto_db.fetch( + """SELECT room_id FROM mx_room_state WHERE encryption='{"resync":true}'""" + ) + room_ids = [row["room_id"] for row in rows] + if not room_ids: + return + self.log.debug(f"Resyncing encryption state event in rooms: {room_ids}") + for room_id in room_ids: + try: + evt = await self.client.get_state_event(room_id, EventType.ROOM_ENCRYPTION) + except (MNotFound, MForbidden) as e: + self.log.debug(f"Failed to get encryption state in {room_id}: {e}") + q = """ + UPDATE mx_room_state SET encryption=NULL + WHERE room_id=$1 AND encryption='{"resync":true}' + """ + await self.crypto_db.execute(q, room_id) + else: + self.log.debug(f"Resynced encryption state in {room_id}: {evt}") + q = """ + UPDATE crypto_megolm_inbound_session SET max_age=$1, max_messages=$2 + WHERE room_id=$3 AND max_age IS NULL and max_messages IS NULL + """ + await self.crypto_db.execute( + q, evt.rotation_period_ms, evt.rotation_period_msgs, room_id + ) async def _verify_keys_are_on_server(self) -> None: self.log.debug("Making sure keys are still on server") @@ -289,6 +334,9 @@ async def _verify_keys_are_on_server(self) -> None: sys.exit(34) async def stop(self) -> None: + if self._key_delete_task: + self._key_delete_task.cancel() + self._key_delete_task = None self.client.stop() await self.crypto_store.close() if self.crypto_db: @@ -308,3 +356,12 @@ def _filter(self) -> Filter: ephemeral=RoomEventFilter(not_types=[all_events]), ), ) + + async def _periodically_delete_keys(self) -> None: + while True: + deleted = await self.crypto_store.redact_expired_group_sessions() + if deleted: + self.log.info(f"Deleted expired megolm sessions: {deleted}") + else: + self.log.debug("No expired megolm sessions found") + await asyncio.sleep(24 * 60 * 60) diff --git a/mautrix/bridge/portal.py b/mautrix/bridge/portal.py index 1a6a1343..05d67e3b 100644 --- a/mautrix/bridge/portal.py +++ b/mautrix/bridge/portal.py @@ -386,7 +386,7 @@ async def _disappear_event(self, msg: br.AbstractDisappearingMessage) -> None: await self._do_disappear(msg.event_id) self.log.debug(f"Expired event {msg.event_id} disappeared successfully") except Exception as e: - self.log.warning(f"Failed to make expired event {msg.event_id} disappear: {e}", e) + self.log.warning(f"Failed to make expired event {msg.event_id} disappear: {e}") async def _do_disappear(self, event_id: EventID) -> None: await self.main_intent.redact(self.mxid, event_id) diff --git a/mautrix/client/state_store/asyncpg/upgrade.py b/mautrix/client/state_store/asyncpg/upgrade.py index 0a489aae..88f115f2 100644 --- a/mautrix/client/state_store/asyncpg/upgrade.py +++ b/mautrix/client/state_store/asyncpg/upgrade.py @@ -14,8 +14,8 @@ ) -@upgrade_table.register(description="Latest revision", upgrades_to=2) -async def upgrade_blank_to_v2(conn: Connection, scheme: Scheme) -> None: +@upgrade_table.register(description="Latest revision", upgrades_to=3) +async def upgrade_blank_to_v3(conn: Connection, scheme: Scheme) -> None: await conn.execute( """CREATE TABLE mx_room_state ( room_id TEXT PRIMARY KEY, @@ -54,3 +54,18 @@ async def upgrade_v2(conn: Connection, scheme: Scheme) -> None: await conn.execute("ALTER TABLE mx_user_profile ALTER COLUMN user_id TYPE TEXT") await conn.execute("ALTER TABLE mx_user_profile ALTER COLUMN displayname TYPE TEXT") await conn.execute("ALTER TABLE mx_user_profile ALTER COLUMN avatar_url TYPE TEXT") + + +@upgrade_table.register(description="Mark rooms that need crypto state event resynced") +async def upgrade_v3(conn: Connection) -> None: + if await conn.table_exists("portal"): + await conn.execute( + """ + INSERT INTO mx_room_state (room_id, encryption) + SELECT portal.mxid, '{"resync":true}' FROM portal + WHERE portal.encrypted=true AND portal.mxid IS NOT NULL + ON CONFLICT (room_id) DO UPDATE + SET encryption=excluded.encryption + WHERE mx_room_state.encryption IS NULL + """ + ) diff --git a/mautrix/crypto/__init__.py b/mautrix/crypto/__init__.py index 39867225..743fc2c6 100644 --- a/mautrix/crypto/__init__.py +++ b/mautrix/crypto/__init__.py @@ -1,6 +1,6 @@ from .account import OlmAccount from .key_share import RejectKeyShare -from .sessions import InboundGroupSession, OutboundGroupSession, Session +from .sessions import InboundGroupSession, OutboundGroupSession, RatchetSafety, Session # These have to be last from .store import ( # isort: skip diff --git a/mautrix/crypto/base.py b/mautrix/crypto/base.py index 4d12e6cf..f403feee 100644 --- a/mautrix/crypto/base.py +++ b/mautrix/crypto/base.py @@ -46,6 +46,13 @@ class BaseOlmMachine: share_keys_min_trust: TrustState allow_key_share: Callable[[crypto.DeviceIdentity, RequestedKeyInfo], Awaitable[bool]] + delete_outbound_keys_on_ack: bool + dont_store_outbound_keys: bool + delete_previous_keys_on_receive: bool + ratchet_keys_on_decrypt: bool + delete_fully_used_keys_on_decrypt: bool + delete_keys_on_device_delete: bool + # Futures that wait for responses to a key request _key_request_waiters: dict[SessionID, asyncio.Future] # Futures that wait for a session to be received (either normally or through a key request) @@ -53,6 +60,7 @@ class BaseOlmMachine: _prev_unwedge: dict[IdentityKey, float] _fetch_keys_lock: asyncio.Lock + _megolm_decrypt_lock: asyncio.Lock _cs_fetch_attempted: set[UserID] async def wait_for_session( diff --git a/mautrix/crypto/decrypt_megolm.py b/mautrix/crypto/decrypt_megolm.py index 8edf5aaf..fd7a7eef 100644 --- a/mautrix/crypto/decrypt_megolm.py +++ b/mautrix/crypto/decrypt_megolm.py @@ -25,6 +25,7 @@ ) from .device_lists import DeviceListMachine +from .sessions import InboundGroupSession class MegolmDecryptionMachine(DeviceListMachine): @@ -45,18 +46,22 @@ async def decrypt_megolm_event(self, evt: EncryptedEvent) -> Event: raise DecryptionError("Unsupported event content class") elif evt.content.algorithm != EncryptionAlgorithm.MEGOLM_V1: raise DecryptionError("Unsupported event encryption algorithm") - session = await self.crypto_store.get_group_session(evt.room_id, evt.content.session_id) - if session is None: - # TODO check if olm session is wedged - raise SessionNotFound(evt.content.session_id, evt.content.sender_key) - try: - plaintext, index = session.decrypt(evt.content.ciphertext) - except olm.OlmGroupSessionError as e: - raise DecryptionError("Failed to decrypt megolm event") from e - if not await self.crypto_store.validate_message_index( - session.sender_key, SessionID(session.id), evt.event_id, index, evt.timestamp - ): - raise DuplicateMessageIndex() + async with self._megolm_decrypt_lock: + session = await self.crypto_store.get_group_session( + evt.room_id, evt.content.session_id + ) + if session is None: + # TODO check if olm session is wedged + raise SessionNotFound(evt.content.session_id, evt.content.sender_key) + try: + plaintext, index = session.decrypt(evt.content.ciphertext) + except olm.OlmGroupSessionError as e: + raise DecryptionError("Failed to decrypt megolm event") from e + if not await self.crypto_store.validate_message_index( + session.sender_key, SessionID(session.id), evt.event_id, index, evt.timestamp + ): + raise DuplicateMessageIndex() + await self._ratchet_session(session, index) forwarded_keys = False if ( @@ -133,3 +138,55 @@ async def decrypt_megolm_event(self, evt: EncryptedEvent) -> Event: "was_encrypted": True, } return result + + async def _ratchet_session(self, sess: InboundGroupSession, index: int) -> None: + expected_message_index = sess.ratchet_safety.next_index + did_modify = True + if index > expected_message_index: + sess.ratchet_safety.missed_indices += list(range(expected_message_index, index)) + sess.ratchet_safety.next_index = index + 1 + elif index == expected_message_index: + sess.ratchet_safety.next_index = index + 1 + else: + try: + sess.ratchet_safety.missed_indices.remove(index) + except ValueError: + did_modify = False + # Use presence of received_at as a sign that this is a recent megolm session, + # and therefore it's safe to drop missed indices entirely. + if ( + sess.received_at + and sess.ratchet_safety.missed_indices + and sess.ratchet_safety.missed_indices[0] < expected_message_index - 10 + ): + i = 0 + for i, lost_index in enumerate(sess.ratchet_safety.missed_indices): + if lost_index < expected_message_index - 10: + sess.ratchet_safety.lost_indices.append(lost_index) + else: + break + sess.ratchet_safety.missed_indices = sess.ratchet_safety.missed_indices[i + 1 :] + ratchet_target_index = sess.ratchet_safety.next_index + if len(sess.ratchet_safety.missed_indices) > 0: + ratchet_target_index = min(sess.ratchet_safety.missed_indices) + self.log.debug( + f"Ratchet safety info for {sess.id}: {sess.ratchet_safety}, {ratchet_target_index=}" + ) + sess_id = SessionID(sess.id) + if ( + sess.max_messages + and ratchet_target_index >= sess.max_messages + and not sess.ratchet_safety.missed_indices + and self.delete_fully_used_keys_on_decrypt + ): + self.log.info(f"Deleting fully used session {sess.id}") + await self.crypto_store.redact_group_session( + sess.room_id, sess_id, reason="maximum messages reached" + ) + return + elif sess.first_known_index < ratchet_target_index and self.ratchet_keys_on_decrypt: + self.log.info(f"Ratcheting session {sess.id} to {ratchet_target_index}") + sess = sess.ratchet_to(ratchet_target_index) + elif not did_modify: + return + await self.crypto_store.put_group_session(sess.room_id, sess.sender_key, sess_id, sess) diff --git a/mautrix/crypto/device_lists.py b/mautrix/crypto/device_lists.py index c0cea43f..9afe7b91 100644 --- a/mautrix/crypto/device_lists.py +++ b/mautrix/crypto/device_lists.py @@ -81,6 +81,16 @@ async def _fetch_keys( data[user_id] = new_devices if changed or len(new_devices) != len(existing_devices): + if self.delete_keys_on_device_delete: + for device_id in new_devices.keys() - existing_devices.keys(): + device = existing_devices[device_id] + removed_ids = await self.crypto_store.redact_group_sessions( + room_id=None, sender_key=device.identity_key, reason="device removed" + ) + self.log.info( + "Redacted megolm sessions sent by removed device " + f"{device.user_id}/{device.device_id}: {removed_ids}" + ) await self.on_devices_changed(user_id) for user_id in missing_users: diff --git a/mautrix/crypto/encrypt_megolm.py b/mautrix/crypto/encrypt_megolm.py index 9b37e486..227fd3fe 100644 --- a/mautrix/crypto/encrypt_megolm.py +++ b/mautrix/crypto/encrypt_megolm.py @@ -5,7 +5,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. from typing import Any, Dict, List, Tuple, Union from collections import defaultdict -from datetime import timedelta +from datetime import datetime, timedelta import asyncio import json import time @@ -173,21 +173,6 @@ async def _share_group_session(self, room_id: RoomID, users: List[UserID]) -> No session = await self._new_outbound_group_session(room_id) self.log.debug(f"Sharing group session {session.id} for room {room_id} with {users}") - encryption_info = await self.state_store.get_encryption_info(room_id) - if encryption_info: - if encryption_info.algorithm != EncryptionAlgorithm.MEGOLM_V1: - raise SessionShareError("Room encryption algorithm is not supported") - session.max_messages = encryption_info.rotation_period_msgs or session.max_messages - session.max_age = ( - timedelta(milliseconds=encryption_info.rotation_period_ms) - if encryption_info.rotation_period_ms - else session.max_age - ) - self.log.debug( - "Got stored encryption state event and configured session to rotate " - f"after {session.max_messages} messages or {session.max_age}" - ) - olm_sessions: DeviceMap = defaultdict(lambda: {}) withhold_key_msgs = defaultdict(lambda: {}) missing_sessions: Dict[UserID, Dict[DeviceID, DeviceIdentity]] = defaultdict(lambda: {}) @@ -253,13 +238,33 @@ async def _share_group_session(self, room_id: RoomID, users: List[UserID]) -> No async def _new_outbound_group_session(self, room_id: RoomID) -> OutboundGroupSession: session = OutboundGroupSession(room_id) - await self._create_group_session( - self.account.identity_key, - self.account.signing_key, - room_id, - SessionID(session.id), - session.session_key, - ) + + encryption_info = await self.state_store.get_encryption_info(room_id) + if encryption_info: + if encryption_info.algorithm != EncryptionAlgorithm.MEGOLM_V1: + raise SessionShareError("Room encryption algorithm is not supported") + session.max_messages = encryption_info.rotation_period_msgs or session.max_messages + session.max_age = ( + timedelta(milliseconds=encryption_info.rotation_period_ms) + if encryption_info.rotation_period_ms + else session.max_age + ) + self.log.debug( + "Got stored encryption state event and configured session to rotate " + f"after {session.max_messages} messages or {session.max_age}" + ) + + if not self.dont_store_outbound_keys: + await self._create_group_session( + self.account.identity_key, + self.account.signing_key, + room_id, + SessionID(session.id), + session.session_key, + max_messages=session.max_messages, + max_age=session.max_age, + is_scheduled=False, + ) return session async def _encrypt_and_share_group_session( @@ -286,6 +291,9 @@ async def _create_group_session( room_id: RoomID, session_id: SessionID, session_key: str, + max_age: Union[timedelta, int], + max_messages: int, + is_scheduled: bool = False, ) -> None: start = time.monotonic() session = InboundGroupSession( @@ -293,6 +301,10 @@ async def _create_group_session( signing_key=signing_key, sender_key=sender_key, room_id=room_id, + received_at=datetime.utcnow(), + max_age=max_age, + max_messages=max_messages, + is_scheduled=is_scheduled, ) olm_duration = time.monotonic() - start if olm_duration > 5: @@ -302,7 +314,10 @@ async def _create_group_session( session_id = session.id await self.crypto_store.put_group_session(room_id, sender_key, session_id, session) self._mark_session_received(session_id) - self.log.debug(f"Created inbound group session {room_id}/{sender_key}/{session_id}") + self.log.debug( + f"Created inbound group session {room_id}/{sender_key}/{session_id} " + f"(max {max_age} / {max_messages} messages, {is_scheduled=})" + ) async def _find_olm_sessions( self, diff --git a/mautrix/crypto/key_request.py b/mautrix/crypto/key_request.py index 9bbd2d67..600ecc75 100644 --- a/mautrix/crypto/key_request.py +++ b/mautrix/crypto/key_request.py @@ -4,6 +4,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. from typing import Dict, List, Optional, Union +from datetime import timedelta import asyncio import uuid @@ -120,9 +121,23 @@ async def _receive_forwarded_room_key(self, evt: DecryptedOlmEvent) -> None: f"{evt.sender_device}, as crypto store says we have it already" ) return + if not key.beeper_max_messages or not key.beeper_max_age_ms: + encryption_info = await self.state_store.get_encryption_info(key.room_id) + if encryption_info: + if not key.beeper_max_age_ms: + key.beeper_max_age_ms = encryption_info.rotation_period_ms + if not key.beeper_max_messages: + key.beeper_max_messages = encryption_info.rotation_period_msgs key.forwarding_key_chain.append(evt.sender_key) sess = InboundGroupSession.import_session( - key.session_key, key.signing_key, key.sender_key, key.room_id, key.forwarding_key_chain + key.session_key, + key.signing_key, + key.sender_key, + key.room_id, + key.forwarding_key_chain, + max_age=key.beeper_max_age_ms, + max_messages=key.beeper_max_messages, + is_scheduled=key.beeper_is_scheduled, ) if key.session_id != sess.id: self.log.warning( diff --git a/mautrix/crypto/machine.py b/mautrix/crypto/machine.py index f9884f7b..fcefc3e3 100644 --- a/mautrix/crypto/machine.py +++ b/mautrix/crypto/machine.py @@ -10,6 +10,7 @@ import logging from mautrix import client as cli +from mautrix.errors import GroupSessionWithheldError from mautrix.types import ( ASToDeviceEvent, DecryptedOlmEvent, @@ -76,7 +77,15 @@ def __init__( self.share_keys_min_trust = TrustState.CROSS_SIGNED_TOFU self.allow_key_share = self.default_allow_key_share + self.delete_outbound_keys_on_ack = False + self.dont_store_outbound_keys = False + self.delete_previous_keys_on_receive = False + self.ratchet_keys_on_decrypt = False + self.delete_fully_used_keys_on_decrypt = False + self.delete_keys_on_device_delete = False + self._fetch_keys_lock = asyncio.Lock() + self._megolm_decrypt_lock = asyncio.Lock() self._key_request_waiters = {} self._inbound_session_waiters = {} self._prev_unwedge = {} @@ -88,6 +97,7 @@ def __init__( self.client.add_event_handler(cli.InternalEventType.DEVICE_LISTS, self.handle_device_lists) self.client.add_event_handler(EventType.TO_DEVICE_ENCRYPTED, self.handle_to_device_event) self.client.add_event_handler(EventType.ROOM_KEY_REQUEST, self.handle_room_key_request) + self.client.add_event_handler(EventType.BEEPER_ROOM_KEY_ACK, self.handle_beep_room_key_ack) # self.client.add_event_handler(EventType.ROOM_KEY_WITHHELD, self.handle_room_key_withheld) # self.client.add_event_handler(EventType.ORG_MATRIX_ROOM_KEY_WITHHELD, # self.handle_room_key_withheld) @@ -123,6 +133,8 @@ async def handle_as_to_device_event(self, evt: ASToDeviceEvent) -> None: await self.handle_to_device_event(evt) elif evt.type == EventType.ROOM_KEY_REQUEST: await self.handle_room_key_request(evt) + elif evt.type == EventType.BEEPER_ROOM_KEY_ACK: + await self.handle_beep_room_key_ack(evt) else: self.log.debug(f"Got unknown to-device event {evt.type} from {evt.sender}") @@ -210,14 +222,55 @@ async def _receive_room_key(self, evt: DecryptedOlmEvent) -> None: # for the case where evt.Keys.Ed25519 is none? if evt.content.algorithm != EncryptionAlgorithm.MEGOLM_V1 or not evt.keys.ed25519: return + if not evt.content.beeper_max_messages or not evt.content.beeper_max_age_ms: + encryption_info = await self.state_store.get_encryption_info(evt.content.room_id) + if encryption_info: + if not evt.content.beeper_max_age_ms: + evt.content.beeper_max_age_ms = encryption_info.rotation_period_ms + if not evt.content.beeper_max_messages: + evt.content.beeper_max_messages = encryption_info.rotation_period_msgs + if self.delete_previous_keys_on_receive and not evt.content.beeper_is_scheduled: + removed_ids = await self.crypto_store.redact_group_sessions( + evt.content.room_id, evt.sender_key, reason="received new key from device" + ) + self.log.info(f"Redacted previous megolm sessions: {removed_ids}") await self._create_group_session( evt.sender_key, evt.keys.ed25519, evt.content.room_id, evt.content.session_id, evt.content.session_key, + max_age=evt.content.beeper_max_age_ms, + max_messages=evt.content.beeper_max_messages, + is_scheduled=evt.content.beeper_is_scheduled, ) + async def handle_beep_room_key_ack(self, evt: ToDeviceEvent) -> None: + try: + sess = await self.crypto_store.get_group_session( + evt.content.room_id, evt.content.session_id + ) + except GroupSessionWithheldError: + self.log.debug( + f"Ignoring room key ack for session {evt.content.session_id}" + " that was already redacted" + ) + return + if not sess: + self.log.debug(f"Ignoring room key ack for unknown session {evt.content.session_id}") + return + if ( + sess.sender_key == self.account.identity_key + and self.delete_outbound_keys_on_ack + and evt.content.first_message_index == 0 + ): + self.log.debug("Redacting inbound copy of outbound group session after ack") + await self.crypto_store.redact_group_session( + evt.content.room_id, evt.content.session_id, reason="outbound session acked" + ) + else: + self.log.debug(f"Received room key ack for {sess.id}") + async def share_keys(self, current_otk_count: int) -> None: """ Share any keys that need to be shared. This is automatically called from diff --git a/mautrix/crypto/sessions.py b/mautrix/crypto/sessions.py index c8164b27..b0b16a18 100644 --- a/mautrix/crypto/sessions.py +++ b/mautrix/crypto/sessions.py @@ -3,10 +3,11 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from typing import List, Optional, Set, Tuple, cast +from typing import List, Optional, Set, Tuple, Union, cast from datetime import datetime, timedelta from _libolm import ffi, lib +from attr import dataclass import olm from mautrix.errors import EncryptionError @@ -18,8 +19,10 @@ OlmMsgType, RoomID, RoomKeyEventContent, + SerializableAttrs, SigningKey, UserID, + field, ) @@ -93,12 +96,25 @@ def describe(self) -> str: return "describe not supported" +@dataclass +class RatchetSafety(SerializableAttrs): + next_index: int = 0 + missed_indices: List[int] = field(factory=lambda: []) + lost_indices: List[int] = field(factory=lambda: []) + + class InboundGroupSession(olm.InboundGroupSession): room_id: RoomID signing_key: SigningKey sender_key: IdentityKey forwarding_chain: List[IdentityKey] + ratchet_safety: RatchetSafety + received_at: datetime + max_age: timedelta + max_messages: int + is_scheduled: bool + def __init__( self, session_key: str, @@ -106,11 +122,23 @@ def __init__( sender_key: IdentityKey, room_id: RoomID, forwarding_chain: Optional[List[IdentityKey]] = None, + ratchet_safety: Optional[RatchetSafety] = None, + received_at: Optional[datetime] = None, + max_age: Union[timedelta, int, None] = None, + max_messages: Optional[int] = None, + is_scheduled: bool = False, ) -> None: self.signing_key = signing_key self.sender_key = sender_key self.room_id = room_id self.forwarding_chain = forwarding_chain or [] + self.ratchet_safety = ratchet_safety or RatchetSafety() + self.received_at = received_at or datetime.utcnow() + if isinstance(max_age, int): + max_age = timedelta(milliseconds=max_age) + self.max_age = max_age + self.max_messages = max_messages + self.is_scheduled = is_scheduled super().__init__(session_key) def __new__(cls, *args, **kwargs): @@ -125,12 +153,22 @@ def from_pickle( sender_key: IdentityKey, room_id: RoomID, forwarding_chain: Optional[List[IdentityKey]] = None, + ratchet_safety: Optional[RatchetSafety] = None, + received_at: Optional[datetime] = None, + max_age: Optional[timedelta] = None, + max_messages: Optional[int] = None, + is_scheduled: bool = False, ) -> "InboundGroupSession": session = super().from_pickle(pickle, passphrase) session.signing_key = signing_key session.sender_key = sender_key session.room_id = room_id session.forwarding_chain = forwarding_chain or [] + session.ratchet_safety = ratchet_safety or RatchetSafety() + session.received_at = received_at + session.max_age = max_age + session.max_messages = max_messages + session.is_scheduled = is_scheduled return session @classmethod @@ -141,14 +179,41 @@ def import_session( sender_key: IdentityKey, room_id: RoomID, forwarding_chain: Optional[List[str]] = None, + ratchet_safety: Optional[RatchetSafety] = None, + received_at: Optional[datetime] = None, + max_age: Union[timedelta, int, None] = None, + max_messages: Optional[int] = None, + is_scheduled: bool = False, ) -> "InboundGroupSession": session = super().import_session(session_key) session.signing_key = signing_key session.sender_key = sender_key session.room_id = room_id session.forwarding_chain = forwarding_chain or [] + session.ratchet_safety = ratchet_safety or RatchetSafety() + session.received_at = received_at or datetime.utcnow() + if isinstance(max_age, int): + max_age = timedelta(milliseconds=max_age) + session.max_age = max_age + session.max_messages = max_messages + session.is_scheduled = is_scheduled return session + def ratchet_to(self, index: int) -> "InboundGroupSession": + exported = self.export_session(index) + return self.import_session( + exported, + signing_key=self.signing_key, + sender_key=self.sender_key, + room_id=self.room_id, + forwarding_chain=self.forwarding_chain, + ratchet_safety=self.ratchet_safety, + received_at=self.received_at, + max_age=self.max_age, + max_messages=self.max_messages, + is_scheduled=self.is_scheduled, + ) + class OutboundGroupSession(olm.OutboundGroupSession): """Outbound group session aware of the users it is shared with. diff --git a/mautrix/crypto/store/abstract.py b/mautrix/crypto/store/abstract.py index 7828524d..4db5aa1d 100644 --- a/mautrix/crypto/store/abstract.py +++ b/mautrix/crypto/store/abstract.py @@ -197,6 +197,46 @@ async def get_group_session( The :class:`InboundGroupSession` object, or ``None`` if not found. """ + @abstractmethod + async def redact_group_session( + self, room_id: RoomID, session_id: SessionID, reason: str + ) -> None: + """ + Remove the keys for a specific Megolm group session. + + Args: + room_id: The room where the session is. + session_id: The session ID to remove. + reason: The reason the session is being removed. + """ + + @abstractmethod + async def redact_group_sessions( + self, room_id: RoomID | None, sender_key: IdentityKey | None, reason: str + ) -> list[SessionID]: + """ + Remove the keys for multiple Megolm group sessions, + based on the room ID and/or sender device. + + Args: + room_id: The room ID to delete keys from. + sender_key: The Olm identity key of the device to delete keys from. + reason: The reason why the keys are being deleted. + + Returns: + The list of session IDs that were deleted. + """ + + @abstractmethod + async def redact_expired_group_sessions(self) -> list[SessionID]: + """ + Remove all Megolm group sessions where at least twice the maximum age has passed since + receiving the keys. + + Returns: + The list of session IDs that were deleted. + """ + @abstractmethod async def has_group_session(self, room_id: RoomID, session_id: SessionID) -> bool: """ diff --git a/mautrix/crypto/store/asyncpg/store.py b/mautrix/crypto/store/asyncpg/store.py index cb15be77..69f378f2 100644 --- a/mautrix/crypto/store/asyncpg/store.py +++ b/mautrix/crypto/store/asyncpg/store.py @@ -21,6 +21,7 @@ EventID, IdentityKey, RoomID, + RoomKeyWithheldCode, SessionID, SigningKey, SyncToken, @@ -31,7 +32,7 @@ from mautrix.util.async_db import Database, Scheme from mautrix.util.logging import TraceLogger -from ... import InboundGroupSession, OlmAccount, OutboundGroupSession, Session +from ... import InboundGroupSession, OlmAccount, OutboundGroupSession, RatchetSafety, Session from ..abstract import CryptoStore, StateStore from .upgrade import upgrade_table @@ -235,12 +236,15 @@ async def put_group_session( forwarding_chains = ",".join(session.forwarding_chain) q = """ INSERT INTO crypto_megolm_inbound_session ( - session_id, sender_key, signing_key, room_id, session, forwarding_chains, account_id - ) VALUES ($1, $2, $3, $4, $5, $6, $7) + session_id, sender_key, signing_key, room_id, session, forwarding_chains, + ratchet_safety, received_at, max_age, max_messages, is_scheduled, account_id + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) ON CONFLICT (session_id, account_id) DO UPDATE SET withheld_code=NULL, withheld_reason=NULL, sender_key=excluded.sender_key, signing_key=excluded.signing_key, room_id=excluded.room_id, session=excluded.session, - forwarding_chains=excluded.forwarding_chains + forwarding_chains=excluded.forwarding_chains, ratchet_safety=excluded.ratchet_safety, + received_at=excluded.received_at, max_age=excluded.max_age, + max_messages=excluded.max_messages, is_scheduled=excluded.is_scheduled """ try: await self.db.execute( @@ -251,6 +255,11 @@ async def put_group_session( room_id, pickle, forwarding_chains, + session.ratchet_safety.json(), + session.received_at, + int(session.max_age.total_seconds() * 1000), + session.max_messages, + session.is_scheduled, self.account_id, ) except (IntegrityError, UniqueViolationError): @@ -260,7 +269,9 @@ async def get_group_session( self, room_id: RoomID, session_id: SessionID ) -> InboundGroupSession | None: q = """ - SELECT sender_key, signing_key, session, forwarding_chains, withheld_code + SELECT + sender_key, signing_key, session, forwarding_chains, withheld_code, + ratchet_safety, received_at, max_age, max_messages, is_scheduled FROM crypto_megolm_inbound_session WHERE room_id=$1 AND session_id=$2 AND account_id=$3 """ @@ -277,7 +288,78 @@ async def get_group_session( sender_key=row["sender_key"], room_id=room_id, forwarding_chain=forwarding_chain, + ratchet_safety=RatchetSafety.parse_json(row["ratchet_safety"] or "{}"), + received_at=row["received_at"], + max_age=timedelta(milliseconds=row["max_age"]) if row["max_age"] else None, + max_messages=row["max_messages"], + is_scheduled=row["is_scheduled"], + ) + + async def redact_group_session( + self, room_id: RoomID, session_id: SessionID, reason: str + ) -> None: + q = """ + UPDATE crypto_megolm_inbound_session + SET withheld_code=$1, withheld_reason=$2, session=NULL, forwarding_chains=NULL + WHERE session_id=$3 AND account_id=$4 AND session IS NOT NULL + """ + await self.db.execute( + q, + RoomKeyWithheldCode.BEEPER_REDACTED.value, + f"Session redacted: {reason}", + session_id, + self.account_id, + ) + + async def redact_group_sessions( + self, room_id: RoomID, sender_key: IdentityKey, reason: str + ) -> list[SessionID]: + if not room_id and not sender_key: + raise ValueError("Either room_id or sender_key must be provided") + q = """ + UPDATE crypto_megolm_inbound_session + SET withheld_code=$1, withheld_reason=$2, session=NULL, forwarding_chains=NULL + WHERE (room_id=$3 OR $3='') AND (sender_key=$4 OR $4='') AND session IS NOT NULL AND account_id=$5 AND is_scheduled=false + RETURNING session_id + """ + rows = await self.db.fetch( + q, + RoomKeyWithheldCode.BEEPER_REDACTED.value, + f"Session redacted: {reason}", + room_id, + sender_key, + self.account_id, + ) + return [row["session_id"] for row in rows] + + async def redact_expired_group_sessions(self) -> list[SessionID]: + if self.db.scheme == Scheme.SQLITE: + q = """ + UPDATE crypto_megolm_inbound_session + SET withheld_code=$1, withheld_reason=$2, session=NULL, forwarding_chains=NULL + WHERE account_id=$3 AND session IS NOT NULL AND is_scheduled=false + AND received_at IS NOT NULL and max_age IS NOT NULL + AND unixepoch(received_at) + (2 * max_age / 1000) < unixepoch(date('now')) + RETURNING session_id + """ + elif self.db.scheme in (Scheme.POSTGRES, Scheme.COCKROACH): + q = """ + UPDATE crypto_megolm_inbound_session + SET withheld_code=$1, withheld_reason=$2, session=NULL, forwarding_chains=NULL + WHERE account_id=$3 AND session IS NOT NULL AND is_scheduled=false + AND received_at IS NOT NULL and max_age IS NOT NULL + AND received_at + 2 * (max_age * interval '1 millisecond') < now() + RETURNING session_id + """ + else: + raise RuntimeError(f"Unsupported dialect {self.db.scheme}") + rows = await self.db.fetch( + q, + RoomKeyWithheldCode.BEEPER_REDACTED.value, + f"Session redacted: expired", + self.account_id, ) + return [row["session_id"] for row in rows] async def has_group_session(self, room_id: RoomID, session_id: SessionID) -> bool: q = """ diff --git a/mautrix/crypto/store/asyncpg/upgrade.py b/mautrix/crypto/store/asyncpg/upgrade.py index 914da23d..e097c5d9 100644 --- a/mautrix/crypto/store/asyncpg/upgrade.py +++ b/mautrix/crypto/store/asyncpg/upgrade.py @@ -16,7 +16,7 @@ ) -@upgrade_table.register(description="Latest revision", upgrades_to=9) +@upgrade_table.register(description="Latest revision", upgrades_to=10) async def upgrade_blank_to_latest(conn: Connection) -> None: await conn.execute( """CREATE TABLE IF NOT EXISTS crypto_account ( @@ -77,6 +77,11 @@ async def upgrade_blank_to_latest(conn: Connection) -> None: forwarding_chains TEXT, withheld_code TEXT, withheld_reason TEXT, + ratchet_safety jsonb, + received_at timestamp, + max_age BIGINT, + max_messages INTEGER, + is_scheduled BOOLEAN NOT NULL DEFAULT false, PRIMARY KEY (account_id, session_id) )""" ) @@ -420,3 +425,19 @@ async def upgrade_v9_sqlite(conn: Connection) -> None: await conn.execute("PRAGMA foreign_key_check") await conn.execute("PRAGMA foreign_keys = ON") + + +@upgrade_table.register( + description="Add metadata for detecting when megolm sessions are safe to delete" +) +async def upgrade_v10(conn: Connection) -> None: + await conn.execute("ALTER TABLE crypto_megolm_inbound_session ADD COLUMN ratchet_safety jsonb") + await conn.execute( + "ALTER TABLE crypto_megolm_inbound_session ADD COLUMN received_at timestamp" + ) + await conn.execute("ALTER TABLE crypto_megolm_inbound_session ADD COLUMN max_age BIGINT") + await conn.execute("ALTER TABLE crypto_megolm_inbound_session ADD COLUMN max_messages INTEGER") + await conn.execute( + "ALTER TABLE crypto_megolm_inbound_session " + "ADD COLUMN is_scheduled BOOLEAN NOT NULL DEFAULT false" + ) diff --git a/mautrix/crypto/store/memory.py b/mautrix/crypto/store/memory.py index 35dc26b5..cfc27125 100644 --- a/mautrix/crypto/store/memory.py +++ b/mautrix/crypto/store/memory.py @@ -110,6 +110,30 @@ async def get_group_session( ) -> InboundGroupSession: return self._inbound_sessions.get((room_id, session_id)) + async def redact_group_session( + self, room_id: RoomID, session_id: SessionID, reason: str + ) -> None: + self._inbound_sessions.pop((room_id, session_id), None) + + async def redact_group_sessions( + self, room_id: RoomID, sender_key: IdentityKey, reason: str + ) -> list[SessionID]: + if not room_id and not sender_key: + raise ValueError("Either room_id or sender_key must be provided") + deleted = [] + keys = list(self._inbound_sessions.keys()) + for key in keys: + item = self._inbound_sessions[key] + if (not room_id or item.room_id == room_id) and ( + not sender_key or item.sender_key == sender_key + ): + deleted.append(SessionID(item.id)) + del self._inbound_sessions[key] + return deleted + + async def redact_expired_group_sessions(self) -> list[SessionID]: + raise NotImplementedError() + async def has_group_session(self, room_id: RoomID, session_id: SessionID) -> bool: return (room_id, session_id) in self._inbound_sessions diff --git a/mautrix/types/event/beeper.py b/mautrix/types/event/beeper.py index 23a9f5cd..0ec16479 100644 --- a/mautrix/types/event/beeper.py +++ b/mautrix/types/event/beeper.py @@ -7,7 +7,7 @@ from attr import dataclass -from ..primitive import EventID +from ..primitive import EventID, RoomID, SessionID from ..util import SerializableAttrs, SerializableEnum, field from .base import BaseRoomEvent from .message import RelatesTo @@ -55,3 +55,10 @@ class BeeperMessageStatusEventContent(SerializableAttrs): @dataclass class BeeperMessageStatusEvent(BaseRoomEvent, SerializableAttrs): content: BeeperMessageStatusEventContent + + +@dataclass +class BeeperRoomKeyAckEventContent(SerializableAttrs): + room_id: RoomID + session_id: SessionID + first_message_index: int diff --git a/mautrix/types/event/to_device.py b/mautrix/types/event/to_device.py index 6a17e36e..d6392177 100644 --- a/mautrix/types/event/to_device.py +++ b/mautrix/types/event/to_device.py @@ -9,8 +9,9 @@ import attr from ..primitive import JSON, DeviceID, IdentityKey, RoomID, SessionID, SigningKey, UserID -from ..util import ExtensibleEnum, Obj, SerializableAttrs, deserializer +from ..util import ExtensibleEnum, Obj, SerializableAttrs, deserializer, field from .base import BaseEvent, EventType +from .beeper import BeeperRoomKeyAckEventContent from .encrypted import EncryptedOlmEventContent, EncryptionAlgorithm @@ -21,6 +22,8 @@ class RoomKeyWithheldCode(ExtensibleEnum): UNAVAILABLE: "RoomKeyWithheldCode" = "m.unavailable" NO_OLM_SESSION: "RoomKeyWithheldCode" = "m.no_olm" + BEEPER_REDACTED: "RoomKeyWithheldCode" = "com.beeper.redacted" + @dataclass class RoomKeyWithheldEventContent(SerializableAttrs): @@ -39,6 +42,10 @@ class RoomKeyEventContent(SerializableAttrs): session_id: SessionID session_key: str + beeper_max_age_ms: Optional[int] = field(json="com.beeper.max_age_ms", default=None) + beeper_max_messages: Optional[int] = field(json="com.beeper.max_messages", default=None) + beeper_is_scheduled: Optional[bool] = field(json="com.beeper.is_scheduled", default=False) + class KeyRequestAction(ExtensibleEnum): REQUEST: "KeyRequestAction" = "request" @@ -61,7 +68,7 @@ class RoomKeyRequestEventContent(SerializableAttrs): body: Optional[RequestedKeyInfo] = None -@dataclass +@dataclass(kw_only=True) class ForwardedRoomKeyEventContent(RoomKeyEventContent, SerializableAttrs): sender_key: IdentityKey signing_key: SigningKey = attr.ib(metadata={"json": "sender_claimed_ed25519_key"}) @@ -75,6 +82,7 @@ class ForwardedRoomKeyEventContent(RoomKeyEventContent, SerializableAttrs): RoomKeyEventContent, RoomKeyRequestEventContent, ForwardedRoomKeyEventContent, + BeeperRoomKeyAckEventContent, ] to_device_event_content_map = { EventType.TO_DEVICE_ENCRYPTED: EncryptedOlmEventContent, @@ -82,12 +90,10 @@ class ForwardedRoomKeyEventContent(RoomKeyEventContent, SerializableAttrs): EventType.ROOM_KEY_REQUEST: RoomKeyRequestEventContent, EventType.ROOM_KEY: RoomKeyEventContent, EventType.FORWARDED_ROOM_KEY: ForwardedRoomKeyEventContent, + EventType.BEEPER_ROOM_KEY_ACK: BeeperRoomKeyAckEventContent, } -# TODO remaining account data event types - - @dataclass class ToDeviceEvent(BaseEvent, SerializableAttrs): sender: UserID diff --git a/mautrix/types/event/type.py b/mautrix/types/event/type.py index 609f40ff..509d6857 100644 --- a/mautrix/types/event/type.py +++ b/mautrix/types/event/type.py @@ -216,6 +216,7 @@ def is_to_device(self) -> bool: "m.room_key_request": "ROOM_KEY_REQUEST", "m.forwarded_room_key": "FORWARDED_ROOM_KEY", "m.dummy": "TO_DEVICE_DUMMY", + "com.beeper.room_key.ack": "BEEPER_ROOM_KEY_ACK", }, EventType.Class.UNKNOWN: { "__ALL__": "ALL", # This is not a real event type diff --git a/mautrix/types/event/type.pyi b/mautrix/types/event/type.pyi index 6c4c6e65..bee0fde0 100644 --- a/mautrix/types/event/type.pyi +++ b/mautrix/types/event/type.pyi @@ -68,6 +68,7 @@ class EventType(Serializable): ORG_MATRIX_ROOM_KEY_WITHHELD: "EventType" ROOM_KEY_REQUEST: "EventType" FORWARDED_ROOM_KEY: "EventType" + BEEPER_ROOM_KEY_ACK: "EventType" ALL: "EventType" From b5b81108f6c4f45637a8501fda163a8a6d6622b6 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 13 Apr 2023 17:24:17 +0300 Subject: [PATCH 075/218] Don't redact old megolm sessions Old meaning received before message index tracking was implemented --- mautrix/crypto/store/asyncpg/store.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mautrix/crypto/store/asyncpg/store.py b/mautrix/crypto/store/asyncpg/store.py index 69f378f2..c070b949 100644 --- a/mautrix/crypto/store/asyncpg/store.py +++ b/mautrix/crypto/store/asyncpg/store.py @@ -319,7 +319,8 @@ async def redact_group_sessions( q = """ UPDATE crypto_megolm_inbound_session SET withheld_code=$1, withheld_reason=$2, session=NULL, forwarding_chains=NULL - WHERE (room_id=$3 OR $3='') AND (sender_key=$4 OR $4='') AND session IS NOT NULL AND account_id=$5 AND is_scheduled=false + WHERE (room_id=$3 OR $3='') AND (sender_key=$4 OR $4='') AND account_id=$5 + AND session IS NOT NULL AND is_scheduled=false AND received_at IS NOT NULL RETURNING session_id """ rows = await self.db.fetch( From e44579bb9000a04c78e8a2b25cce4e81926f8952 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 13 Apr 2023 21:20:28 +0300 Subject: [PATCH 076/218] Bump version to 0.19.10 --- CHANGELOG.md | 5 +++++ mautrix/__init__.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e7cabdd6..74c384f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +## v0.19.10 (2023-04-13) + +* *(crypto, bridge)* Added options to automatically ratchet/delete megolm + sessions to minimize access to old messages. + ## v0.19.9 (2023-04-12) * *(crypto)* Fixed bug in crypto store migration when using outbound sessions diff --git a/mautrix/__init__.py b/mautrix/__init__.py index de48a11a..2cbb2011 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.19.9" +__version__ = "0.19.10" __author__ = "Tulir Asokan " __all__ = [ "api", From 57a7a5c03697178e5cadc8477a2297840617168b Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 13 Apr 2023 23:30:04 +0300 Subject: [PATCH 077/218] Fix putting group sessions with no max_age Fixes mautrix/signal#355 --- mautrix/crypto/store/asyncpg/store.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/crypto/store/asyncpg/store.py b/mautrix/crypto/store/asyncpg/store.py index c070b949..0642d210 100644 --- a/mautrix/crypto/store/asyncpg/store.py +++ b/mautrix/crypto/store/asyncpg/store.py @@ -257,7 +257,7 @@ async def put_group_session( forwarding_chains, session.ratchet_safety.json(), session.received_at, - int(session.max_age.total_seconds() * 1000), + int(session.max_age.total_seconds() * 1000) if session.max_age else None, session.max_messages, session.is_scheduled, self.account_id, From 1e464f8e8b0722215c82b04e2902bf5e1fb1b335 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 14 Apr 2023 12:07:05 +0300 Subject: [PATCH 078/218] Fetch encryption info from server if not cached --- mautrix/crypto/base.py | 34 +++++++++++++++++++++++++++++++++- mautrix/crypto/key_request.py | 10 ++-------- mautrix/crypto/machine.py | 9 ++------- 3 files changed, 37 insertions(+), 16 deletions(-) diff --git a/mautrix/crypto/base.py b/mautrix/crypto/base.py index f403feee..5af3ec0a 100644 --- a/mautrix/crypto/base.py +++ b/mautrix/crypto/base.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022 Tulir Asokan +# Copyright (c) 2023 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this @@ -12,13 +12,17 @@ import olm +from mautrix.errors import MForbidden, MNotFound from mautrix.types import ( DeviceID, EncryptionKeyAlgorithm, + EventType, IdentityKey, KeyID, RequestedKeyInfo, + RoomEncryptionStateEventContent, RoomID, + RoomKeyEventContent, SessionID, SigningKey, TrustState, @@ -82,6 +86,34 @@ def _mark_session_received(self, session_id: SessionID) -> None: except KeyError: return + async def _fill_encryption_info(self, evt: RoomKeyEventContent) -> None: + encryption_info = await self.state_store.get_encryption_info(evt.room_id) + if not encryption_info: + self.log.warning( + f"Encryption info for {evt.room_id} not found in state store, fetching from server" + ) + try: + encryption_info = await self.client.get_state_event( + evt.room_id, EventType.ROOM_ENCRYPTION + ) + except (MNotFound, MForbidden) as e: + self.log.warning( + f"Failed to get encryption info for {evt.room_id} from server: {e}," + " using defaults" + ) + encryption_info = RoomEncryptionStateEventContent() + if not encryption_info: + self.log.warning( + f"Didn't find encryption info for {evt.room_id} on server either," + " using defaults" + ) + encryption_info = RoomEncryptionStateEventContent() + + if not evt.beeper_max_age_ms: + evt.beeper_max_age_ms = encryption_info.rotation_period_ms + if not evt.beeper_max_messages: + evt.beeper_max_messages = encryption_info.rotation_period_msgs + canonical_json = functools.partial( json.dumps, ensure_ascii=False, separators=(",", ":"), sort_keys=True diff --git a/mautrix/crypto/key_request.py b/mautrix/crypto/key_request.py index 600ecc75..ecaf994d 100644 --- a/mautrix/crypto/key_request.py +++ b/mautrix/crypto/key_request.py @@ -1,10 +1,9 @@ -# Copyright (c) 2022 Tulir Asokan +# Copyright (c) 2023 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. from typing import Dict, List, Optional, Union -from datetime import timedelta import asyncio import uuid @@ -122,12 +121,7 @@ async def _receive_forwarded_room_key(self, evt: DecryptedOlmEvent) -> None: ) return if not key.beeper_max_messages or not key.beeper_max_age_ms: - encryption_info = await self.state_store.get_encryption_info(key.room_id) - if encryption_info: - if not key.beeper_max_age_ms: - key.beeper_max_age_ms = encryption_info.rotation_period_ms - if not key.beeper_max_messages: - key.beeper_max_messages = encryption_info.rotation_period_msgs + await self._fill_encryption_info(key) key.forwarding_key_chain.append(evt.sender_key) sess = InboundGroupSession.import_session( key.session_key, diff --git a/mautrix/crypto/machine.py b/mautrix/crypto/machine.py index fcefc3e3..9cf53312 100644 --- a/mautrix/crypto/machine.py +++ b/mautrix/crypto/machine.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022 Tulir Asokan +# Copyright (c) 2023 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this @@ -223,12 +223,7 @@ async def _receive_room_key(self, evt: DecryptedOlmEvent) -> None: if evt.content.algorithm != EncryptionAlgorithm.MEGOLM_V1 or not evt.keys.ed25519: return if not evt.content.beeper_max_messages or not evt.content.beeper_max_age_ms: - encryption_info = await self.state_store.get_encryption_info(evt.content.room_id) - if encryption_info: - if not evt.content.beeper_max_age_ms: - evt.content.beeper_max_age_ms = encryption_info.rotation_period_ms - if not evt.content.beeper_max_messages: - evt.content.beeper_max_messages = encryption_info.rotation_period_msgs + await self._fill_encryption_info(evt.content) if self.delete_previous_keys_on_receive and not evt.content.beeper_is_scheduled: removed_ids = await self.crypto_store.redact_group_sessions( evt.content.room_id, evt.sender_key, reason="received new key from device" From 2f57efa20d2af76a1cb8acd931f6f67d6c156acd Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 14 Apr 2023 12:10:30 +0300 Subject: [PATCH 079/218] Bump version to 0.19.11 --- CHANGELOG.md | 8 ++++++++ mautrix/__init__.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 74c384f5..55ecc068 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,11 @@ +## v0.19.11 (2023-04-14) + +* *(crypto)* Fixed bug in previous release which caused errors if the `max_age` + of a megolm session was not known. +* *(crypto)* Changed key receiving handler to fetch encryption config from + server if it's not cached locally (to find `max_age` and `max_messages` more + reliably). + ## v0.19.10 (2023-04-13) * *(crypto, bridge)* Added options to automatically ratchet/delete megolm diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 2cbb2011..e9e88943 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.19.10" +__version__ = "0.19.11" __author__ = "Tulir Asokan " __all__ = [ "api", From a1443a7542eca7d82148dfa9ea6a25340dccf38e Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 17 Apr 2023 17:16:08 +0300 Subject: [PATCH 080/218] Fix backwards compatibility with old bridges --- mautrix/bridge/e2ee.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/mautrix/bridge/e2ee.py b/mautrix/bridge/e2ee.py index 95b419f7..805efd4f 100644 --- a/mautrix/bridge/e2ee.py +++ b/mautrix/bridge/e2ee.py @@ -119,15 +119,17 @@ def __init__( self.az.device_list_handler = self.crypto.handle_as_device_lists self.az.to_device_handler = self.crypto.handle_as_to_device_event - delete_cfg = bridge.config["bridge.encryption.delete_keys"] - self.crypto.delete_outbound_keys_on_ack = delete_cfg["delete_outbound_on_ack"] - self.crypto.dont_store_outbound_keys = delete_cfg["dont_store_outbound"] - self.crypto.delete_previous_keys_on_receive = delete_cfg["delete_prev_on_new_session"] - self.crypto.ratchet_keys_on_decrypt = delete_cfg["ratchet_on_decrypt"] - self.crypto.delete_fully_used_keys_on_decrypt = delete_cfg["delete_fully_used_on_decrypt"] - self.crypto.delete_keys_on_device_delete = delete_cfg["delete_on_device_delete"] - self.periodically_delete_expired_keys = delete_cfg["periodically_delete_expired"] + self.periodically_delete_expired_keys = False self._key_delete_task = None + del_cfg = bridge.config["bridge.encryption.delete_keys"] + if del_cfg: + self.crypto.delete_outbound_keys_on_ack = del_cfg["delete_outbound_on_ack"] + self.crypto.dont_store_outbound_keys = del_cfg["dont_store_outbound"] + self.crypto.delete_previous_keys_on_receive = del_cfg["delete_prev_on_new_session"] + self.crypto.ratchet_keys_on_decrypt = del_cfg["ratchet_on_decrypt"] + self.crypto.delete_fully_used_keys_on_decrypt = del_cfg["delete_fully_used_on_decrypt"] + self.crypto.delete_keys_on_device_delete = del_cfg["delete_on_device_delete"] + self.periodically_delete_expired_keys = del_cfg["periodically_delete_expired"] async def _exit_on_sync_fail(self, data) -> None: if data["error"]: From 1bf463552890833db01301648def311d2d7e0dd5 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 18 Apr 2023 19:03:36 +0300 Subject: [PATCH 081/218] Remove is_bot flag for bridge bots --- mautrix/bridge/matrix.py | 1 - 1 file changed, 1 deletion(-) diff --git a/mautrix/bridge/matrix.py b/mautrix/bridge/matrix.py index 6ce3bfde..b60706cd 100644 --- a/mautrix/bridge/matrix.py +++ b/mautrix/bridge/matrix.py @@ -279,7 +279,6 @@ async def init_as_bot(self) -> None: "com.beeper.bridge.service": self.bridge.beeper_service_name, "com.beeper.bridge.network": self.bridge.beeper_network_name, "com.beeper.bridge.is_bridge_bot": True, - "com.beeper.bridge.is_bot": True, } ) From aa8d1c89d581c9aa99e60c91205388dd1a820deb Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 18 Apr 2023 19:04:38 +0300 Subject: [PATCH 082/218] Bump version to 0.19.12 --- CHANGELOG.md | 4 ++++ mautrix/__init__.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 55ecc068..96a206c9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## v0.19.12 (2023-04-18) + +* *(bridge)* Fixed backwards-compatibility with new key deletion config options. + ## v0.19.11 (2023-04-14) * *(crypto)* Fixed bug in previous release which caused errors if the `max_age` diff --git a/mautrix/__init__.py b/mautrix/__init__.py index e9e88943..1f48593d 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.19.11" +__version__ = "0.19.12" __author__ = "Tulir Asokan " __all__ = [ "api", From 0b69dd6af0c2a3126dd235a1c762ac77336e2e10 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 24 Apr 2023 18:33:26 +0300 Subject: [PATCH 083/218] Fix deleting megolm sessions on device removal --- CHANGELOG.md | 4 ++++ mautrix/__init__.py | 2 +- mautrix/crypto/device_lists.py | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 96a206c9..730f2a2d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## v0.19.13 (2023-04-24) + +* *(crypto)* Fixed bug with redacting megolm sessions when device is deleted. + ## v0.19.12 (2023-04-18) * *(bridge)* Fixed backwards-compatibility with new key deletion config options. diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 1f48593d..440745cf 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.19.12" +__version__ = "0.19.13" __author__ = "Tulir Asokan " __all__ = [ "api", diff --git a/mautrix/crypto/device_lists.py b/mautrix/crypto/device_lists.py index 9afe7b91..d49c7e19 100644 --- a/mautrix/crypto/device_lists.py +++ b/mautrix/crypto/device_lists.py @@ -82,7 +82,7 @@ async def _fetch_keys( if changed or len(new_devices) != len(existing_devices): if self.delete_keys_on_device_delete: - for device_id in new_devices.keys() - existing_devices.keys(): + for device_id in existing_devices.keys() - new_devices.keys(): device = existing_devices[device_id] removed_ids = await self.crypto_store.redact_group_sessions( room_id=None, sender_key=device.identity_key, reason="device removed" From 714d6eb450a28c398ff29355ec2718971781cc1d Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 1 May 2023 10:55:16 +0300 Subject: [PATCH 084/218] Implement MSC2659 --- mautrix/appservice/api/intent.py | 8 ++++++ mautrix/appservice/appservice.py | 3 +++ mautrix/appservice/as_handler.py | 19 +++++++++++++ mautrix/bridge/bridge.py | 3 +++ mautrix/bridge/matrix.py | 46 +++++++++++++++++++++----------- 5 files changed, 63 insertions(+), 16 deletions(-) diff --git a/mautrix/appservice/api/intent.py b/mautrix/appservice/api/intent.py index 8b5ca16f..e09db0d0 100644 --- a/mautrix/appservice/api/intent.py +++ b/mautrix/appservice/api/intent.py @@ -489,6 +489,14 @@ async def mark_read( ) self.state_store.set_read(room_id, self.mxid, event_id) + async def appservice_ping(self, appservice_id: str, txn_id: str | None = None) -> int: + resp = await self.api.request( + Method.POST, + Path.unstable["fi.mau.msc2659"].appservice[appservice_id].ping, + content={"transaction_id": txn_id} if txn_id is not None else {}, + ) + return resp.get("duration_ms") or resp.get("duration") or -1 + async def batch_send( self, room_id: RoomID, diff --git a/mautrix/appservice/appservice.py b/mautrix/appservice/appservice.py index 551eab5c..892db195 100644 --- a/mautrix/appservice/appservice.py +++ b/mautrix/appservice/appservice.py @@ -194,3 +194,6 @@ async def _liveness_probe(self, _: web.Request) -> web.Response: async def _readiness_probe(self, _: web.Request) -> web.Response: return web.Response(status=200 if self.ready else 500, text="{}") + + async def ping_self(self, txn_id: str | None = None) -> int: + return await self.intent.appservice_ping(self.id, txn_id=txn_id) diff --git a/mautrix/appservice/as_handler.py b/mautrix/appservice/as_handler.py index 78cd4307..2adcb2de 100644 --- a/mautrix/appservice/as_handler.py +++ b/mautrix/appservice/as_handler.py @@ -75,6 +75,8 @@ def register_routes(self, app: web.Application) -> None: ) app.router.add_route("GET", "/_matrix/app/v1/rooms/{alias}", self._http_query_alias) app.router.add_route("GET", "/_matrix/app/v1/users/{user_id}", self._http_query_user) + app.router.add_route("POST", "/_matrix/app/v1/ping", self._http_ping) + app.router.add_route("POST", "/_matrix/app/unstable/fi.mau.msc2659/ping", self._http_ping) def _check_token(self, request: web.Request) -> bool: try: @@ -128,6 +130,23 @@ async def _http_query_alias(self, request: web.Request) -> web.Response: return web.json_response({}, status=404) return web.json_response(response) + async def _http_ping(self, request: web.Request) -> web.Response: + if not self._check_token(request): + raise web.HTTPUnauthorized( + content_type="application/json", + text=json.dumps({"error": "Invalid auth token", "errcode": "M_UNKNOWN_TOKEN"}), + ) + try: + body = await request.json() + except JSONDecodeError: + raise web.HTTPBadRequest( + content_type="application/json", + text=json.dumps({"error": "Body is not JSON", "errcode": "M_NOT_JSON"}), + ) + txn_id = body.get("transaction_id") + self.log.info(f"Received ping from homeserver with transaction ID {txn_id}") + return web.json_response({}) + @staticmethod def _get_with_fallback( json: dict[str, Any], field: str, unstable_prefix: str, default: Any = None diff --git a/mautrix/bridge/bridge.py b/mautrix/bridge/bridge.py index d8241038..c474109b 100644 --- a/mautrix/bridge/bridge.py +++ b/mautrix/bridge/bridge.py @@ -246,6 +246,9 @@ async def start(self) -> None: "correct, and do they match the values in the registration?" ) sys.exit(16) + except Exception: + self.log.critical("Failed to check connection to homeserver", exc_info=True) + sys.exit(16) await self.matrix.init_encryption() self.add_startup_actions(self.matrix.init_as_bot()) diff --git a/mautrix/bridge/matrix.py b/mautrix/bridge/matrix.py index b60706cd..e6b280a5 100644 --- a/mautrix/bridge/matrix.py +++ b/mautrix/bridge/matrix.py @@ -222,33 +222,47 @@ async def check_versions(self) -> None: async def wait_for_connection(self) -> None: self.log.info("Ensuring connectivity to homeserver") errors = 0 - tried_to_register = False while True: try: self.versions = await self.az.intent.versions() - await self.check_versions() - await self.az.intent.whoami() break - except (MUnknownToken, MExclusive): - # These are probably not going to resolve themselves by waiting - raise - except MForbidden: - if not tried_to_register: - self.log.debug( - "Whoami endpoint returned M_FORBIDDEN, " - "trying to register bridge bot before retrying..." - ) - await self.az.intent.ensure_registered() - tried_to_register = True - else: - raise except Exception: errors += 1 if errors <= 6: self.log.exception("Connection to homeserver failed, retrying in 10 seconds") await asyncio.sleep(10) + continue else: raise + await self.check_versions() + try: + await self.az.intent.whoami() + except (MUnknownToken, MExclusive): + raise + except MForbidden: + self.log.debug( + "Whoami endpoint returned M_FORBIDDEN, " + "trying to register bridge bot before retrying..." + ) + await self.az.intent.ensure_registered() + await self.az.intent.whoami() + if self.versions.supports("fi.mau.msc2659"): + try: + txn_id = self.az.intent.api.get_txn_id() + duration = await self.az.ping_self(txn_id) + self.log.debug( + "Homeserver->bridge connection works, " + f"roundtrip time is {duration} ms (txn ID: {txn_id})" + ) + except Exception: + self.log.exception( + "Error checking homeserver -> bridge connection, retrying in 10 seconds" + ) + sys.exit(16) + else: + self.log.debug( + "Homeserver does not support checking status of homeserver -> bridge connection" + ) try: self.media_config = await self.az.intent.get_media_repo_config() except Exception: From 7330fb17b84879203ec97e88c0d4cab7ddf6530f Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 1 May 2023 21:50:23 +0300 Subject: [PATCH 085/218] Add beeper_update_profile to methods that call ensure_registered --- mautrix/appservice/api/intent.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mautrix/appservice/api/intent.py b/mautrix/appservice/api/intent.py index e09db0d0..22a93354 100644 --- a/mautrix/appservice/api/intent.py +++ b/mautrix/appservice/api/intent.py @@ -71,6 +71,7 @@ def quote(*args, **kwargs): ClientAPI.search_users, ClientAPI.set_displayname, ClientAPI.set_avatar_url, + ClientAPI.beeper_update_profile, ClientAPI.unstable_create_mxc, ClientAPI.upload_media, ClientAPI.send_receipt, From e1b0dfdeb33953561d796f01a25f858550be259c Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 2 May 2023 15:11:01 +0300 Subject: [PATCH 086/218] Stabilize MSC2659 --- mautrix/appservice/api/intent.py | 4 ++-- mautrix/bridge/matrix.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mautrix/appservice/api/intent.py b/mautrix/appservice/api/intent.py index 22a93354..e52d8016 100644 --- a/mautrix/appservice/api/intent.py +++ b/mautrix/appservice/api/intent.py @@ -493,10 +493,10 @@ async def mark_read( async def appservice_ping(self, appservice_id: str, txn_id: str | None = None) -> int: resp = await self.api.request( Method.POST, - Path.unstable["fi.mau.msc2659"].appservice[appservice_id].ping, + Path.v1.appservice[appservice_id].ping, content={"transaction_id": txn_id} if txn_id is not None else {}, ) - return resp.get("duration_ms") or resp.get("duration") or -1 + return resp.get("duration_ms") or -1 async def batch_send( self, diff --git a/mautrix/bridge/matrix.py b/mautrix/bridge/matrix.py index e6b280a5..45d087f2 100644 --- a/mautrix/bridge/matrix.py +++ b/mautrix/bridge/matrix.py @@ -246,7 +246,7 @@ async def wait_for_connection(self) -> None: ) await self.az.intent.ensure_registered() await self.az.intent.whoami() - if self.versions.supports("fi.mau.msc2659"): + if self.versions.supports("fi.mau.msc2659.stable"): try: txn_id = self.az.intent.api.get_txn_id() duration = await self.az.ping_self(txn_id) From 3d90a2e89e2472834c240d034e68b5d6ce4470bc Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 2 May 2023 15:11:08 +0300 Subject: [PATCH 087/218] Add spec version constants --- mautrix/types/versions.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mautrix/types/versions.py b/mautrix/types/versions.py index 8b42de39..388927e8 100644 --- a/mautrix/types/versions.py +++ b/mautrix/types/versions.py @@ -70,6 +70,9 @@ class SpecVersions: V11 = Version.deserialize("v1.1") V12 = Version.deserialize("v1.2") V13 = Version.deserialize("v1.3") + V14 = Version.deserialize("v1.4") + V15 = Version.deserialize("v1.5") + V16 = Version.deserialize("v1.6") @dataclass From 7b6c727c6d2a0e29bc957609cc0bea1de1f7ac3f Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 5 May 2023 15:00:02 +0300 Subject: [PATCH 088/218] Retry /whoami forever in bridges like mautrix-go --- mautrix/bridge/matrix.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/mautrix/bridge/matrix.py b/mautrix/bridge/matrix.py index 45d087f2..a525b9ac 100644 --- a/mautrix/bridge/matrix.py +++ b/mautrix/bridge/matrix.py @@ -221,19 +221,13 @@ async def check_versions(self) -> None: async def wait_for_connection(self) -> None: self.log.info("Ensuring connectivity to homeserver") - errors = 0 while True: try: self.versions = await self.az.intent.versions() break except Exception: - errors += 1 - if errors <= 6: - self.log.exception("Connection to homeserver failed, retrying in 10 seconds") - await asyncio.sleep(10) - continue - else: - raise + self.log.exception("Connection to homeserver failed, retrying in 10 seconds") + await asyncio.sleep(10) await self.check_versions() try: await self.az.intent.whoami() From 3a339a2f91f93b1eec2b4f16eabb7d6655455281 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 5 May 2023 15:00:52 +0300 Subject: [PATCH 089/218] Enable AS ping if server advertises spec v1.7 --- mautrix/bridge/matrix.py | 4 +++- mautrix/types/versions.py | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/mautrix/bridge/matrix.py b/mautrix/bridge/matrix.py index a525b9ac..204fbbeb 100644 --- a/mautrix/bridge/matrix.py +++ b/mautrix/bridge/matrix.py @@ -240,7 +240,9 @@ async def wait_for_connection(self) -> None: ) await self.az.intent.ensure_registered() await self.az.intent.whoami() - if self.versions.supports("fi.mau.msc2659.stable"): + if self.versions.supports("fi.mau.msc2659.stable") or self.versions.supports_at_least( + SpecVersions.V17 + ): try: txn_id = self.az.intent.api.get_txn_id() duration = await self.az.ping_self(txn_id) diff --git a/mautrix/types/versions.py b/mautrix/types/versions.py index 388927e8..9ad81710 100644 --- a/mautrix/types/versions.py +++ b/mautrix/types/versions.py @@ -73,6 +73,7 @@ class SpecVersions: V14 = Version.deserialize("v1.4") V15 = Version.deserialize("v1.5") V16 = Version.deserialize("v1.6") + V17 = Version.deserialize("v1.7") @dataclass From 858f98c567eb196e58d16313d9bf6340df96c8c8 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 5 May 2023 15:01:35 +0300 Subject: [PATCH 090/218] Remove redundant except --- mautrix/bridge/matrix.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/mautrix/bridge/matrix.py b/mautrix/bridge/matrix.py index 204fbbeb..1fa1e589 100644 --- a/mautrix/bridge/matrix.py +++ b/mautrix/bridge/matrix.py @@ -231,8 +231,6 @@ async def wait_for_connection(self) -> None: await self.check_versions() try: await self.az.intent.whoami() - except (MUnknownToken, MExclusive): - raise except MForbidden: self.log.debug( "Whoami endpoint returned M_FORBIDDEN, " From fda258a0f124b72dcf140a44cc7bf6fec3a89737 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 5 May 2023 15:02:13 +0300 Subject: [PATCH 091/218] Fix error message --- mautrix/bridge/matrix.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/mautrix/bridge/matrix.py b/mautrix/bridge/matrix.py index 1fa1e589..957aa9be 100644 --- a/mautrix/bridge/matrix.py +++ b/mautrix/bridge/matrix.py @@ -249,9 +249,7 @@ async def wait_for_connection(self) -> None: f"roundtrip time is {duration} ms (txn ID: {txn_id})" ) except Exception: - self.log.exception( - "Error checking homeserver -> bridge connection, retrying in 10 seconds" - ) + self.log.exception("Error checking homeserver -> bridge connection") sys.exit(16) else: self.log.debug( From cd23618415e1fde197df955ab3bafd428eae391a Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 5 May 2023 15:11:27 +0300 Subject: [PATCH 092/218] Send MSS events for commands --- mautrix/bridge/matrix.py | 56 +++++++++++++++++++++++++++++++--------- 1 file changed, 44 insertions(+), 12 deletions(-) diff --git a/mautrix/bridge/matrix.py b/mautrix/bridge/matrix.py index 957aa9be..dc7d9fcf 100644 --- a/mautrix/bridge/matrix.py +++ b/mautrix/bridge/matrix.py @@ -557,6 +557,28 @@ def is_command(self, message: MessageEventContent) -> tuple[bool, str]: text = text[len(prefix) + 1 :].lstrip() return is_command, text + async def _send_mss( + self, + evt: Event, + status: MessageStatus, + reason: MessageStatusReason | None = None, + error: str | None = None, + message: str | None = None, + ) -> None: + if not self.config.get("bridge.message_status_events", False): + return + status_content = BeeperMessageStatusEventContent( + network="", # TODO set network properly + relates_to=RelatesTo(rel_type=RelationType.REFERENCE, event_id=evt.event_id), + status=status, + reason=reason, + error=error, + message=message, + ) + await self.az.intent.send_message_event( + evt.room_id, EventType.BEEPER_MESSAGE_STATUS, status_content + ) + async def _send_crypto_status_error( self, evt: Event, @@ -594,18 +616,13 @@ async def _send_crypto_status_error( "it by default on the bridge (bridge -> encryption -> default)." ) - if self.config.get("bridge.message_status_events", False): - status_content = BeeperMessageStatusEventContent( - network="", # TODO set network properly - relates_to=RelatesTo(rel_type=RelationType.REFERENCE, event_id=evt.event_id), - status=MessageStatus.RETRIABLE if is_final else MessageStatus.PENDING, - reason=MessageStatusReason.UNDECRYPTABLE, - error=msg, - message=err.human_message if err else None, - ) - await self.az.intent.send_message_event( - evt.room_id, EventType.BEEPER_MESSAGE_STATUS, status_content - ) + await self._send_mss( + evt, + status=MessageStatus.RETRIABLE if is_final else MessageStatus.PENDING, + reason=MessageStatusReason.UNDECRYPTABLE, + error=msg, + message=err.human_message if err else None, + ) return event_id @@ -697,6 +714,13 @@ async def handle_message(self, evt: MessageEvent, was_encrypted: bool = False) - except Exception as e: self.log.debug(f"Error handling command {command} from {sender}: {e}") self._send_message_checkpoint(evt, MessageSendCheckpointStep.COMMAND, e) + await self._send_mss( + evt, + status=MessageStatus.FAIL, + reason=MessageStatusReason.GENERIC_ERROR, + error="", + message="Command execution failed", + ) else: await MessageSendCheckpoint( event_id=event_id, @@ -712,6 +736,7 @@ async def handle_message(self, evt: MessageEvent, was_encrypted: bool = False) - self.az.as_token, self.log, ) + await self._send_mss(evt, status=MessageStatus.SUCCESS) else: self.log.debug( f"Ignoring event {event_id} from {sender.mxid}:" @@ -720,6 +745,13 @@ async def handle_message(self, evt: MessageEvent, was_encrypted: bool = False) - self._send_message_checkpoint( evt, MessageSendCheckpointStep.COMMAND, "not a command and not a portal room" ) + await self._send_mss( + evt, + status=MessageStatus.FAIL, + reason=MessageStatusReason.UNSUPPORTED, + error="Unknown room", + message="Unknown room", + ) async def _is_direct_chat(self, room_id: RoomID) -> tuple[bool, bool]: try: From 4ebc1a3ec56783869f6a104d60b577426b4dacda Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 5 May 2023 18:44:45 +0300 Subject: [PATCH 093/218] Reuse aiosqlite connection pool for crypto db Otherwise stopping bridges gets stuck because the crypto pool isn't closed. asyncpg was already reusing the pool. --- mautrix/bridge/bridge.py | 5 ++++- mautrix/util/async_db/aiosqlite.py | 20 ++++++++++++++++++-- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/mautrix/bridge/bridge.py b/mautrix/bridge/bridge.py index c474109b..1dd9c0f9 100644 --- a/mautrix/bridge/bridge.py +++ b/mautrix/bridge/bridge.py @@ -262,8 +262,11 @@ async def start(self) -> None: async def system_exit(self) -> None: if hasattr(self, "db") and isinstance(self.db, Database): - self.log.trace("Stopping database due to SystemExit") + self.log.debug("Stopping database due to SystemExit") await self.db.stop() + self.log.debug("Database stopped") + elif getattr(self, "db", None): + self.log.trace("Database not started at SystemExit") async def stop(self) -> None: if self.manhole: diff --git a/mautrix/util/async_db/aiosqlite.py b/mautrix/util/async_db/aiosqlite.py index 9b7fa16a..22c2ffa6 100644 --- a/mautrix/util/async_db/aiosqlite.py +++ b/mautrix/util/async_db/aiosqlite.py @@ -5,7 +5,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import annotations -from typing import Any +from typing import Any, AsyncContextManager from contextlib import asynccontextmanager import asyncio import logging @@ -81,6 +81,7 @@ async def fetchval( class SQLiteDatabase(Database): scheme = Scheme.SQLITE + _parent: SQLiteDatabase | None _pool: asyncio.Queue[TxnConnection] _stopped: bool _conns: int @@ -103,6 +104,7 @@ def __init__( owner_name=owner_name, ignore_foreign_tables=ignore_foreign_tables, ) + self._parent = None self._path = url.path if self._path.startswith("/"): self._path = self._path[1:] @@ -134,7 +136,14 @@ def _add_missing_pragmas(init_commands: list[str]) -> list[str]: init_commands.append("PRAGMA busy_timeout = 5000") return init_commands + def override_pool(self, db: Database) -> None: + assert isinstance(db, SQLiteDatabase) + self._parent = db + async def start(self) -> None: + if self._parent: + await super().start() + return if self._conns: raise RuntimeError("database pool has already been started") elif self._stopped: @@ -155,14 +164,21 @@ async def start(self) -> None: await super().start() async def stop(self) -> None: + if self._parent: + return self._stopped = True while self._conns > 0: conn = await self._pool.get() self._conns -= 1 await conn.close() + def acquire(self) -> AsyncContextManager[LoggingConnection]: + if self._parent: + return self._parent.acquire() + return self._acquire() + @asynccontextmanager - async def acquire(self) -> LoggingConnection: + async def _acquire(self) -> LoggingConnection: if self._stopped: raise RuntimeError("database pool has been stopped") conn = await self._pool.get() From 488ce5cb0582b68f5b9558f3e30900aed79656b4 Mon Sep 17 00:00:00 2001 From: Nick Barrett Date: Tue, 16 May 2023 10:38:11 +0100 Subject: [PATCH 094/218] Use the current proxy as fallback when failing to fetch a new one Prevents accidentally resetting the proxy! --- mautrix/util/proxy.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mautrix/util/proxy.py b/mautrix/util/proxy.py index 5edd6531..4b5a4153 100644 --- a/mautrix/util/proxy.py +++ b/mautrix/util/proxy.py @@ -51,11 +51,10 @@ def get_proxy_url_from_api(self, reason: str | None = None) -> str | None: response = json.loads(f.read().decode()) except Exception: self.log.exception("Failed to retrieve proxy from API") + return self.current_proxy_url else: return response["proxy_url"] - return None - def update_proxy_url(self, reason: str | None = None) -> bool: old_proxy = self.current_proxy_url new_proxy = None From 27e3e0a7a587824b5672bda40fdfa3e6aee4d417 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 16 May 2023 12:44:21 +0300 Subject: [PATCH 095/218] Bump version to 0.19.14 --- CHANGELOG.md | 7 +++++++ mautrix/__init__.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 730f2a2d..4dda779a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## v0.19.14 (2023-05-16) + +* *(bridge)* Implemented appservice pinging using MSC2659. +* *(bridge)* Started reusing aiosqlite connection pool for crypto db. + * This fixes the crypto pool getting stuck if the bridge exits unexpectedly + (the default pool is closed automatically at any type of exit). + ## v0.19.13 (2023-04-24) * *(crypto)* Fixed bug with redacting megolm sessions when device is deleted. diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 440745cf..f3419a7b 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.19.13" +__version__ = "0.19.14" __author__ = "Tulir Asokan " __all__ = [ "api", From 21b97d7e3116b648a4f0c6cca34aaad9a2c32925 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 17 May 2023 20:52:20 +0300 Subject: [PATCH 096/218] Add trace logs for incorrect tokens in AS requests --- mautrix/appservice/as_handler.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mautrix/appservice/as_handler.py b/mautrix/appservice/as_handler.py index 2adcb2de..b49a8583 100644 --- a/mautrix/appservice/as_handler.py +++ b/mautrix/appservice/as_handler.py @@ -85,9 +85,11 @@ def _check_token(self, request: web.Request) -> bool: try: token = request.headers["Authorization"].removeprefix("Bearer ") except (KeyError, AttributeError): + self.log.trace("No access_token nor Authorization header in request") return False if token != self.hs_token: + self.log.trace(f"Incorrect hs_token in request ({token!r} != {self.hs_token!r})") return False return True From 204342da0a57311742e656dd8e5dbe01b3796647 Mon Sep 17 00:00:00 2001 From: Nick Mills-Barrett Date: Thu, 18 May 2023 17:40:01 +0100 Subject: [PATCH 097/218] Add `reset_after_seconds` to `proxy_with_retry` helper This is handy where wrapping long running tasks that may fail from time to time (hours) and we don't want them to raise an exception if only failing rarely (and presumably running fine most of the time). --- mautrix/util/proxy.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/mautrix/util/proxy.py b/mautrix/util/proxy.py index 4b5a4153..3041d164 100644 --- a/mautrix/util/proxy.py +++ b/mautrix/util/proxy.py @@ -4,6 +4,7 @@ import asyncio import json import logging +import time import urllib.request from aiohttp import ClientConnectionError @@ -93,8 +94,10 @@ async def proxy_with_retry( max_wait_seconds: int = 60, multiply_wait_seconds: int = 10, retryable_exceptions: tuple[Exception] = RETRYABLE_PROXY_EXCEPTIONS, + reset_after_seconds: int | None = None, ) -> T: errors = 0 + last_error = 0 while True: try: @@ -116,3 +119,11 @@ async def proxy_with_retry( f"{e.__class__.__name__} while trying to {name}" ): await on_proxy_change() + + # If sufficient time has passed since the previous error, reset the + # error count. Useful for long running tasks with rare failures. + if reset_after_seconds is not None: + now = time.time() + if now - last_error > reset_after_seconds: + errors = 0 + last_error = now From 50fa320617dd471fe74a54a98e3ec5d4fc066c26 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 22 May 2023 19:13:42 +0300 Subject: [PATCH 098/218] Handle ephemeral events inside rooms in sync --- mautrix/client/syncer.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/mautrix/client/syncer.py b/mautrix/client/syncer.py index 9f942665..62ae97a1 100644 --- a/mautrix/client/syncer.py +++ b/mautrix/client/syncer.py @@ -340,6 +340,13 @@ def handle_sync(self, data: JSON) -> list[asyncio.Task]: self._try_deserialize(Event, raw_event), source=SyncStream.JOINED_ROOM | SyncStream.TIMELINE, ) + + for raw_event in room_data.get("ephemeral", {}).get("events", []): + raw_event["room_id"] = room_id + tasks += self.dispatch_event( + self._try_deserialize(EphemeralEvent, raw_event), + source=SyncStream.JOINED_ROOM | SyncStream.EPHEMERAL, + ) for room_id, room_data in rooms.get("invite", {}).items(): events: list[dict[str, JSON]] = room_data.get("invite_state", {}).get("events", []) for raw_event in events: From 437e6e585fc26be682d080f736fd4802fbbf2213 Mon Sep 17 00:00:00 2001 From: Nick Mills-Barrett Date: Wed, 24 May 2023 10:39:41 +0100 Subject: [PATCH 099/218] Fix check for last error when using `reset_after_seconds` --- mautrix/util/proxy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/util/proxy.py b/mautrix/util/proxy.py index 3041d164..f36da73d 100644 --- a/mautrix/util/proxy.py +++ b/mautrix/util/proxy.py @@ -124,6 +124,6 @@ async def proxy_with_retry( # error count. Useful for long running tasks with rare failures. if reset_after_seconds is not None: now = time.time() - if now - last_error > reset_after_seconds: + if last_error and now - last_error > reset_after_seconds: errors = 0 last_error = now From 0da6f45c9921a1e6b7cccd97113f2a31c670feba Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 24 May 2023 12:45:25 +0300 Subject: [PATCH 100/218] Bump version to 0.19.15 --- CHANGELOG.md | 4 ++++ mautrix/__init__.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4dda779a..d43a914d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## v0.19.15 (2023-05-24) + +* *(client)* Fixed dispatching room ephemeral events (i.e. typing notifications) in syncer. + ## v0.19.14 (2023-05-16) * *(bridge)* Implemented appservice pinging using MSC2659. diff --git a/mautrix/__init__.py b/mautrix/__init__.py index f3419a7b..ce7df73a 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.19.14" +__version__ = "0.19.15" __author__ = "Tulir Asokan " __all__ = [ "api", From fbbe039b85974b993079fcd92d5a4aa354b53855 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 24 May 2023 19:58:45 +0300 Subject: [PATCH 101/218] Use path as-is for SQLite database Also add bridge config migration to remove extra slashes --- mautrix/bridge/config.py | 7 ++++++- mautrix/util/async_db/aiosqlite.py | 2 -- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/mautrix/bridge/config.py b/mautrix/bridge/config.py index f3e7720c..2674256b 100644 --- a/mautrix/bridge/config.py +++ b/mautrix/bridge/config.py @@ -115,7 +115,12 @@ def do_update(self, helper: ConfigUpdateHelper) -> None: copy("appservice.tls_cert") copy("appservice.tls_key") - copy("appservice.database") + if "appservice.database" in self and self["appservice.database"].startswith("sqlite:///"): + helper.base["appservice.database"] = self["appservice.database"].replace( + "sqlite:///", "sqlite:" + ) + else: + copy("appservice.database") copy("appservice.database_opts") copy("appservice.id") diff --git a/mautrix/util/async_db/aiosqlite.py b/mautrix/util/async_db/aiosqlite.py index 22c2ffa6..d9857b8c 100644 --- a/mautrix/util/async_db/aiosqlite.py +++ b/mautrix/util/async_db/aiosqlite.py @@ -106,8 +106,6 @@ def __init__( ) self._parent = None self._path = url.path - if self._path.startswith("/"): - self._path = self._path[1:] self._pool = asyncio.Queue(self._db_args.pop("min_size", 1)) self._db_args.pop("max_size", None) self._stopped = False From 7c60ee9175ad7d9edb64e8095c181e60485a6990 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 18 May 2023 12:37:26 +0300 Subject: [PATCH 102/218] Remove SQLAlchemy state store stuff. Fixes #147 --- docs/api/mautrix.client.state_store/index.rst | 1 - .../mautrix.client.state_store/sqlalchemy.rst | 5 - mautrix/appservice/state_store/__init__.py | 2 +- mautrix/appservice/state_store/sqlalchemy.py | 14 -- mautrix/bridge/crypto_state_store.py | 23 --- mautrix/bridge/e2ee.py | 7 - mautrix/bridge/state_store/__init__.py | 2 +- mautrix/bridge/state_store/sqlalchemy.py | 41 ---- mautrix/client/state_store/__init__.py | 1 - .../client/state_store/sqlalchemy/__init__.py | 3 - .../state_store/sqlalchemy/mx_room_state.py | 66 ------- .../state_store/sqlalchemy/mx_user_profile.py | 94 --------- .../state_store/sqlalchemy/sqlstatestore.py | 183 ------------------ .../client/state_store/tests/store_test.py | 15 +- setup.py | 2 +- 15 files changed, 4 insertions(+), 455 deletions(-) delete mode 100644 docs/api/mautrix.client.state_store/sqlalchemy.rst delete mode 100644 mautrix/appservice/state_store/sqlalchemy.py delete mode 100644 mautrix/bridge/state_store/sqlalchemy.py delete mode 100644 mautrix/client/state_store/sqlalchemy/__init__.py delete mode 100644 mautrix/client/state_store/sqlalchemy/mx_room_state.py delete mode 100644 mautrix/client/state_store/sqlalchemy/mx_user_profile.py delete mode 100644 mautrix/client/state_store/sqlalchemy/sqlstatestore.py diff --git a/docs/api/mautrix.client.state_store/index.rst b/docs/api/mautrix.client.state_store/index.rst index 91f2ba42..0a934308 100644 --- a/docs/api/mautrix.client.state_store/index.rst +++ b/docs/api/mautrix.client.state_store/index.rst @@ -13,5 +13,4 @@ Implementations In-memory Async database (asyncpg/aiosqlite) - Legacy database (SQLAlchemy) Flat file diff --git a/docs/api/mautrix.client.state_store/sqlalchemy.rst b/docs/api/mautrix.client.state_store/sqlalchemy.rst deleted file mode 100644 index 767e0595..00000000 --- a/docs/api/mautrix.client.state_store/sqlalchemy.rst +++ /dev/null @@ -1,5 +0,0 @@ -mautrix.client.state\_store.sqlalchemy -====================================== - -.. autoclass:: mautrix.client.state_store.sqlalchemy.SQLStateStore - :no-undoc-members: diff --git a/mautrix/appservice/state_store/__init__.py b/mautrix/appservice/state_store/__init__.py index bc23b5f5..771ac252 100644 --- a/mautrix/appservice/state_store/__init__.py +++ b/mautrix/appservice/state_store/__init__.py @@ -1,4 +1,4 @@ from .file import FileASStateStore from .memory import ASStateStore -__all__ = ["FileASStateStore", "ASStateStore", "sqlalchemy", "asyncpg"] +__all__ = ["FileASStateStore", "ASStateStore", "asyncpg"] diff --git a/mautrix/appservice/state_store/sqlalchemy.py b/mautrix/appservice/state_store/sqlalchemy.py deleted file mode 100644 index 30c0b1fc..00000000 --- a/mautrix/appservice/state_store/sqlalchemy.py +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright (c) 2022 Tulir Asokan -# -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. -from mautrix.client.state_store.sqlalchemy import SQLStateStore as SQLClientStateStore - -from .memory import ASStateStore - - -class SQLASStateStore(SQLClientStateStore, ASStateStore): - def __init__(self) -> None: - SQLClientStateStore.__init__(self) - ASStateStore.__init__(self) diff --git a/mautrix/bridge/crypto_state_store.py b/mautrix/bridge/crypto_state_store.py index 84169e2e..f08dec2c 100644 --- a/mautrix/bridge/crypto_state_store.py +++ b/mautrix/bridge/crypto_state_store.py @@ -28,29 +28,6 @@ async def is_encrypted(self, room_id: RoomID) -> bool: return portal.encrypted if portal else False -try: - from mautrix.client.state_store.sqlalchemy import RoomState, UserProfile - - class SQLCryptoStateStore(BaseCryptoStateStore): - @staticmethod - async def find_shared_rooms(user_id: UserID) -> list[RoomID]: - return [profile.room_id for profile in UserProfile.find_rooms_with_user(user_id)] - - @staticmethod - async def get_encryption_info(room_id: RoomID) -> RoomEncryptionStateEventContent | None: - state = RoomState.get(room_id) - if not state: - return None - return state.encryption - -except ImportError: - if __optional_imports__: - raise - UserProfile = None - RoomState = None - SQLCryptoStateStore = None - - class PgCryptoStateStore(BaseCryptoStateStore): db: Database diff --git a/mautrix/bridge/e2ee.py b/mautrix/bridge/e2ee.py index 805efd4f..3961be58 100644 --- a/mautrix/bridge/e2ee.py +++ b/mautrix/bridge/e2ee.py @@ -41,13 +41,6 @@ from .. import bridge as br from .crypto_state_store import PgCryptoStateStore -try: - from mautrix.client.state_store.sqlalchemy import UserProfile -except ImportError: - if __optional_imports__: - raise - UserProfile = None - class EncryptionManager: loop: asyncio.AbstractEventLoop diff --git a/mautrix/bridge/state_store/__init__.py b/mautrix/bridge/state_store/__init__.py index 0e8137a5..b990ac86 100644 --- a/mautrix/bridge/state_store/__init__.py +++ b/mautrix/bridge/state_store/__init__.py @@ -1 +1 @@ -__all__ = ["asyncpg", "sqlalchemy"] +__all__ = ["asyncpg"] diff --git a/mautrix/bridge/state_store/sqlalchemy.py b/mautrix/bridge/state_store/sqlalchemy.py deleted file mode 100644 index 2fa7353e..00000000 --- a/mautrix/bridge/state_store/sqlalchemy.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright (c) 2022 Tulir Asokan -# -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import annotations - -from typing import Awaitable, Callable, Union - -from mautrix.appservice.state_store.sqlalchemy import SQLASStateStore -from mautrix.types import UserID - -from ..puppet import BasePuppet - -GetPuppetFunc = Union[ - Callable[[UserID], Awaitable[BasePuppet]], Callable[[UserID, bool], Awaitable[BasePuppet]] -] - - -class SQLBridgeStateStore(SQLASStateStore): - def __init__(self, get_puppet: GetPuppetFunc, get_double_puppet: GetPuppetFunc) -> None: - super().__init__() - self.get_puppet = get_puppet - self.get_double_puppet = get_double_puppet - - async def is_registered(self, user_id: UserID) -> bool: - puppet = await self.get_puppet(user_id) - if puppet: - return puppet.is_registered - custom_puppet = await self.get_double_puppet(user_id) - if custom_puppet: - return True - return await super().is_registered(user_id) - - async def registered(self, user_id: UserID) -> None: - puppet = await self.get_puppet(user_id, True) - if puppet: - puppet.is_registered = True - await puppet.save() - else: - await super().registered(user_id) diff --git a/mautrix/client/state_store/__init__.py b/mautrix/client/state_store/__init__.py index 5f76c7dc..98150ca4 100644 --- a/mautrix/client/state_store/__init__.py +++ b/mautrix/client/state_store/__init__.py @@ -10,5 +10,4 @@ "MemorySyncStore", "SyncStore", "asyncpg", - "sqlalchemy", ] diff --git a/mautrix/client/state_store/sqlalchemy/__init__.py b/mautrix/client/state_store/sqlalchemy/__init__.py deleted file mode 100644 index 6ce78f46..00000000 --- a/mautrix/client/state_store/sqlalchemy/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .mx_room_state import RoomState, SerializableType -from .mx_user_profile import UserProfile -from .sqlstatestore import SQLStateStore diff --git a/mautrix/client/state_store/sqlalchemy/mx_room_state.py b/mautrix/client/state_store/sqlalchemy/mx_room_state.py deleted file mode 100644 index 9f97056f..00000000 --- a/mautrix/client/state_store/sqlalchemy/mx_room_state.py +++ /dev/null @@ -1,66 +0,0 @@ -# Copyright (c) 2022 Tulir Asokan -# -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import annotations - -from typing import Type -import json - -from sqlalchemy import Boolean, Column, Text, types - -from mautrix.types import ( - PowerLevelStateEventContent as PowerLevels, - RoomEncryptionStateEventContent as EncryptionInfo, - RoomID, - Serializable, -) -from mautrix.util.db import Base - - -class SerializableType(types.TypeDecorator): - impl = types.Text - - def __init__(self, python_type: Type[Serializable], *args, **kwargs) -> None: - super().__init__(*args, **kwargs) - self._python_type = python_type - - @property - def python_type(self) -> Type[Serializable]: - return self._python_type - - def process_bind_param(self, value: Serializable, dialect) -> str | None: - if value is not None: - return json.dumps(value.serialize() if isinstance(value, Serializable) else value) - return None - - def process_result_value(self, value: str, dialect) -> Serializable | None: - if value is not None: - return self.python_type.deserialize(json.loads(value)) - return None - - def process_literal_param(self, value, dialect): - return value - - -class RoomState(Base): - __tablename__ = "mx_room_state" - - room_id: RoomID = Column(Text, primary_key=True) - is_encrypted: bool = Column(Boolean, nullable=True) - has_full_member_list: bool = Column(Boolean, nullable=True) - encryption: EncryptionInfo = Column(SerializableType(EncryptionInfo), nullable=True) - power_levels: PowerLevels = Column(SerializableType(PowerLevels), nullable=True) - - @property - def has_power_levels(self) -> bool: - return bool(self.power_levels) - - @property - def has_encryption_info(self) -> bool: - return self.is_encrypted is not None - - @classmethod - def get(cls, room_id: RoomID) -> RoomState | None: - return cls._select_one_or_none(cls.c.room_id == room_id) diff --git a/mautrix/client/state_store/sqlalchemy/mx_user_profile.py b/mautrix/client/state_store/sqlalchemy/mx_user_profile.py deleted file mode 100644 index 0bb6b766..00000000 --- a/mautrix/client/state_store/sqlalchemy/mx_user_profile.py +++ /dev/null @@ -1,94 +0,0 @@ -# Copyright (c) 2022 Tulir Asokan -# -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import annotations - -from typing import Iterable - -from sqlalchemy import Column, Enum, Text - -from mautrix.types import ContentURI, Member, Membership, RoomID, UserID -from mautrix.util.db import Base - -from .mx_room_state import RoomState - - -class UserProfile(Base): - __tablename__ = "mx_user_profile" - - room_id: RoomID = Column(Text, primary_key=True) - user_id: UserID = Column(Text, primary_key=True) - membership: Membership = Column(Enum(Membership), nullable=False, default=Membership.LEAVE) - displayname: str = Column(Text, nullable=True) - avatar_url: ContentURI = Column(Text, nullable=True) - - def member(self) -> Member: - return Member( - membership=self.membership, displayname=self.displayname, avatar_url=self.avatar_url - ) - - @classmethod - def get(cls, room_id: RoomID, user_id: UserID) -> UserProfile | None: - return cls._select_one_or_none((cls.c.room_id == room_id) & (cls.c.user_id == user_id)) - - @classmethod - def all_in_room( - cls, - room_id: RoomID, - memberships: tuple[Membership, ...], - prefix: str = None, - suffix: str = None, - bot: str = None, - ) -> Iterable[UserProfile]: - clause = cls.c.membership == memberships[0] - for membership in memberships[1:]: - clause |= cls.c.membership == membership - clause &= cls.c.room_id == room_id - if bot: - clause &= cls.c.user_id != bot - if prefix: - clause &= ~cls.c.user_id.startswith(prefix, autoescape=True) - if suffix: - clause &= ~cls.c.user_id.startswith(suffix, autoescape=True) - return cls._select_all(clause) - - @classmethod - def find_rooms_with_user(cls, user_id: UserID) -> Iterable[UserProfile]: - return cls._select_all( - (cls.c.user_id == user_id) - & (cls.c.room_id == RoomState.c.room_id) - & (RoomState.c.is_encrypted == True) - ) - - @classmethod - def delete_all(cls, room_id: RoomID) -> None: - with cls.db.begin() as conn: - conn.execute(cls.t.delete().where(cls.c.room_id == room_id)) - - @classmethod - def bulk_replace( - cls, - room_id: RoomID, - members: dict[UserID, Member], - only_membership: Membership | None = None, - ) -> None: - with cls.db.begin() as conn: - delete_condition = cls.c.room_id == room_id - if only_membership is not None: - delete_condition &= cls.c.membership == only_membership - cls.db.execute(cls.t.delete().where(delete_condition)) - conn.execute( - cls.t.insert(), - [ - dict( - room_id=room_id, - user_id=user_id, - membership=member.membership, - displayname=member.displayname, - avatar_url=member.avatar_url, - ) - for user_id, member in members.items() - ], - ) diff --git a/mautrix/client/state_store/sqlalchemy/sqlstatestore.py b/mautrix/client/state_store/sqlalchemy/sqlstatestore.py deleted file mode 100644 index 1145dec7..00000000 --- a/mautrix/client/state_store/sqlalchemy/sqlstatestore.py +++ /dev/null @@ -1,183 +0,0 @@ -# Copyright (c) 2022 Tulir Asokan -# -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import annotations - -from typing import Any - -from mautrix.types import ( - Member, - Membership, - PowerLevelStateEventContent, - RoomEncryptionStateEventContent, - RoomID, - UserID, -) - -from ..abstract import StateStore -from .mx_room_state import RoomState -from .mx_user_profile import UserProfile - - -class SQLStateStore(StateStore): - _profile_cache: dict[RoomID, dict[UserID, UserProfile]] - _room_state_cache: dict[RoomID, RoomState] - - def __init__(self) -> None: - super().__init__() - self._profile_cache = {} - self._room_state_cache = {} - - def _get_user_profile( - self, room_id: RoomID, user_id: UserID, create: bool = False - ) -> UserProfile: - if not room_id: - raise ValueError("room_id is empty") - elif not user_id: - raise ValueError("user_id is empty") - try: - return self._profile_cache[room_id][user_id] - except KeyError: - pass - if room_id not in self._profile_cache: - self._profile_cache[room_id] = {} - - profile = UserProfile.get(room_id, user_id) - if profile: - self._profile_cache[room_id][user_id] = profile - elif create: - profile = UserProfile(room_id=room_id, user_id=user_id, membership=Membership.LEAVE) - profile.insert() - self._profile_cache[room_id][user_id] = profile - return profile - - async def get_member(self, room_id: RoomID, user_id: UserID) -> Member | None: - profile = self._get_user_profile(room_id, user_id) - if not profile: - return None - return profile.member() - - async def set_member(self, room_id: RoomID, user_id: UserID, member: Member) -> None: - if not member: - raise ValueError("member info is empty") - profile = self._get_user_profile(room_id, user_id, create=True) - profile.edit( - membership=member.membership, - displayname=member.displayname or profile.displayname, - avatar_url=member.avatar_url or profile.avatar_url, - ) - - async def set_membership( - self, room_id: RoomID, user_id: UserID, membership: Membership - ) -> None: - await self.set_member(room_id, user_id, Member(membership=membership)) - - async def get_member_profiles( - self, - room_id: RoomID, - memberships: tuple[Membership, ...] = (Membership.JOIN, Membership.INVITE), - ) -> dict[UserID, Member]: - self._profile_cache[room_id] = {} - for profile in UserProfile.all_in_room(room_id, memberships): - self._profile_cache[room_id][profile.user_id] = profile - return { - profile.user_id: profile.member() for profile in self._profile_cache[room_id].values() - } - - async def get_members_filtered( - self, - room_id: RoomID, - not_prefix: str, - not_suffix: str, - not_id: str, - memberships: tuple[Membership, ...] = (Membership.JOIN, Membership.INVITE), - ) -> list[UserID]: - return [ - profile.user_id - for profile in UserProfile.all_in_room( - room_id, memberships, not_suffix, not_prefix, not_id - ) - ] - - async def set_members( - self, - room_id: RoomID, - members: dict[UserID, Member], - only_membership: Membership | None = None, - ) -> None: - UserProfile.bulk_replace(room_id, members, only_membership=only_membership) - self._get_room_state(room_id, create=True).edit(has_full_member_list=True) - try: - del self._profile_cache[room_id] - except KeyError: - pass - - async def has_full_member_list(self, room_id: RoomID) -> bool: - room = self._get_room_state(room_id) - if not room: - return False - return room.has_full_member_list - - async def find_shared_rooms(self, user_id: UserID) -> list[RoomID]: - return [profile.room_id for profile in UserProfile.find_rooms_with_user(user_id)] - - def _get_room_state(self, room_id: RoomID, create: bool = False) -> RoomState: - if not room_id: - raise ValueError("room_id is empty") - try: - return self._room_state_cache[room_id] - except KeyError: - pass - - room = RoomState.get(room_id) - if room: - self._room_state_cache[room_id] = room - elif create: - room = RoomState(room_id=room_id) - room.insert() - self._room_state_cache[room_id] = room - return room - - async def has_power_levels_cached(self, room_id: RoomID) -> bool: - room = self._get_room_state(room_id) - if not room: - return False - return room.has_power_levels - - async def get_power_levels(self, room_id: RoomID) -> PowerLevelStateEventContent | None: - room = self._get_room_state(room_id) - if not room: - return None - return room.power_levels - - async def set_power_levels( - self, room_id: RoomID, content: PowerLevelStateEventContent - ) -> None: - if not content: - raise ValueError("content is empty") - self._get_room_state(room_id, create=True).edit(power_levels=content) - - async def is_encrypted(self, room_id: RoomID) -> bool | None: - room = self._get_room_state(room_id) - if not room: - return None - return room.is_encrypted - - async def has_encryption_info_cached(self, room_id: RoomID) -> bool: - room = self._get_room_state(room_id) - return room and room.has_encryption_info - - async def get_encryption_info(self, room_id: RoomID) -> RoomEncryptionStateEventContent | None: - room = self._get_room_state(room_id) - if not room: - return None - return room.encryption - - async def set_encryption_info( - self, room_id: RoomID, content: RoomEncryptionStateEventContent | dict[str, Any] - ) -> None: - if not content: - raise ValueError("content is empty") - self._get_room_state(room_id, create=True).edit(encryption=content, is_encrypted=True) diff --git a/mautrix/client/state_store/tests/store_test.py b/mautrix/client/state_store/tests/store_test.py index dbbd376a..f81ff76d 100644 --- a/mautrix/client/state_store/tests/store_test.py +++ b/mautrix/client/state_store/tests/store_test.py @@ -16,7 +16,6 @@ import asyncpg import pytest -import sqlalchemy as sql from mautrix.types import EncryptionAlgorithm, Member, Membership, RoomID, StateEvent, UserID from mautrix.util.async_db import Database @@ -24,7 +23,6 @@ from .. import MemoryStateStore, StateStore from ..asyncpg import PgStateStore -from ..sqlalchemy import RoomState, SQLStateStore, UserProfile @asynccontextmanager @@ -62,23 +60,12 @@ async def async_sqlite_store() -> AsyncIterator[PgStateStore]: await db.stop() -@asynccontextmanager -async def alchemy_store() -> AsyncIterator[SQLStateStore]: - db = sql.create_engine("sqlite:///:memory:") - Base.metadata.bind = db - for table in (RoomState, UserProfile): - table.bind(db) - Base.metadata.create_all() - yield SQLStateStore() - db.dispose() - - @asynccontextmanager async def memory_store() -> AsyncIterator[MemoryStateStore]: yield MemoryStateStore() -@pytest.fixture(params=[async_postgres_store, async_sqlite_store, alchemy_store, memory_store]) +@pytest.fixture(params=[async_postgres_store, async_sqlite_store, memory_store]) async def store(request) -> AsyncIterator[StateStore]: param: Callable[[], AsyncContextManager[StateStore]] = request.param async with param() as state_store: diff --git a/setup.py b/setup.py index cabdf15a..a345d37e 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ from mautrix import __version__ encryption_dependencies = ["python-olm", "unpaddedbase64", "pycryptodome"] -test_dependencies = ["aiosqlite", "sqlalchemy<2", "asyncpg", *encryption_dependencies] +test_dependencies = ["aiosqlite", "asyncpg", *encryption_dependencies] setuptools.setup( name="mautrix", From 1af59f960bd4aba4afef09ea720078bf3e5e86bd Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 24 May 2023 20:08:30 +0300 Subject: [PATCH 103/218] Update changelog --- CHANGELOG.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d43a914d..9531d6f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,14 @@ +## v0.20.0 (unreleased) + +* **Breaking change *(.state_store)*** Removed legacy SQLAlchemy state store + implementations. +* **Mildly breaking change *(util.async_db)*** Changed `SQLiteDatabase` to not + remove prefix slashes from database paths. + * Library users should use `sqlite:path.db` instead of `sqlite:///path.db` + for relative paths, and `sqlite:/path.db` instead of `sqlite:////path.db` + for absolute paths. + * Bridge configs do this migration automatically. + ## v0.19.15 (2023-05-24) * *(client)* Fixed dispatching room ephemeral events (i.e. typing notifications) in syncer. From 6faf041f0a285423ddb1ace48a2cab6dceed8393 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 24 May 2023 20:31:00 +0300 Subject: [PATCH 104/218] Remove extra import --- mautrix/client/state_store/tests/store_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/mautrix/client/state_store/tests/store_test.py b/mautrix/client/state_store/tests/store_test.py index f81ff76d..750b87a9 100644 --- a/mautrix/client/state_store/tests/store_test.py +++ b/mautrix/client/state_store/tests/store_test.py @@ -19,7 +19,6 @@ from mautrix.types import EncryptionAlgorithm, Member, Membership, RoomID, StateEvent, UserID from mautrix.util.async_db import Database -from mautrix.util.db import Base from .. import MemoryStateStore, StateStore from ..asyncpg import PgStateStore From 1d49625f87c86f82da394ac40c8249eddaaac844 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 24 May 2023 20:31:51 +0300 Subject: [PATCH 105/218] Fix SQLite paths in tests --- mautrix/client/state_store/tests/store_test.py | 2 +- mautrix/crypto/store/tests/store_test.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mautrix/client/state_store/tests/store_test.py b/mautrix/client/state_store/tests/store_test.py index 750b87a9..46eee750 100644 --- a/mautrix/client/state_store/tests/store_test.py +++ b/mautrix/client/state_store/tests/store_test.py @@ -51,7 +51,7 @@ async def async_postgres_store() -> AsyncIterator[PgStateStore]: @asynccontextmanager async def async_sqlite_store() -> AsyncIterator[PgStateStore]: db = Database.create( - "sqlite:///:memory:", upgrade_table=PgStateStore.upgrade_table, db_args={"min_size": 1} + "sqlite::memory:", upgrade_table=PgStateStore.upgrade_table, db_args={"min_size": 1} ) store = PgStateStore(db) await db.start() diff --git a/mautrix/crypto/store/tests/store_test.py b/mautrix/crypto/store/tests/store_test.py index 8d3fc851..949949b8 100644 --- a/mautrix/crypto/store/tests/store_test.py +++ b/mautrix/crypto/store/tests/store_test.py @@ -50,7 +50,7 @@ async def async_postgres_store() -> AsyncIterator[PgCryptoStore]: @asynccontextmanager async def async_sqlite_store() -> AsyncIterator[PgCryptoStore]: db = Database.create( - "sqlite:///:memory:", upgrade_table=PgCryptoStore.upgrade_table, db_args={"min_size": 1} + "sqlite::memory:", upgrade_table=PgCryptoStore.upgrade_table, db_args={"min_size": 1} ) store = PgCryptoStore("", "test", db) await db.start() From 40328415b9d2a18c74d12351f51577d5b1ea90d5 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 26 May 2023 21:04:30 +0300 Subject: [PATCH 106/218] Drop Python 3.8 compatibility --- .github/workflows/python-package.yml | 2 +- CHANGELOG.md | 5 +++++ setup.py | 3 +-- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 96b55773..8c6c7606 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -9,7 +9,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.8", "3.9", "3.10", "3.11"] + python-version: ["3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v3 diff --git a/CHANGELOG.md b/CHANGELOG.md index 9531d6f7..da1a8d5a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ ## v0.20.0 (unreleased) +* Dropped Python 3.8 support. * **Breaking change *(.state_store)*** Removed legacy SQLAlchemy state store implementations. * **Mildly breaking change *(util.async_db)*** Changed `SQLiteDatabase` to not @@ -9,6 +10,10 @@ for absolute paths. * Bridge configs do this migration automatically. +## v0.19.16 (2023-05-26) + +* *(appservice)* Fixed Python 3.8 compatibility. + ## v0.19.15 (2023-05-24) * *(client)* Fixed dispatching room ephemeral events (i.e. typing notifications) in syncer. diff --git a/setup.py b/setup.py index a345d37e..04a2a8c3 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,7 @@ "encryption": encryption_dependencies, }, tests_require=test_dependencies, - python_requires="~=3.8", + python_requires="~=3.9", classifiers=[ "Development Status :: 4 - Beta", @@ -42,7 +42,6 @@ "Framework :: AsyncIO", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", From c1e3a66465b814c4c6319b2ad94b8b9983817357 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 26 May 2023 21:08:11 +0300 Subject: [PATCH 107/218] Only catch KeyError in HS token checker --- mautrix/appservice/as_handler.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mautrix/appservice/as_handler.py b/mautrix/appservice/as_handler.py index b49a8583..c0a41f73 100644 --- a/mautrix/appservice/as_handler.py +++ b/mautrix/appservice/as_handler.py @@ -84,12 +84,12 @@ def _check_token(self, request: web.Request) -> bool: except KeyError: try: token = request.headers["Authorization"].removeprefix("Bearer ") - except (KeyError, AttributeError): - self.log.trace("No access_token nor Authorization header in request") + except KeyError: + self.log.debug("No access_token nor Authorization header in request") return False if token != self.hs_token: - self.log.trace(f"Incorrect hs_token in request ({token!r} != {self.hs_token!r})") + self.log.debug(f"Incorrect hs_token in request") return False return True From cedf2aa89d341f8be7fe3814405c9af80950ac30 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 31 May 2023 14:40:16 +0300 Subject: [PATCH 108/218] Retry sending unconfigured bridge state --- mautrix/bridge/bridge.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mautrix/bridge/bridge.py b/mautrix/bridge/bridge.py index 1dd9c0f9..37e1646d 100644 --- a/mautrix/bridge/bridge.py +++ b/mautrix/bridge/bridge.py @@ -8,6 +8,7 @@ from typing import Any from abc import ABC, abstractmethod from enum import Enum +import asyncio import sys from aiohttp import web @@ -258,7 +259,8 @@ async def start(self) -> None: status_endpoint = self.config["homeserver.status_endpoint"] if status_endpoint and await self.count_logged_in_users() == 0: state = BridgeState(state_event=BridgeStateEvent.UNCONFIGURED).fill() - await state.send(status_endpoint, self.az.as_token, self.log) + while not await state.send(status_endpoint, self.az.as_token, self.log): + await asyncio.sleep(5) async def system_exit(self) -> None: if hasattr(self, "db") and isinstance(self.db, Database): From dd81640904af047692a5f307eef3763f23f48a17 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 10 Jun 2023 16:25:41 +0300 Subject: [PATCH 109/218] Stabilize async uploads (#149) --- CHANGELOG.md | 3 + .../client/api/modules/media_repository.py | 68 +++++++++---------- mautrix/types/media.py | 2 +- 3 files changed, 35 insertions(+), 38 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index da1a8d5a..7b92b469 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,9 @@ for relative paths, and `sqlite:/path.db` instead of `sqlite:////path.db` for absolute paths. * Bridge configs do this migration automatically. +* *(client)* Stabilized support for asynchronous uploads. + * `unstable_create_msc` was renamed to `create_mxc`, and the `max_stall_ms` + parameters for downloading were renamed to `timeout_ms`.. ## v0.19.16 (2023-05-26) diff --git a/mautrix/client/api/modules/media_repository.py b/mautrix/client/api/modules/media_repository.py index edf5e701..0d51d9f6 100644 --- a/mautrix/client/api/modules/media_repository.py +++ b/mautrix/client/api/modules/media_repository.py @@ -46,22 +46,19 @@ class MediaRepositoryMethods(BaseClientAPI): downloading content from the media repository and for getting URL previews without leaking client IPs. - See also: `API reference `__ - - There are also methods for supporting `MSC2246 - `__ which allows asynchronous - uploads of media. + See also: `API reference `__ """ - async def unstable_create_mxc(self) -> MediaCreateResponse: + async def create_mxc(self) -> MediaCreateResponse: """ - Create a media ID for uploading media to the homeserver. Requires the homeserver to have - `MSC2246 `__ support. + Create a media ID for uploading media to the homeserver. + + See also: `API reference `__ Returns: - MediaCreateResponse Containing the MXC URI that can be used to upload a file to later, as well as an optional upload URL + MediaCreateResponse Containing the MXC URI that can be used to upload a file to later """ - resp = await self.api.request(Method.POST, MediaPath.unstable["fi.mau.msc2246"].create) + resp = await self.api.request(Method.POST, MediaPath.v1.create) return MediaCreateResponse.deserialize(resp) @contextmanager @@ -87,21 +84,18 @@ async def upload_media( """ Upload a file to the content repository. - See also: `API reference `__ + See also: `API reference `__ Args: data: The data to upload. mime_type: The MIME type to send with the upload request. filename: The filename to send with the upload request. size: The file size to send with the upload request. - mxc: An existing MXC URI which doesn't have content yet to upload into. Requires the - homeserver to have MSC2246_ support. - async_upload: Should the media be uploaded in the background (using MSC2246_)? - If ``True``, this will create a MXC URI, start uploading in the background and then - immediately return the created URI. This is mutually exclusive with manually - passing the ``mxc`` parameter. - - .. _MSC2246: https://github.com/matrix-org/matrix-spec-proposals/pull/2246 + mxc: An existing MXC URI which doesn't have content yet to upload into. + async_upload: Should the media be uploaded in the background? + If ``True``, this will create a MXC URI using :meth:`create_mxc`, start uploading + in the background, and then immediately return the created URI. This is mutually + exclusive with manually passing the ``mxc`` parameter. Returns: The MXC URI to the uploaded file. @@ -128,19 +122,21 @@ async def upload_media( if async_upload: if mxc: raise ValueError("async_upload and mxc can't be provided simultaneously") - create_response = await self.unstable_create_mxc() + create_response = await self.create_mxc() mxc = create_response.content_uri - upload_url = create_response.upload_url + upload_url = create_response.unstable_upload_url path = MediaPath.v3.upload method = Method.POST if mxc: server_name, media_id = self.api.parse_mxc_uri(mxc) if upload_url is None: - path = MediaPath.unstable["fi.mau.msc2246"].upload[server_name][media_id] + path = MediaPath.v3.upload[server_name][media_id] method = Method.PUT else: - path = MediaPath.unstable["fi.mau.msc2246"].upload[server_name][media_id].complete + path = ( + MediaPath.unstable["com.beeper.msc3870"].upload[server_name][media_id].complete + ) if upload_url is not None: task = self._upload_to_url(upload_url, path, headers, data, post_upload_query=query) @@ -168,25 +164,24 @@ async def _try_upload(): except KeyError: raise MatrixResponseError("`content_uri` not in response.") - async def download_media(self, url: ContentURI, max_stall_ms: int | None = None) -> bytes: + async def download_media(self, url: ContentURI, timeout_ms: int | None = None) -> bytes: """ Download a file from the content repository. - See also: `API reference `__ + See also: `API reference `__ Args: url: The MXC URI to download. - max_stall_ms: The maximum number of milliseconds that the client is willing to wait to - start receiving data. Used for MSC2246 Asynchronous Uploads. + timeout_ms: The maximum number of milliseconds that the client is willing to wait to + start receiving data. Used for asynchronous uploads. Returns: The raw downloaded data. """ url = self.api.get_download_url(url) query_params: dict[str, Any] = {"allow_redirect": "true"} - if max_stall_ms is not None: - query_params["max_stall_ms"] = max_stall_ms - query_params["fi.mau.msc2246.max_stall_ms"] = max_stall_ms + if timeout_ms is not None: + query_params["timeout_ms"] = timeout_ms async with self.api.session.get(url, params=query_params) as response: return await response.read() @@ -197,12 +192,12 @@ async def download_thumbnail( height: int | None = None, resize_method: Literal["crop", "scale"] = None, allow_remote: bool = True, - max_stall_ms: int | None = None, + timeout_ms: int | None = None, ): """ Download a thumbnail for a file in the content repository. - See also: `API reference `__ + See also: `API reference `__ Args: url: The MXC URI to download. @@ -214,8 +209,8 @@ async def download_thumbnail( allow_remote: Indicates to the server that it should not attempt to fetch the media if it is deemed remote. This is to prevent routing loops where the server contacts itself. - max_stall_ms: The maximum number of milliseconds that the client is willing to wait to - start receiving data. Used for MSC2246 Asynchronous Uploads. + timeout_ms: The maximum number of milliseconds that the client is willing to wait to + start receiving data. Used for asynchronous Uploads. Returns: The raw downloaded data. @@ -230,9 +225,8 @@ async def download_thumbnail( query_params["method"] = resize_method if allow_remote is not None: query_params["allow_remote"] = allow_remote - if max_stall_ms is not None: - query_params["max_stall_ms"] = max_stall_ms - query_params["fi.mau.msc2246.max_stall_ms"] = max_stall_ms + if timeout_ms is not None: + query_params["timeout_ms"] = timeout_ms async with self.api.session.get(url, params=query_params) as response: return await response.read() diff --git a/mautrix/types/media.py b/mautrix/types/media.py index 0b277382..1d0ee66a 100644 --- a/mautrix/types/media.py +++ b/mautrix/types/media.py @@ -71,4 +71,4 @@ class MediaCreateResponse(SerializableAttrs): content_uri: ContentURI unused_expired_at: Optional[int] = None - upload_url: Optional[str] = None + unstable_upload_url: Optional[str] = field(default=None, json="com.beeper.msc3870.upload_url") From 7cdebe553b2c9700e1a6c2b34613001d628628c4 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 4 Jun 2023 18:01:12 +0300 Subject: [PATCH 110/218] Log warning if db file doesn't seem writable --- mautrix/util/async_db/aiosqlite.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/mautrix/util/async_db/aiosqlite.py b/mautrix/util/async_db/aiosqlite.py index d9857b8c..99d8fa44 100644 --- a/mautrix/util/async_db/aiosqlite.py +++ b/mautrix/util/async_db/aiosqlite.py @@ -9,6 +9,7 @@ from contextlib import asynccontextmanager import asyncio import logging +import os import re import sqlite3 @@ -148,6 +149,11 @@ async def start(self) -> None: raise RuntimeError("database pool can't be restarted") self.log.debug(f"Connecting to {self.url}") self.log.debug(f"Database connection init commands: {self._init_commands}") + if os.path.exists(self._path): + if not os.access(self._path, os.W_OK): + self.log.warning("Database file doesn't seem writable") + elif not os.access(os.path.dirname(os.path.abspath(self._path)), os.W_OK): + self.log.warning("Database file doesn't exist and directory doesn't seem writable") for _ in range(self._pool.maxsize): conn = await TxnConnection(self._path, **self._db_args) if self._init_commands: From afa904edce4911add55a0d69f54f43d99ee36d83 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 31 May 2023 14:39:57 +0300 Subject: [PATCH 111/218] Fix ensure registered methods --- mautrix/appservice/api/intent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/appservice/api/intent.py b/mautrix/appservice/api/intent.py index e52d8016..ab9e1e82 100644 --- a/mautrix/appservice/api/intent.py +++ b/mautrix/appservice/api/intent.py @@ -72,7 +72,7 @@ def quote(*args, **kwargs): ClientAPI.set_displayname, ClientAPI.set_avatar_url, ClientAPI.beeper_update_profile, - ClientAPI.unstable_create_mxc, + ClientAPI.create_mxc, ClientAPI.upload_media, ClientAPI.send_receipt, ClientAPI.set_fully_read_marker, From 6c58b51461c3134fec04b823f4d0a89469cf18ed Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 13 Jun 2023 13:51:28 +0300 Subject: [PATCH 112/218] Add option to not rotate keys when devices change --- mautrix/bridge/config.py | 1 + mautrix/bridge/e2ee.py | 3 +++ mautrix/crypto/base.py | 1 + mautrix/crypto/device_lists.py | 2 ++ mautrix/crypto/machine.py | 1 + 5 files changed, 8 insertions(+) diff --git a/mautrix/bridge/config.py b/mautrix/bridge/config.py index 2674256b..0dc536e6 100644 --- a/mautrix/bridge/config.py +++ b/mautrix/bridge/config.py @@ -166,6 +166,7 @@ def do_update(self, helper: ConfigUpdateHelper) -> None: copy("bridge.encryption.rotation.enable_custom") copy("bridge.encryption.rotation.milliseconds") copy("bridge.encryption.rotation.messages") + copy("bridge.encryption.rotation.disable_device_change_key_rotation") copy("bridge.relay.enabled") copy_dict("bridge.relay.message_formats", override_existing_map=False) diff --git a/mautrix/bridge/e2ee.py b/mautrix/bridge/e2ee.py index 3961be58..b1a75b2c 100644 --- a/mautrix/bridge/e2ee.py +++ b/mautrix/bridge/e2ee.py @@ -123,6 +123,9 @@ def __init__( self.crypto.delete_fully_used_keys_on_decrypt = del_cfg["delete_fully_used_on_decrypt"] self.crypto.delete_keys_on_device_delete = del_cfg["delete_on_device_delete"] self.periodically_delete_expired_keys = del_cfg["periodically_delete_expired"] + self.crypto.disable_device_change_key_rotation = bridge.config[ + "bridge.encryption.rotation.disable_device_change_key_rotation" + ] async def _exit_on_sync_fail(self, data) -> None: if data["error"]: diff --git a/mautrix/crypto/base.py b/mautrix/crypto/base.py index 5af3ec0a..e1386834 100644 --- a/mautrix/crypto/base.py +++ b/mautrix/crypto/base.py @@ -56,6 +56,7 @@ class BaseOlmMachine: ratchet_keys_on_decrypt: bool delete_fully_used_keys_on_decrypt: bool delete_keys_on_device_delete: bool + disable_device_change_key_rotation: bool # Futures that wait for responses to a key request _key_request_waiters: dict[SessionID, asyncio.Future] diff --git a/mautrix/crypto/device_lists.py b/mautrix/crypto/device_lists.py index d49c7e19..1b5e0dbd 100644 --- a/mautrix/crypto/device_lists.py +++ b/mautrix/crypto/device_lists.py @@ -244,6 +244,8 @@ async def get_or_fetch_device_by_key( return None async def on_devices_changed(self, user_id: UserID) -> None: + if self.disable_device_change_key_rotation: + return shared_rooms = await self.state_store.find_shared_rooms(user_id) self.log.debug( f"Devices of {user_id} changed, invalidating group session in {shared_rooms}" diff --git a/mautrix/crypto/machine.py b/mautrix/crypto/machine.py index 9cf53312..c68335af 100644 --- a/mautrix/crypto/machine.py +++ b/mautrix/crypto/machine.py @@ -83,6 +83,7 @@ def __init__( self.ratchet_keys_on_decrypt = False self.delete_fully_used_keys_on_decrypt = False self.delete_keys_on_device_delete = False + self.disable_device_change_key_rotation = False self._fetch_keys_lock = asyncio.Lock() self._megolm_decrypt_lock = asyncio.Lock() From 69328fa4d7a524a24699c39742235c0d85bf52b0 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 13 Jun 2023 14:01:48 +0300 Subject: [PATCH 113/218] Update changelog --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b92b469..74214d56 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,9 +9,12 @@ for relative paths, and `sqlite:/path.db` instead of `sqlite:////path.db` for absolute paths. * Bridge configs do this migration automatically. +* *(util.async_db)* Added warning log if using SQLite database path that isn't + writable. * *(client)* Stabilized support for asynchronous uploads. * `unstable_create_msc` was renamed to `create_mxc`, and the `max_stall_ms` parameters for downloading were renamed to `timeout_ms`.. +* *(crypto)* Added option to not rotate keys when devices change. ## v0.19.16 (2023-05-26) From a28b7576978f263354f433ef20b8943d8829f81b Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 14 Jun 2023 16:11:33 +0300 Subject: [PATCH 114/218] Simplify SimpleLock check --- mautrix/util/simple_lock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/util/simple_lock.py b/mautrix/util/simple_lock.py index ad979aec..c6bd08ce 100644 --- a/mautrix/util/simple_lock.py +++ b/mautrix/util/simple_lock.py @@ -47,7 +47,7 @@ def locked(self) -> bool: return not self.noop_mode and not self._event.is_set() async def wait(self, task: str | None = None) -> None: - if not self.noop_mode and not self._event.is_set(): + if self.locked: if self.log and self.message: self.log.debug(self.message, task) await self._event.wait() From 80c5a0fdb2768bcb883bd86711cd472c57141d3f Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 14 Jun 2023 16:12:43 +0300 Subject: [PATCH 115/218] Bump version to 0.20.0rc1 --- mautrix/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/__init__.py b/mautrix/__init__.py index ce7df73a..59187608 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.19.15" +__version__ = "0.20.0rc1" __author__ = "Tulir Asokan " __all__ = [ "api", From ad42ba80605ff257ef7d490e1e7bdcfedaa9a320 Mon Sep 17 00:00:00 2001 From: Max Sandholm Date: Thu, 22 Jun 2023 09:47:15 +0300 Subject: [PATCH 116/218] Add option to delete outdated inbound keys (keys which lack the metadata about when they're safe to delete) --- mautrix/bridge/config.py | 1 + mautrix/bridge/e2ee.py | 9 +++++++++ mautrix/crypto/store/abstract.py | 10 ++++++++++ mautrix/crypto/store/asyncpg/store.py | 15 +++++++++++++++ mautrix/crypto/store/memory.py | 3 +++ 5 files changed, 38 insertions(+) diff --git a/mautrix/bridge/config.py b/mautrix/bridge/config.py index 0dc536e6..1705e914 100644 --- a/mautrix/bridge/config.py +++ b/mautrix/bridge/config.py @@ -150,6 +150,7 @@ def do_update(self, helper: ConfigUpdateHelper) -> None: copy("bridge.encryption.delete_keys.delete_prev_on_new_session") copy("bridge.encryption.delete_keys.delete_on_device_delete") copy("bridge.encryption.delete_keys.periodically_delete_expired") + copy("bridge.encryption.delete_keys.delete_outdated_inbound") copy("bridge.encryption.verification_levels.receive") copy("bridge.encryption.verification_levels.send") copy("bridge.encryption.verification_levels.share") diff --git a/mautrix/bridge/e2ee.py b/mautrix/bridge/e2ee.py index b1a75b2c..7ae66abf 100644 --- a/mautrix/bridge/e2ee.py +++ b/mautrix/bridge/e2ee.py @@ -56,6 +56,7 @@ class EncryptionManager: key_sharing_enabled: bool appservice_mode: bool periodically_delete_expired_keys: bool + delete_outdated_inbound: bool bridge: br.Bridge az: AppService @@ -113,6 +114,7 @@ def __init__( self.az.to_device_handler = self.crypto.handle_as_to_device_event self.periodically_delete_expired_keys = False + self.delete_outdated_inbound = False self._key_delete_task = None del_cfg = bridge.config["bridge.encryption.delete_keys"] if del_cfg: @@ -123,6 +125,7 @@ def __init__( self.crypto.delete_fully_used_keys_on_decrypt = del_cfg["delete_fully_used_on_decrypt"] self.crypto.delete_keys_on_device_delete = del_cfg["delete_on_device_delete"] self.periodically_delete_expired_keys = del_cfg["periodically_delete_expired"] + self.delete_outdated_inbound = del_cfg["delete_outdated_inbound"] self.crypto.disable_device_change_key_rotation = bridge.config[ "bridge.encryption.rotation.disable_device_change_key_rotation" ] @@ -279,6 +282,12 @@ async def start(self) -> None: else: _ = self.client.start(self._filter) self.log.info("End-to-bridge encryption support is enabled (sync mode)") + if self.delete_outdated_inbound: + deleted = await self.crypto_store.redact_outdated_group_sessions() + if len(deleted) > 0: + self.log.debug( + f"Deleted {len(deleted)} inbound keys which lacked expiration metadata" + ) if self.periodically_delete_expired_keys: self._key_delete_task = background_task.create(self._periodically_delete_keys()) background_task.create(self._resync_encryption_info()) diff --git a/mautrix/crypto/store/abstract.py b/mautrix/crypto/store/abstract.py index 4db5aa1d..787d5225 100644 --- a/mautrix/crypto/store/abstract.py +++ b/mautrix/crypto/store/abstract.py @@ -237,6 +237,16 @@ async def redact_expired_group_sessions(self) -> list[SessionID]: The list of session IDs that were deleted. """ + @abstractmethod + async def redact_outdated_group_sessions(self) -> list[SessionID]: + """ + Remove all Megolm group sessions which lack the metadata to determine when they should + expire. + + Returns: + The list of session IDs that were deleted. + """ + @abstractmethod async def has_group_session(self, room_id: RoomID, session_id: SessionID) -> bool: """ diff --git a/mautrix/crypto/store/asyncpg/store.py b/mautrix/crypto/store/asyncpg/store.py index 0642d210..a29f7737 100644 --- a/mautrix/crypto/store/asyncpg/store.py +++ b/mautrix/crypto/store/asyncpg/store.py @@ -362,6 +362,21 @@ async def redact_expired_group_sessions(self) -> list[SessionID]: ) return [row["session_id"] for row in rows] + async def redact_outdated_group_sessions(self) -> list[SessionID]: + q = """ + UPDATE crypto_megolm_inbound_session + SET withheld_code=$1, withheld_reason=$2, session=NULL, forwarding_chains=NULL + WHERE account_id=$3 AND session IS NOT NULL AND received_at IS NULL + RETURNING session_id + """ + rows = await self.db.fetch( + q, + RoomKeyWithheldCode.BEEPER_REDACTED.value, + f"Session redacted: outdated", + self.account_id, + ) + return [row["session_id"] for row in rows] + async def has_group_session(self, room_id: RoomID, session_id: SessionID) -> bool: q = """ SELECT COUNT(session) FROM crypto_megolm_inbound_session diff --git a/mautrix/crypto/store/memory.py b/mautrix/crypto/store/memory.py index cfc27125..c26f86bc 100644 --- a/mautrix/crypto/store/memory.py +++ b/mautrix/crypto/store/memory.py @@ -134,6 +134,9 @@ async def redact_group_sessions( async def redact_expired_group_sessions(self) -> list[SessionID]: raise NotImplementedError() + async def redact_outdated_group_sessions(self) -> list[SessionID]: + raise NotImplementedError() + async def has_group_session(self, room_id: RoomID, session_id: SessionID) -> bool: return (room_id, session_id) in self._inbound_sessions From 2f9654c97db1bde2373b8adf2e13e5a9b25fd06e Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 23 Jun 2023 12:18:01 +0300 Subject: [PATCH 117/218] Match mautrix-go behavior for reply fallback removal --- mautrix/types/event/message.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/mautrix/types/event/message.py b/mautrix/types/event/message.py index 021e91f9..faaf0809 100644 --- a/mautrix/types/event/message.py +++ b/mautrix/types/event/message.py @@ -350,14 +350,12 @@ def trim_reply_fallback(self) -> None: setattr(self, "__reply_fallback_trimmed", True) def _trim_reply_fallback_text(self) -> None: - if not self.body.startswith("> ") or "\n" not in self.body: + if not self.body.startswith("> <") or "\n" not in self.body: return lines = self.body.split("\n") while len(lines) > 0 and lines[0].startswith("> "): lines.pop(0) - # Pop extra newline at end of fallback - lines.pop(0) - self.body = "\n".join(lines) + self.body = "\n".join(lines).strip() def _trim_reply_fallback_html(self) -> None: if self.formatted_body and self.format == Format.HTML: From 0634700d07540b8d6928712f45136d4ac3032546 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 23 Jun 2023 12:29:43 +0300 Subject: [PATCH 118/218] Handle m.emote fallbacks in reply fallback trimming safety --- mautrix/types/event/message.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mautrix/types/event/message.py b/mautrix/types/event/message.py index faaf0809..8d5f1bba 100644 --- a/mautrix/types/event/message.py +++ b/mautrix/types/event/message.py @@ -350,7 +350,9 @@ def trim_reply_fallback(self) -> None: setattr(self, "__reply_fallback_trimmed", True) def _trim_reply_fallback_text(self) -> None: - if not self.body.startswith("> <") or "\n" not in self.body: + if ( + not self.body.startswith("> <") and not self.body.startswith("> * <") + ) or "\n" not in self.body: return lines = self.body.split("\n") while len(lines) > 0 and lines[0].startswith("> "): From 654b215a9fbad3b506859e7a00f17d21c7f66253 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 25 Jun 2023 13:31:48 +0300 Subject: [PATCH 119/218] Fix using manual_stop while startup task is running --- mautrix/util/program.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/util/program.py b/mautrix/util/program.py index 80d2298b..f7a7b448 100644 --- a/mautrix/util/program.py +++ b/mautrix/util/program.py @@ -214,6 +214,7 @@ def _run(self) -> None: signal.signal(signal.SIGINT, signal.default_int_handler) signal.signal(signal.SIGTERM, signal.default_int_handler) + self._stop_task = self.loop.create_future() exit_code = 0 try: self.log.debug("Running startup actions...") @@ -224,7 +225,6 @@ def _run(self) -> None: f"Startup actions complete in {round(end_ts - start_ts, 2)} seconds, " "now running forever" ) - self._stop_task = self.loop.create_future() exit_code = self.loop.run_until_complete(self._stop_task) self.log.debug("manual_stop() called, stopping...") except KeyboardInterrupt: From 2ab1cdccf79a49f45189d246421223690d487cf5 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 25 Jun 2023 13:44:48 +0300 Subject: [PATCH 120/218] Bump version to 0.20.0 --- CHANGELOG.md | 9 +++++++-- mautrix/__init__.py | 2 +- setup.py | 2 +- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 74214d56..40771515 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -## v0.20.0 (unreleased) +## v0.20.0 (2023-06-25) * Dropped Python 3.8 support. * **Breaking change *(.state_store)*** Removed legacy SQLAlchemy state store @@ -11,10 +11,15 @@ * Bridge configs do this migration automatically. * *(util.async_db)* Added warning log if using SQLite database path that isn't writable. +* *(util.program)* Fixed `manual_stop` not working if it's called during startup. * *(client)* Stabilized support for asynchronous uploads. * `unstable_create_msc` was renamed to `create_mxc`, and the `max_stall_ms` - parameters for downloading were renamed to `timeout_ms`.. + parameters for downloading were renamed to `timeout_ms`. * *(crypto)* Added option to not rotate keys when devices change. +* *(crypto)* Added option to remove all keys that were received before the + automatic ratcheting was implemented (in v0.19.10). +* *(types)* Improved reply fallback removal to have a smaller chance of false + positives for messages that don't use reply fallbacks. ## v0.19.16 (2023-05-26) diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 59187608..16693197 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.20.0rc1" +__version__ = "0.20.0" __author__ = "Tulir Asokan " __all__ = [ "api", diff --git a/setup.py b/setup.py index 04a2a8c3..966e9f49 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ ], extras_require={ "detect_mimetype": ["python-magic>=0.4.15,<0.5"], - "lint": ["black~=22.1", "isort"], + "lint": ["black~=23.1", "isort"], "test": ["pytest", "pytest-asyncio", *test_dependencies], "encryption": encryption_dependencies, }, From 9b67b17ac4927f63cc9a71dc450b9296c849b3ab Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 1 Jul 2023 13:58:59 +0300 Subject: [PATCH 121/218] Remove --base-config flag from program utility --- CHANGELOG.md | 6 ++++++ mautrix/util/program.py | 23 ++++++----------------- 2 files changed, 12 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 40771515..7ddf6b9c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## unreleased + +* *(util.program)* Removed `--base-config` flag in bridges, as there are no + valid use cases (package data should always work) and it's easy to cause + issues by pointing the flag at the wrong file. + ## v0.20.0 (2023-06-25) * Dropped Python 3.8 support. diff --git a/mautrix/util/program.py b/mautrix/util/program.py index f7a7b448..2fb6b141 100644 --- a/mautrix/util/program.py +++ b/mautrix/util/program.py @@ -119,7 +119,7 @@ def preinit(self) -> None: self.check_config() @property - def _default_base_config(self) -> str: + def base_config_path(self) -> str: return f"pkg://{self.module}/example-config.yaml" def prepare_arg_parser(self) -> None: @@ -133,21 +133,13 @@ def prepare_arg_parser(self) -> None: metavar="", help="the path to your config file", ) - self.parser.add_argument( - "-b", - "--base-config", - type=str, - default=self._default_base_config, - metavar="", - help="the path to the example config (for automatic config updates)", - ) self.parser.add_argument( "-n", "--no-update", action="store_true", help="Don't save updated config to disk" ) def prepare_config(self) -> None: """Pre-init lifecycle method. Extend this if you want to customize config loading.""" - self.config = self.config_class(self.args.config, self.args.base_config) + self.config = self.config_class(self.args.config, self.base_config_path) self.load_and_update_config() def load_and_update_config(self) -> None: @@ -155,13 +147,10 @@ def load_and_update_config(self) -> None: try: self.config.update(save=not self.args.no_update) except BaseMissingError: - if self.args.base_config != self._default_base_config: - print(f"Failed to read base config from {self.args.base_config}") - else: - print( - "Failed to read base config from the default path " - f"({self._default_base_config}). Maybe your installation is corrupted?" - ) + print( + "Failed to read base config from the default path " + f"({self.base_config_path}). Maybe your installation is corrupted?" + ) sys.exit(12) def check_config(self) -> None: From 782f568d7873972e0b5ef7ba698df4243e3a4151 Mon Sep 17 00:00:00 2001 From: Sumner Evans Date: Fri, 28 Jul 2023 17:38:44 -0600 Subject: [PATCH 122/218] bridge/prepare_config: replace last instance of args.base_config with self.base_config_path Signed-off-by: Sumner Evans --- mautrix/bridge/bridge.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/bridge/bridge.py b/mautrix/bridge/bridge.py index 37e1646d..9ce9360e 100644 --- a/mautrix/bridge/bridge.py +++ b/mautrix/bridge/bridge.py @@ -136,7 +136,7 @@ def prepare_config(self) -> None: self.config = self.config_class( self.args.config, self.args.registration, - self.args.base_config, + self.base_config_path, env_prefix=self.module.upper(), ) if self.args.generate_registration: From 37c791b8fb877c283180259c2d16864302d00f96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gabriel=20Rodr=C3=ADguez?= Date: Sun, 30 Jul 2023 15:08:54 -0400 Subject: [PATCH 123/218] fix typo: seconds -> milliseconds --- mautrix/client/api/modules/misc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/client/api/modules/misc.py b/mautrix/client/api/modules/misc.py index 443877c1..8ff9c85a 100644 --- a/mautrix/client/api/modules/misc.py +++ b/mautrix/client/api/modules/misc.py @@ -50,7 +50,7 @@ async def set_typing(self, room_id: RoomID, timeout: int = 0) -> None: Args: room_id: The ID of the room in which the user is typing. - timeout: The length of time in seconds to mark this user as typing. + timeout: The length of time in milliseconds to mark this user as typing. """ if timeout > 0: content = {"typing": True, "timeout": timeout} From 2d598551f7e795840e3c22e66511f05ddbf5be3a Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 4 Aug 2023 21:08:19 +0300 Subject: [PATCH 124/218] Add support for `com.devture.shared_secret_auth` for double puppeting --- mautrix/bridge/custom_puppet.py | 50 +++++++++++++++++---------------- mautrix/types/auth.py | 2 ++ 2 files changed, 28 insertions(+), 24 deletions(-) diff --git a/mautrix/bridge/custom_puppet.py b/mautrix/bridge/custom_puppet.py index e2759513..c09095ab 100644 --- a/mautrix/bridge/custom_puppet.py +++ b/mautrix/bridge/custom_puppet.py @@ -17,8 +17,8 @@ from aiohttp import ClientConnectionError from yarl import URL -from mautrix.api import Path from mautrix.appservice import AppService, IntentAPI +from mautrix.client import ClientAPI from mautrix.errors import ( IntentError, MatrixError, @@ -33,6 +33,7 @@ Filter, FilterID, LoginType, + MatrixUserIdentifier, PresenceState, RoomEventFilter, RoomFilter, @@ -182,31 +183,32 @@ async def _login_with_shared_secret(cls, mxid: UserID) -> str: base_url = cls.az.intent.api.base_url else: raise AutologinError(f"No homeserver URL configured for {server}") - url = base_url / str(Path.v3.login) - headers = {"Content-Type": "application/json"} - login_req = { - "initial_device_display_name": cls.login_device_name, - "device_id": cls.login_device_name, - "identifier": { - "type": "m.id.user", - "user": mxid, - }, - } + client = ClientAPI(base_url=base_url) + login_args = {} if secret == b"appservice": - login_req["type"] = str(LoginType.APPSERVICE) - headers["Authorization"] = f"Bearer {cls.az.as_token}" + login_type = LoginType.APPSERVICE + client.api.token = cls.az.as_token else: - login_req["type"] = str(LoginType.PASSWORD) - login_req["password"] = hmac.new( - secret, mxid.encode("utf-8"), hashlib.sha512 - ).hexdigest() - resp = await cls.az.http_session.post(url, data=json.dumps(login_req), headers=headers) - data = await resp.json() - try: - return data["access_token"] - except KeyError: - error_msg = data.get("error", data.get("errcode", f"HTTP {resp.status}")) - raise AutologinError(f"Didn't get an access token: {error_msg}") from None + flows = await client.get_login_flows() + flow = flows.get_first_of_type(LoginType.DEVTURE_SHARED_SECRET, LoginType.PASSWORD) + if not flow: + raise AutologinError("No supported shared secret auth login flows") + login_type = flow.type + token = hmac.new(secret, mxid.encode("utf-8"), hashlib.sha512).hexdigest() + if login_type == LoginType.DEVTURE_SHARED_SECRET: + login_args["token"] = token + elif login_type == LoginType.PASSWORD: + login_args["password"] = token + resp = await client.login( + identifier=MatrixUserIdentifier(user=mxid), + device_id=cls.login_device_name, + initial_device_display_name=cls.login_device_name, + login_type=login_type, + **login_args, + store_access_token=False, + update_hs_url=False, + ) + return resp.access_token async def switch_mxid( self, access_token: str | None, mxid: UserID | None, start_sync_task: bool = True diff --git a/mautrix/types/auth.py b/mautrix/types/auth.py index 198d2f53..ad582118 100644 --- a/mautrix/types/auth.py +++ b/mautrix/types/auth.py @@ -26,6 +26,8 @@ class LoginType(ExtensibleEnum): UNSTABLE_JWT: "LoginType" = "org.matrix.login.jwt" + DEVTURE_SHARED_SECRET: "LoginType" = "com.devture.shared_secret_auth" + @dataclass class LoginFlow(SerializableAttrs): From c4ffe9be4ecd55624a79a5161a40fb84ee20f072 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 7 Aug 2023 17:23:06 +0300 Subject: [PATCH 125/218] Fix error code in crypto-related MSS events --- mautrix/bridge/matrix.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/bridge/matrix.py b/mautrix/bridge/matrix.py index dc7d9fcf..da42a95e 100644 --- a/mautrix/bridge/matrix.py +++ b/mautrix/bridge/matrix.py @@ -620,7 +620,7 @@ async def _send_crypto_status_error( evt, status=MessageStatus.RETRIABLE if is_final else MessageStatus.PENDING, reason=MessageStatusReason.UNDECRYPTABLE, - error=msg, + error=str(err), message=err.human_message if err else None, ) From e0aad7111253c41669814df3fbfab96a6a31b4df Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 27 Aug 2023 00:23:09 +0300 Subject: [PATCH 126/218] Drop support for syncing with double puppets --- mautrix/bridge/commands/login_matrix.py | 21 ---- mautrix/bridge/custom_puppet.py | 136 ++---------------------- 2 files changed, 8 insertions(+), 149 deletions(-) diff --git a/mautrix/bridge/commands/login_matrix.py b/mautrix/bridge/commands/login_matrix.py index 2fd89d2d..6cc6f7aa 100644 --- a/mautrix/bridge/commands/login_matrix.py +++ b/mautrix/bridge/commands/login_matrix.py @@ -70,24 +70,3 @@ async def ping_matrix(evt: CommandEvent) -> EventID: except InvalidAccessToken: return await evt.reply("Your access token is invalid.") return await evt.reply("Your Matrix login is working.") - - -@command_handler( - needs_auth=True, - help_section=SECTION_AUTH, - help_text="Clear the Matrix sync token stored for your double puppet.", -) -async def clear_cache_matrix(evt: CommandEvent) -> EventID: - try: - puppet = await evt.sender.get_puppet() - except NotImplementedError: - return await evt.reply("This bridge has not implemented the clear-cache-matrix command") - if not puppet.is_real_user: - return await evt.reply("You are not logged in with your Matrix account.") - try: - puppet.stop() - puppet.next_batch = None - await puppet.start() - except InvalidAccessToken: - return await evt.reply("Your access token is invalid.") - return await evt.reply("Cleared cache successfully.") diff --git a/mautrix/bridge/custom_puppet.py b/mautrix/bridge/custom_puppet.py index c09095ab..2605e003 100644 --- a/mautrix/bridge/custom_puppet.py +++ b/mautrix/bridge/custom_puppet.py @@ -116,7 +116,6 @@ class CustomPuppetMixin(ABC): intent: The primary IntentAPI. """ - sync_with_custom_puppets: bool = True allow_discover_url: bool = False homeserver_url_map: dict[str, URL] = {} only_handle_own_synced_events: bool = True @@ -135,12 +134,9 @@ class CustomPuppetMixin(ABC): custom_mxid: UserID | None access_token: str | None base_url: URL | None - next_batch: SyncToken | None intent: IntentAPI - _sync_task: asyncio.Task | None = None - @abstractmethod async def save(self) -> None: """Save the information of this puppet. Called from :meth:`switch_mxid`""" @@ -221,7 +217,6 @@ async def switch_mxid( the appservice-owned ID. mxid: The expected Matrix user ID of the custom account, or ``None`` when ``access_token`` is None. - start_sync_task: Whether or not syncing should be started after logging in. """ if access_token == "auto": access_token = await self._login_with_shared_secret(mxid) @@ -251,7 +246,7 @@ async def switch_mxid( self.base_url = base_url self.intent = self._fresh_intent() - await self.start(start_sync_task=start_sync_task, check_e2ee_keys=True) + await self.start(check_e2ee_keys=True) try: del self.by_custom_mxid[prev_mxid] @@ -276,7 +271,6 @@ async def _invalidate_double_puppet(self) -> None: del self.by_custom_mxid[self.custom_mxid] self.custom_mxid = None self.access_token = None - self.next_batch = None await self.save() self.intent = self._fresh_intent() @@ -296,7 +290,7 @@ async def start( except MatrixInvalidToken as e: if retry_auto_login and self.custom_mxid and self.can_auto_login(self.custom_mxid): self.log.debug(f"Got {e.errcode} while trying to initialize custom mxid") - await self.switch_mxid("auto", self.custom_mxid, start_sync_task=start_sync_task) + await self.switch_mxid("auto", self.custom_mxid) return self.log.warning(f"Got {e.errcode} while trying to initialize custom mxid") whoami = None @@ -319,19 +313,14 @@ async def start( if device_keys and len(device_keys.keys) > 0: await self._invalidate_double_puppet() raise EncryptionKeysFound() - if self.sync_with_custom_puppets and start_sync_task: - if self._sync_task: - self._sync_task.cancel() - self.log.info(f"Initialized custom mxid: {whoami.user_id}. Starting sync task") - self._sync_task = asyncio.create_task(self._try_sync()) - else: - self.log.info(f"Initialized custom mxid: {whoami.user_id}. Not starting sync task") + self.log.info(f"Initialized custom mxid: {whoami.user_id}") def stop(self) -> None: - """Cancel the sync task.""" - if self._sync_task: - self._sync_task.cancel() - self._sync_task = None + """ + No-op + + .. deprecated:: 0.20.1 + """ async def default_puppet_should_leave_room(self, room_id: RoomID) -> bool: """ @@ -354,112 +343,3 @@ async def _leave_rooms_with_default_user(self) -> None: await self.intent.ensure_joined(room_id) except (IntentError, MatrixRequestError): pass - - def _create_sync_filter(self) -> Awaitable[FilterID]: - all_events = EventType.find("*") - return self.intent.create_filter( - Filter( - account_data=EventFilter(types=[all_events]), - presence=EventFilter( - types=[EventType.PRESENCE], - senders=[self.custom_mxid] if self.only_handle_own_synced_events else None, - ), - room=RoomFilter( - include_leave=False, - state=StateFilter(not_types=[all_events]), - timeline=RoomEventFilter(not_types=[all_events]), - account_data=RoomEventFilter(not_types=[all_events]), - ephemeral=RoomEventFilter( - types=[ - EventType.TYPING, - EventType.RECEIPT, - ] - ), - ), - ) - ) - - def _filter_events(self, room_id: RoomID, events: list[dict]) -> Iterator[Event]: - for event in events: - event["room_id"] = room_id - if self.only_handle_own_synced_events: - # We only want events about the custom puppet user, but we can't use - # filters for typing and read receipt events. - evt_type = EventType.find(event.get("type", None)) - event.setdefault("content", {}) - if evt_type == EventType.TYPING: - is_typing = self.custom_mxid in event["content"].get("user_ids", []) - event["content"]["user_ids"] = [self.custom_mxid] if is_typing else [] - elif evt_type == EventType.RECEIPT: - try: - event_id, receipt = event["content"].popitem() - data = receipt["m.read"][self.custom_mxid] - event["content"] = {event_id: {"m.read": {self.custom_mxid: data}}} - except KeyError: - continue - yield event - - def _handle_sync(self, sync_resp: dict) -> None: - # Get events from rooms -> join -> [room_id] -> ephemeral -> events (array) - ephemeral_events = ( - event - for room_id, data in sync_resp.get("rooms", {}).get("join", {}).items() - for event in self._filter_events(room_id, data.get("ephemeral", {}).get("events", [])) - ) - - # Get events from presence -> events (array) - presence_events = sync_resp.get("presence", {}).get("events", []) - - # Deserialize and handle all events - for event in chain(ephemeral_events, presence_events): - background_task.create(self.mx.try_handle_sync_event(Event.deserialize(event))) - - async def _try_sync(self) -> None: - try: - await self._sync() - except asyncio.CancelledError: - self.log.info(f"Syncing for {self.custom_mxid} cancelled") - except Exception: - self.log.critical(f"Fatal error syncing {self.custom_mxid}", exc_info=True) - - async def _sync(self) -> None: - if not self.is_real_user: - self.log.warning("Called sync() for non-custom puppet.") - return - custom_mxid: UserID = self.custom_mxid - access_token_at_start: str = self.access_token - errors: int = 0 - filter_id: FilterID = await self._create_sync_filter() - self.log.debug(f"Starting syncer for {custom_mxid} with sync filter {filter_id}.") - while access_token_at_start == self.access_token: - try: - cur_batch = self.next_batch - sync_resp = await self.intent.sync( - filter_id=filter_id, since=cur_batch, set_presence=PresenceState.OFFLINE - ) - try: - self.next_batch = sync_resp.get("next_batch", None) - except Exception: - self.log.warning("Failed to store next batch", exc_info=True) - errors = 0 - if cur_batch is not None: - self._handle_sync(sync_resp) - except MatrixInvalidToken: - # TODO when not using syncing, we should still check this occasionally and relogin - self.log.warning(f"Access token for {custom_mxid} got invalidated, restarting...") - await self.start(retry_auto_login=True, start_sync_task=False) - if self.is_real_user: - self.log.info("Successfully relogined custom puppet, continuing sync") - filter_id = await self._create_sync_filter() - access_token_at_start = self.access_token - else: - self.log.warning("Something went wrong during relogin") - raise - except (MatrixError, ClientConnectionError, asyncio.TimeoutError) as e: - errors += 1 - wait = min(errors, 11) ** 2 - self.log.warning( - f"Syncer for {custom_mxid} errored: {e}. Waiting for {wait} seconds..." - ) - await asyncio.sleep(wait) - self.log.debug(f"Syncer for custom puppet {custom_mxid} stopped.") From af04ca12389e344fbda10463085ed5df4b62db69 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 27 Aug 2023 00:37:50 +0300 Subject: [PATCH 127/218] Add support for double puppeting with another as_token --- mautrix/appservice/api/appservice.py | 26 ++++++++++++++++++++++---- mautrix/appservice/api/intent.py | 10 +++++++--- mautrix/bridge/custom_puppet.py | 18 +++++++++++++++++- 3 files changed, 46 insertions(+), 8 deletions(-) diff --git a/mautrix/appservice/api/appservice.py b/mautrix/appservice/api/appservice.py index af6ae48d..6654ec20 100644 --- a/mautrix/appservice/api/appservice.py +++ b/mautrix/appservice/api/appservice.py @@ -51,6 +51,7 @@ def __init__( client_session: ClientSession = None, child: bool = False, real_user: bool = False, + real_user_as_token: bool = False, bridge_name: str | None = None, default_retry_count: int = None, loop: asyncio.AbstractEventLoop | None = None, @@ -66,6 +67,7 @@ def __init__( client_session: The aiohttp ClientSession to use. child: Whether or not this is instance is a child of another AppServiceAPI. real_user: Whether or not this is a real (non-appservice-managed) user. + real_user_as_token: Whether this real user is actually using another ``as_token``. bridge_name: The name of the bridge to put in the ``fi.mau.double_puppet_source`` field in outgoing message events sent through real users. """ @@ -85,6 +87,7 @@ def __init__( self._bot_intent = None self.state_store = state_store self.is_real_user = real_user + self.is_real_user_as_token = real_user_as_token self.bridge_name = bridge_name if not child: @@ -113,7 +116,9 @@ def user(self, user: UserID) -> ChildAppServiceAPI: self.children[user] = child return child - def real_user(self, mxid: UserID, token: str, base_url: URL | None = None) -> AppServiceAPI: + def real_user( + self, mxid: UserID, token: str, base_url: URL | None = None, as_token: bool = False + ) -> AppServiceAPI: """ Get the AppServiceAPI for a real (non-appservice-managed) Matrix user. @@ -122,6 +127,8 @@ def real_user(self, mxid: UserID, token: str, base_url: URL | None = None) -> Ap token: The access token for the user. base_url: The base URL of the homeserver client-server API to use. Defaults to the appservice homeserver URL. + as_token: Whether the token is actually an as_token + (meaning the ``user_id`` query parameter needs to be used). Returns: The AppServiceAPI object for the user. @@ -136,6 +143,7 @@ def real_user(self, mxid: UserID, token: str, base_url: URL | None = None) -> Ap child = self.real_users[mxid] child.base_url = base_url or child.base_url child.token = token or child.token + child.is_real_user_as_token = as_token except KeyError: child = type(self)( base_url=base_url or self.base_url, @@ -145,6 +153,7 @@ def real_user(self, mxid: UserID, token: str, base_url: URL | None = None) -> Ap state_store=self.state_store, client_session=self.session, real_user=True, + real_user_as_token=as_token, bridge_name=self.bridge_name, default_retry_count=self.default_retry_count, ) @@ -163,7 +172,11 @@ def bot_intent(self) -> as_api.IntentAPI: return self._bot_intent def intent( - self, user: UserID = None, token: str | None = None, base_url: str | None = None + self, + user: UserID = None, + token: str | None = None, + base_url: str | None = None, + real_user_as_token: bool = False, ) -> as_api.IntentAPI: """ Get the intent API of a child user. @@ -173,6 +186,8 @@ def intent( token: The access token to use. Only applicable for non-appservice-managed users. base_url: The base URL of the homeserver client-server API to use. Only applicable for non-appservice users. Defaults to the appservice homeserver URL. + real_user_as_token: When providing a token, whether it's actually another as_token + (meaning the ``user_id`` query parameter needs to be used). Returns: The IntentAPI object for the given user. @@ -184,7 +199,10 @@ def intent( raise ValueError("Can't get child intent of real user") if token: return as_api.IntentAPI( - user, self.real_user(user, token, base_url), self.bot_intent(), self.state_store + user, + self.real_user(user, token, base_url, as_token=real_user_as_token), + self.bot_intent(), + self.state_store, ) return as_api.IntentAPI(user, self.user(user), self.bot_intent(), self.state_store) @@ -229,7 +247,7 @@ def request( if isinstance(timestamp, datetime): timestamp = int(timestamp.replace(tzinfo=timezone.utc).timestamp() * 1000) query_params["ts"] = timestamp - if not self.is_real_user: + if not self.is_real_user or self.is_real_user_as_token: query_params["user_id"] = self.identity or self.bot_mxid return super().request( diff --git a/mautrix/appservice/api/intent.py b/mautrix/appservice/api/intent.py index ab9e1e82..1892743e 100644 --- a/mautrix/appservice/api/intent.py +++ b/mautrix/appservice/api/intent.py @@ -144,7 +144,11 @@ async def wrapper(*args, __self=self, __method=method, **kwargs): setattr(self, method.__name__, wrapper) def user( - self, user_id: UserID, token: str | None = None, base_url: str | None = None + self, + user_id: UserID, + token: str | None = None, + base_url: str | None = None, + as_token: bool = False, ) -> IntentAPI: """ Get the intent API for a specific user. @@ -162,10 +166,10 @@ def user( The IntentAPI for the given user. """ if not self.bot: - return self.api.intent(user_id, token, base_url) + return self.api.intent(user_id, token, base_url, real_user_as_token=as_token) else: self.log.warning("Called IntentAPI#user() of child intent object.") - return self.bot.api.intent(user_id, token, base_url) + return self.bot.api.intent(user_id, token, base_url, real_user_as_token=as_token) # region User actions diff --git a/mautrix/bridge/custom_puppet.py b/mautrix/bridge/custom_puppet.py index 2605e003..a702ea25 100644 --- a/mautrix/bridge/custom_puppet.py +++ b/mautrix/bridge/custom_puppet.py @@ -152,6 +152,19 @@ def is_real_user(self) -> bool: return bool(self.custom_mxid and self.access_token) def _fresh_intent(self) -> IntentAPI: + if self.access_token == "appservice-config" and self.custom_mxid: + _, server = self.az.intent.parse_user_id(self.custom_mxid) + try: + secret = self.login_shared_secret_map[server] + except KeyError: + raise AutologinError(f"No shared secret configured for {server}") + self.log.debug(f"Using as_token for double puppeting {self.custom_mxid}") + return self.az.intent.user( + self.custom_mxid, + secret.decode("utf-8").removeprefix("as_token:"), + self.base_url, + as_token=True, + ) return ( self.az.intent.user(self.custom_mxid, self.access_token, self.base_url) if self.is_real_user @@ -172,6 +185,8 @@ async def _login_with_shared_secret(cls, mxid: UserID) -> str: secret = cls.login_shared_secret_map[server] except KeyError: raise AutologinError(f"No shared secret configured for {server}") + if secret.startswith(b"as_token:"): + return "appservice-config" try: base_url = cls.homeserver_url_map[server] except KeyError: @@ -220,7 +235,8 @@ async def switch_mxid( """ if access_token == "auto": access_token = await self._login_with_shared_secret(mxid) - self.log.debug(f"Logged in for {mxid} using shared secret") + if access_token != "appservice-config": + self.log.debug(f"Logged in for {mxid} using shared secret") if mxid is not None: _, mxid_domain = self.az.intent.parse_user_id(mxid) From 6e4c86de6865fee68c6a3c83eb9868e7ab2ecdce Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 27 Aug 2023 00:49:32 +0300 Subject: [PATCH 128/218] Remove unused imports --- mautrix/bridge/custom_puppet.py | 22 +--------------------- 1 file changed, 1 insertion(+), 21 deletions(-) diff --git a/mautrix/bridge/custom_puppet.py b/mautrix/bridge/custom_puppet.py index a702ea25..9057877c 100644 --- a/mautrix/bridge/custom_puppet.py +++ b/mautrix/bridge/custom_puppet.py @@ -5,16 +5,12 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import annotations -from typing import Awaitable, Iterator from abc import ABC, abstractmethod -from itertools import chain import asyncio import hashlib import hmac -import json import logging -from aiohttp import ClientConnectionError from yarl import URL from mautrix.appservice import AppService, IntentAPI @@ -26,23 +22,7 @@ MatrixRequestError, WellKnownError, ) -from mautrix.types import ( - Event, - EventFilter, - EventType, - Filter, - FilterID, - LoginType, - MatrixUserIdentifier, - PresenceState, - RoomEventFilter, - RoomFilter, - RoomID, - StateFilter, - SyncToken, - UserID, -) -from mautrix.util import background_task +from mautrix.types import LoginType, MatrixUserIdentifier, RoomID, UserID from .. import bridge as br From a48ef9f39139cc01286efd46fa8c70d3f3ab19fe Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 27 Aug 2023 00:49:36 +0300 Subject: [PATCH 129/218] Update changelog --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ddf6b9c..4adf9eb2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,11 @@ * *(util.program)* Removed `--base-config` flag in bridges, as there are no valid use cases (package data should always work) and it's easy to cause issues by pointing the flag at the wrong file. +* *(bridge)* Added support for the `com.devture.shared_secret_auth` login type + for automatic double puppeting. +* *(bridge)* Dropped support for syncing with double puppets. MSC2409 is now + the only way to receive ephemeral events. +* *(bridge)* Added support for double puppeting with arbitrary `as_token`s. ## v0.20.0 (2023-06-25) From 958e198ef21c14e08cd4335c3afb0bb1e2f315d7 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 29 Aug 2023 20:49:17 +0300 Subject: [PATCH 130/218] Bump version to 0.20.1 --- CHANGELOG.md | 2 +- mautrix/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4adf9eb2..6278822a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -## unreleased +## v0.20.1 (2023-08-29) * *(util.program)* Removed `--base-config` flag in bridges, as there are no valid use cases (package data should always work) and it's easy to cause diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 16693197..19015ee4 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.20.0" +__version__ = "0.20.1" __author__ = "Tulir Asokan " __all__ = [ "api", From cef7b5af66e50bb440712cd94c5979b80d852685 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 6 Sep 2023 21:23:26 +0300 Subject: [PATCH 131/218] Add option to run appservice handlers synchronously --- mautrix/appservice/as_handler.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/mautrix/appservice/as_handler.py b/mautrix/appservice/as_handler.py index c0a41f73..fc44ac66 100644 --- a/mautrix/appservice/as_handler.py +++ b/mautrix/appservice/as_handler.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022 Tulir Asokan +# Copyright (c) 2023 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this @@ -8,7 +8,6 @@ from typing import Any, Awaitable, Callable from json import JSONDecodeError -import asyncio import json import logging @@ -33,12 +32,12 @@ class AppServiceServerMixin: - loop: asyncio.AbstractEventLoop log: logging.Logger hs_token: str ephemeral_events: bool encryption_events: bool + synchronous_handlers: bool query_user: Callable[[UserID], JSON] query_alias: Callable[[RoomAlias], JSON] @@ -57,6 +56,7 @@ def __init__(self, ephemeral_events: bool = False, encryption_events: bool = Fal self.device_list_handler = None self.ephemeral_events = ephemeral_events self.encryption_events = encryption_events + self.synchronous_handlers = False async def default_query_handler(_): return None @@ -309,7 +309,7 @@ async def handle_transaction( except SerializerError: self.log.exception("Failed to deserialize ephemeral event %s", raw_edu) else: - self.handle_matrix_event(edu, ephemeral=True) + await self.handle_matrix_event(edu, ephemeral=True) for raw_event in events: try: self._fix_prev_content(raw_event) @@ -317,10 +317,10 @@ async def handle_transaction( except SerializerError: self.log.exception("Failed to deserialize event %s", raw_event) else: - self.handle_matrix_event(event) + await self.handle_matrix_event(event) return {} - def handle_matrix_event(self, event: Event, ephemeral: bool = False) -> None: + async def handle_matrix_event(self, event: Event, ephemeral: bool = False) -> None: if ephemeral: event.type = event.type.with_class(EventType.Class.EPHEMERAL) elif getattr(event, "state_key", None) is not None: @@ -334,9 +334,12 @@ async def try_handle(handler_func: HandlerFunc): except Exception: self.log.exception("Exception in Matrix event handler") - for handler in self.event_handlers: - # TODO add option to handle events synchronously - background_task.create(try_handle(handler)) + if self.synchronous_handlers: + for handler in self.event_handlers: + await handler(event) + else: + for handler in self.event_handlers: + background_task.create(try_handle(handler)) def matrix_event_handler(self, func: HandlerFunc) -> HandlerFunc: self.event_handlers.append(func) From cb26030cc16b1c2541c77453366718c5328a46c4 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 6 Sep 2023 21:23:58 +0300 Subject: [PATCH 132/218] Make AppServiceServerMixin usable standalone --- mautrix/appservice/as_handler.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/mautrix/appservice/as_handler.py b/mautrix/appservice/as_handler.py index fc44ac66..33bf4bf7 100644 --- a/mautrix/appservice/as_handler.py +++ b/mautrix/appservice/as_handler.py @@ -48,7 +48,17 @@ class AppServiceServerMixin: otk_handler: Callable[[dict[UserID, dict[DeviceID, DeviceOTKCount]]], Awaitable] | None device_list_handler: Callable[[DeviceLists], Awaitable] | None - def __init__(self, ephemeral_events: bool = False, encryption_events: bool = False) -> None: + def __init__( + self, + ephemeral_events: bool = False, + encryption_events: bool = False, + log: logging.Logger | None = None, + hs_token: str | None = None, + ) -> None: + if log is not None: + self.log = log + if hs_token is not None: + self.hs_token = hs_token self.transactions = set() self.event_handlers = [] self.to_device_handler = None From c6979e877f94a4849872e5c66984947ff643006f Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 6 Sep 2023 21:24:26 +0300 Subject: [PATCH 133/218] Clean up AppServiceServerMixin --- mautrix/appservice/as_handler.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mautrix/appservice/as_handler.py b/mautrix/appservice/as_handler.py index 33bf4bf7..ec7e339f 100644 --- a/mautrix/appservice/as_handler.py +++ b/mautrix/appservice/as_handler.py @@ -86,7 +86,6 @@ def register_routes(self, app: web.Application) -> None: app.router.add_route("GET", "/_matrix/app/v1/rooms/{alias}", self._http_query_alias) app.router.add_route("GET", "/_matrix/app/v1/users/{user_id}", self._http_query_user) app.router.add_route("POST", "/_matrix/app/v1/ping", self._http_ping) - app.router.add_route("POST", "/_matrix/app/unstable/fi.mau.msc2659/ping", self._http_ping) def _check_token(self, request: web.Request) -> bool: try: @@ -301,7 +300,7 @@ async def handle_transaction( else: try: await self.to_device_handler(td) - except: + except Exception: self.log.exception("Exception in Matrix to-device event handler") if device_lists and self.device_list_handler: try: From e1b28d39ce7d2e0e1ada8afea3d6a07415fdc24e Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 6 Sep 2023 21:24:47 +0300 Subject: [PATCH 134/218] Allow using OlmMachine.share_keys without OTK count --- mautrix/crypto/base.py | 2 ++ mautrix/crypto/machine.py | 26 +++++++++++++++++++++++--- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/mautrix/crypto/base.py b/mautrix/crypto/base.py index e1386834..1f8e9a62 100644 --- a/mautrix/crypto/base.py +++ b/mautrix/crypto/base.py @@ -66,6 +66,8 @@ class BaseOlmMachine: _prev_unwedge: dict[IdentityKey, float] _fetch_keys_lock: asyncio.Lock _megolm_decrypt_lock: asyncio.Lock + _share_keys_lock: asyncio.Lock + _last_key_share: float _cs_fetch_attempted: set[UserID] async def wait_for_session( diff --git a/mautrix/crypto/machine.py b/mautrix/crypto/machine.py index c68335af..0cfe6ea3 100644 --- a/mautrix/crypto/machine.py +++ b/mautrix/crypto/machine.py @@ -8,6 +8,7 @@ from typing import Optional import asyncio import logging +import time from mautrix import client as cli from mautrix.errors import GroupSessionWithheldError @@ -18,6 +19,7 @@ DeviceLists, DeviceOTKCount, EncryptionAlgorithm, + EncryptionKeyAlgorithm, EventType, Member, Membership, @@ -87,6 +89,8 @@ def __init__( self._fetch_keys_lock = asyncio.Lock() self._megolm_decrypt_lock = asyncio.Lock() + self._share_keys_lock = asyncio.Lock() + self._last_key_share = time.monotonic() - 60 self._key_request_waiters = {} self._inbound_session_waiters = {} self._prev_unwedge = {} @@ -267,14 +271,29 @@ async def handle_beep_room_key_ack(self, evt: ToDeviceEvent) -> None: else: self.log.debug(f"Received room key ack for {sess.id}") - async def share_keys(self, current_otk_count: int) -> None: + async def share_keys(self, current_otk_count: int | None = None) -> None: """ Share any keys that need to be shared. This is automatically called from :meth:`handle_otk_count`, so you should not need to call this yourself. Args: current_otk_count: The current number of signed curve25519 keys present on the server. + If omitted, the count will be fetched from the server. """ + async with self._share_keys_lock: + await self._share_keys(current_otk_count) + + async def _share_keys(self, current_otk_count: int | None) -> None: + if current_otk_count is None or ( + # If the last key share was recent and the new count is very low, re-check the count + # from the server to avoid any race conditions. + self._last_key_share + 60 > time.monotonic() + and current_otk_count < 10 + ): + self.log.debug("Checking OTK count on server") + current_otk_count = (await self.client.upload_keys()).get( + EncryptionKeyAlgorithm.SIGNED_CURVE25519 + ) device_keys = ( self.account.get_device_keys(self.client.mxid, self.client.device_id) if not self.account.shared @@ -289,7 +308,8 @@ async def share_keys(self, current_otk_count: int) -> None: if device_keys: self.log.debug("Going to upload initial account keys") self.log.debug(f"Uploading {len(one_time_keys)} one-time keys") - await self.client.upload_keys(one_time_keys=one_time_keys, device_keys=device_keys) + resp = await self.client.upload_keys(one_time_keys=one_time_keys, device_keys=device_keys) self.account.shared = True + self._last_key_share = time.monotonic() await self.crypto_store.put_account(self.account) - self.log.debug("Shared keys and saved account") + self.log.debug(f"Shared keys and saved account, new keys: {resp}") From e9f8bf8189f04755a586bf929fb00126af3296f7 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 6 Sep 2023 21:24:56 +0300 Subject: [PATCH 135/218] Add missing item in errors.__all__ --- mautrix/errors/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mautrix/errors/__init__.py b/mautrix/errors/__init__.py index d3646a37..afe68dc9 100644 --- a/mautrix/errors/__init__.py +++ b/mautrix/errors/__init__.py @@ -73,6 +73,7 @@ "DeviceValidationError", "DuplicateMessageIndex", "EncryptionError", + "GroupSessionWithheldError", "MatchingSessionDecryptionError", "MismatchingRoomError", "SessionNotFound", From 980d8e4350f7efe7519a50db8d6a80b3393ce70b Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 6 Sep 2023 23:08:02 +0300 Subject: [PATCH 136/218] Use proper background task for async event handlers --- mautrix/client/syncer.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mautrix/client/syncer.py b/mautrix/client/syncer.py index 62ae97a1..8e115bbb 100644 --- a/mautrix/client/syncer.py +++ b/mautrix/client/syncer.py @@ -34,6 +34,7 @@ ToDeviceEvent, UserID, ) +from mautrix.util import background_task from mautrix.util.logging import TraceLogger from . import dispatcher @@ -248,9 +249,10 @@ def dispatch_manual_event( handlers = self.global_event_handlers + handlers tasks = [] for handler, wait_sync in handlers: - task = asyncio.create_task(self._catch_errors(handler, data)) if force_synchronous or wait_sync: - tasks.append(task) + tasks.append(asyncio.create_task(self._catch_errors(handler, data))) + else: + background_task.create(self._catch_errors(handler, data)) return tasks async def run_internal_event( From 92e6091760c7e49498de4134be16998f11b4db82 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 6 Sep 2023 23:08:09 +0300 Subject: [PATCH 137/218] Update changelog --- CHANGELOG.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6278822a..d761b89c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,12 @@ +## v0.20.2 (unreleased) + +* *(crypto)* Changed `OlmMachine.share_keys` to make the OTK count parameter + optional. When omitted, the count is fetched from the server. +* *(appservice)* Added option to run appservice transaction event handlers + synchronously. +* *(appservice)* Added `log` and `hs_token` parameters to `AppServiceServerMixin` + to allow using it as a standalone class without extending. + ## v0.20.1 (2023-08-29) * *(util.program)* Removed `--base-config` flag in bridges, as there are no From 8ff8d07128325dc2894b9be47d8fdce548bef3fb Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 6 Sep 2023 23:18:38 +0300 Subject: [PATCH 138/218] Add appservice user/device masquerading support to base HTTPAPI --- mautrix/api.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/mautrix/api.py b/mautrix/api.py index f6c9f475..d034149b 100644 --- a/mautrix/api.py +++ b/mautrix/api.py @@ -5,7 +5,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import annotations -from typing import AsyncGenerator, ClassVar, Literal, Mapping, Union +from typing import ClassVar, Literal, Mapping from enum import Enum from json.decoder import JSONDecodeError from urllib.parse import quote as urllib_quote, urljoin as urllib_join @@ -28,7 +28,7 @@ if __optional_imports__: # Safe to import, but it's not actually needed, so don't force-import the whole types module. - from mautrix.types import JSON + from mautrix.types import JSON, DeviceID, UserID API_CALLS = Counter( name="bridge_matrix_api_calls", @@ -193,6 +193,13 @@ class HTTPAPI: default_retry_count: int """The default retry count to use if a custom value is not passed to :meth:`request`""" + as_user_id: UserID | None + """An optional user ID to set as the user_id query parameter for appservice requests.""" + as_device_id: DeviceID | None + """ + An optional device ID to set as the user_id query parameter for appservice requests (MSC3202). + """ + def __init__( self, base_url: URL | str, @@ -203,6 +210,8 @@ def __init__( txn_id: int = 0, log: TraceLogger | None = None, loop: asyncio.AbstractEventLoop | None = None, + as_user_id: UserID | None = None, + as_device_id: UserID | None = None, ) -> None: """ Args: @@ -212,6 +221,10 @@ def __init__( txn_id: The outgoing transaction ID to start with. log: The :class:`logging.Logger` instance to log requests with. default_retry_count: Default number of retries to do when encountering network errors. + as_user_id: An optional user ID to set as the user_id query parameter for + appservice requests. + as_device_id: An optional device ID to set as the user_id query parameter for + appservice requests (MSC3202). """ self.base_url = URL(base_url) self.token = token @@ -219,6 +232,8 @@ def __init__( self.session = client_session or ClientSession( loop=loop, headers={"User-Agent": self.default_ua} ) + self.as_user_id = as_user_id + self.as_device_id = as_device_id if txn_id is not None: self.txn_id = txn_id if default_retry_count is not None: @@ -360,6 +375,11 @@ async def request( query_params = query_params or {} if isinstance(query_params, dict): query_params = {k: v for k, v in query_params.items() if v is not None} + if self.as_user_id: + query_params["user_id"] = self.as_user_id + if self.as_device_id: + query_params["org.matrix.msc3202.device_id"] = self.as_device_id + query_params["device_id"] = self.as_device_id if method != Method.GET: content = content or {} From 3471a8feb8546bc144d588cbf0a63a9ada5c0191 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 9 Sep 2023 15:40:01 +0300 Subject: [PATCH 139/218] Bump version to 0.20.2 --- CHANGELOG.md | 4 +++- mautrix/__init__.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d761b89c..9599f6cc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -## v0.20.2 (unreleased) +## v0.20.2 (2023-09-09) * *(crypto)* Changed `OlmMachine.share_keys` to make the OTK count parameter optional. When omitted, the count is fetched from the server. @@ -6,6 +6,8 @@ synchronously. * *(appservice)* Added `log` and `hs_token` parameters to `AppServiceServerMixin` to allow using it as a standalone class without extending. +* *(api)* Added support for setting appservice `user_id` and `device_id` query + parameters manually without using `AppServiceAPI`. ## v0.20.1 (2023-08-29) diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 19015ee4..d52cd2af 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.20.1" +__version__ = "0.20.2" __author__ = "Tulir Asokan " __all__ = [ "api", From 193acb7660e7650487906ec95fb1fb1ed44218da Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 1 Nov 2023 01:02:50 +0200 Subject: [PATCH 140/218] Add wrapper for beeper batch send endpoint --- mautrix/appservice/api/intent.py | 55 ++++++++++++++++++++++++++++++-- mautrix/types/__init__.py | 1 + mautrix/types/misc.py | 5 +++ 3 files changed, 59 insertions(+), 2 deletions(-) diff --git a/mautrix/appservice/api/intent.py b/mautrix/appservice/api/intent.py index 1892743e..00990671 100644 --- a/mautrix/appservice/api/intent.py +++ b/mautrix/appservice/api/intent.py @@ -25,6 +25,7 @@ BatchSendEvent, BatchSendResponse, BatchSendStateEvent, + BeeperBatchSendResponse, ContentURI, EventContent, EventID, @@ -40,7 +41,6 @@ RoomNameStateEventContent, RoomPinnedEventsStateEventContent, RoomTopicStateEventContent, - SerializableAttrs, StateEventContent, UserID, ) @@ -161,6 +161,8 @@ def user( user_id: The Matrix ID of the user whose intent API to get. token: The access token to use for the Matrix ID. base_url: An optional URL to use for API requests. + as_token: Whether the provided token is actually another as_token + (meaning the ``user_id`` query parameter needs to be used). Returns: The IntentAPI for the given user. @@ -187,7 +189,7 @@ async def set_presence( Args: presence: The online status of the user. status: The status message. - ignore_cache: Whether or not to set presence even if the cache says the presence is + ignore_cache: Whether to set presence even if the cache says the presence is already set to that value. """ await self.ensure_registered() @@ -520,6 +522,9 @@ async def batch_send( .. versionadded:: v0.12.5 + .. deprecated:: v0.20.3 + MSC2716 was abandoned by upstream and Beeper has forked the endpoint. + Args: room_id: The room ID to send the events to. prev_event_id: The anchor event. The batch will be inserted immediately after this event. @@ -554,6 +559,52 @@ async def batch_send( ) return BatchSendResponse.deserialize(resp) + async def beeper_batch_send( + self, + room_id: RoomID, + events: Iterable[BatchSendEvent], + *, + forward: bool = False, + forward_if_no_messages: bool = False, + send_notification: bool = False, + mark_read_by: UserID | None = None, + ) -> BeeperBatchSendResponse: + """ + Send a batch of events into a room. Only for Beeper/hungryserv. + + .. versionadded:: v0.20.3 + + Args: + room_id: The room ID to send the events to. + events: The events to send. + forward: Send events to the end of the room instead of the beginning + forward_if_no_messages: Send events to the end of the room, but only if there are no + messages in the room. If there are messages, send the new messages to the beginning. + send_notification: Send a push notification for the new messages. + Only applies when sending to the end of the room. + mark_read_by: Send a read receipt from the given user ID atomically. + + Returns: + All the event IDs generated. + """ + body = { + "events": [evt.serialize() for evt in events], + } + if forward: + body["forward"] = forward + elif forward_if_no_messages: + body["forward_if_no_messages"] = forward_if_no_messages + if send_notification: + body["send_notification"] = send_notification + if mark_read_by: + body["mark_read_by"] = mark_read_by + resp = await self.api.request( + Method.POST, + Path.unstable["com.beeper.backfill"].rooms[room_id].batch_send, + content=body, + ) + return BeeperBatchSendResponse.deserialize(resp) + async def beeper_delete_room(self, room_id: RoomID) -> None: versions = await self.versions() if not versions.supports("com.beeper.room_yeeting"): diff --git a/mautrix/types/__init__.py b/mautrix/types/__init__.py index 6d2ffa7c..42b9068c 100644 --- a/mautrix/types/__init__.py +++ b/mautrix/types/__init__.py @@ -149,6 +149,7 @@ ) from .misc import ( BatchSendResponse, + BeeperBatchSendResponse, DeviceLists, DeviceOTKCount, DirectoryPaginationToken, diff --git a/mautrix/types/misc.py b/mautrix/types/misc.py index 7a978bd1..699fc67e 100644 --- a/mautrix/types/misc.py +++ b/mautrix/types/misc.py @@ -129,3 +129,8 @@ class BatchSendResponse(SerializableAttrs): batch_event_id: EventID next_batch_id: BatchID base_insertion_event_id: Optional[EventID] = None + + +@dataclass +class BeeperBatchSendResponse(SerializableAttrs): + event_ids: List[EventID] From 1d8c2f44588e7c5b6e79a27b7df9061e86e5ca68 Mon Sep 17 00:00:00 2001 From: Nick Mills-Barrett Date: Thu, 31 Aug 2023 09:59:38 +0100 Subject: [PATCH 141/218] Default synchronous = NORMAL for sqlite --- mautrix/util/async_db/aiosqlite.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/mautrix/util/async_db/aiosqlite.py b/mautrix/util/async_db/aiosqlite.py index 99d8fa44..18594c9c 100644 --- a/mautrix/util/async_db/aiosqlite.py +++ b/mautrix/util/async_db/aiosqlite.py @@ -117,6 +117,7 @@ def __init__( def _add_missing_pragmas(init_commands: list[str]) -> list[str]: has_foreign_keys = False has_journal_mode = False + has_synchronous = False has_busy_timeout = False for cmd in init_commands: if "PRAGMA" not in cmd: @@ -125,12 +126,16 @@ def _add_missing_pragmas(init_commands: list[str]) -> list[str]: has_foreign_keys = True elif "journal_mode" in cmd: has_journal_mode = True + elif "synchronous" in cmd: + has_synchronous = True elif "busy_timeout" in cmd: has_busy_timeout = True if not has_foreign_keys: init_commands.append("PRAGMA foreign_keys = ON") if not has_journal_mode: init_commands.append("PRAGMA journal_mode = WAL") + if not has_synchronous: + init_commands.append("PRAGMA synchronous = NORMAL") if not has_busy_timeout: init_commands.append("PRAGMA busy_timeout = 5000") return init_commands From 4c638f0b21f12a0be357293fc25d8bbd1ecb5fb3 Mon Sep 17 00:00:00 2001 From: Nick Mills-Barrett Date: Fri, 1 Sep 2023 18:53:48 +0100 Subject: [PATCH 142/218] Add safety check before applying default synchronous Co-authored-by: Tulir Asokan --- mautrix/util/async_db/aiosqlite.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/util/async_db/aiosqlite.py b/mautrix/util/async_db/aiosqlite.py index 18594c9c..80fa3cf8 100644 --- a/mautrix/util/async_db/aiosqlite.py +++ b/mautrix/util/async_db/aiosqlite.py @@ -134,7 +134,7 @@ def _add_missing_pragmas(init_commands: list[str]) -> list[str]: init_commands.append("PRAGMA foreign_keys = ON") if not has_journal_mode: init_commands.append("PRAGMA journal_mode = WAL") - if not has_synchronous: + if not has_synchronous and "PRAGMA journal_mode = WAL" in init_commands: init_commands.append("PRAGMA synchronous = NORMAL") if not has_busy_timeout: init_commands.append("PRAGMA busy_timeout = 5000") From 1e135d1a874be325cf2a271ba2c94b0055db597e Mon Sep 17 00:00:00 2001 From: Ashish Kumar Date: Wed, 8 Nov 2023 19:23:43 +0400 Subject: [PATCH 143/218] Fix `guest_can_join` field name in room directory response (#163) --- mautrix/types/misc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/types/misc.py b/mautrix/types/misc.py index 699fc67e..cf576c2b 100644 --- a/mautrix/types/misc.py +++ b/mautrix/types/misc.py @@ -87,7 +87,7 @@ class PublicRoomInfo(SerializableAttrs): num_joined_members: int world_readable: bool - guests_can_join: bool + guest_can_join: bool name: str = None topic: str = None From c29c3712ed92e69ef7d468a2baed3183f4c44196 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 10 Nov 2023 22:04:21 +0200 Subject: [PATCH 144/218] Bump version to 0.20.3 --- CHANGELOG.md | 11 +++++++++++ mautrix/__init__.py | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9599f6cc..75b535e0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,14 @@ +## v0.20.3 (2023-11-10) + +* *(client)* Deprecated MSC2716 methods and added new Beeper-specific batch + send methods, as upstream MSC2716 support has been abandoned. +* *(util.async_db)* Added `PRAGMA synchronous = NORMAL;` to default pragmas. +* *(types)* Fixed `guest_can_join` field name in room directory response + (thanks to [@ashfame] in [#163]). + +[@ashfame]: https://github.com/ashfame +[#163]: https://github.com/mautrix/python/pull/163 + ## v0.20.2 (2023-09-09) * *(crypto)* Changed `OlmMachine.share_keys` to make the OTK count parameter diff --git a/mautrix/__init__.py b/mautrix/__init__.py index d52cd2af..c5e20954 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.20.2" +__version__ = "0.20.3" __author__ = "Tulir Asokan " __all__ = [ "api", From 8d39c733d7ad2d1a11b70c255babfad6941faf3a Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 26 Nov 2023 20:12:39 +0200 Subject: [PATCH 145/218] Mark Python 3.12 as supported --- .github/workflows/python-package.yml | 6 +++--- .pre-commit-config.yaml | 4 ++-- README.rst | 2 +- setup.py | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 8c6c7606..cde25b45 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -9,7 +9,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.9", "3.10", "3.11"] + python-version: ["3.9", "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v3 @@ -50,14 +50,14 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-python@v3 with: - python-version: "3.11" + python-version: "3.12" - uses: isort/isort-action@master with: sortPaths: "./mautrix" - uses: psf/black@stable with: src: "./mautrix" - version: "23.1.0" + version: "23.11.0" - name: pre-commit run: | pip install pre-commit diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 77beb7ac..7bed3e1e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v4.5.0 hooks: - id: trailing-whitespace exclude_types: [markdown] @@ -8,7 +8,7 @@ repos: - id: check-yaml - id: check-added-large-files - repo: https://github.com/psf/black - rev: 23.1.0 + rev: 23.11.0 hooks: - id: black language_version: python3 diff --git a/README.rst b/README.rst index 75ce8f6a..e03596d9 100644 --- a/README.rst +++ b/README.rst @@ -3,7 +3,7 @@ mautrix-python |PyPI| |Python versions| |License| |Docs| |Code style| |Imports| -A Python 3.8+ asyncio Matrix framework. +A Python 3.10+ asyncio Matrix framework. Matrix room: `#maunium:maunium.net`_ diff --git a/setup.py b/setup.py index 966e9f49..e81d6c73 100644 --- a/setup.py +++ b/setup.py @@ -42,9 +42,9 @@ "Framework :: AsyncIO", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", ], package_data={ From fc5604a8187db0221a893a3593109721371b1bb0 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 8 Jan 2024 14:08:32 +0200 Subject: [PATCH 146/218] Log media requests and raise on bad status --- CHANGELOG.md | 5 +++++ mautrix/api.py | 14 +++++++++++-- .../client/api/modules/media_repository.py | 20 +++++++++++++++++-- 3 files changed, 35 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 75b535e0..e4a28618 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +## v0.20.4 (unreleased) + +* *(client)* Changed media download methods to log requests and to raise + exceptions on non-successful status codes. + ## v0.20.3 (2023-11-10) * *(client)* Deprecated MSC2716 methods and added new Beeper-specific batch diff --git a/mautrix/api.py b/mautrix/api.py index d034149b..39871bb5 100644 --- a/mautrix/api.py +++ b/mautrix/api.py @@ -275,7 +275,7 @@ def _log_request( self, method: Method, url: URL, - content: str | bytes | bytearray | AsyncBody, + content: str | bytes | bytearray | AsyncBody | None, orig_content, query_params: dict[str, str], headers: dict[str, str], @@ -314,7 +314,7 @@ def _log_request( ) def _log_request_done( - self, path: PathBuilder, req_id: int, duration: float, status: int + self, path: PathBuilder | str, req_id: int, duration: float, status: int ) -> None: level = 5 if path == Path.v3.sync else 10 duration_str = f"{duration * 1000:.1f}ms" if duration < 1 else f"{duration:.3f}s" @@ -334,6 +334,16 @@ def _full_path(self, path: PathBuilder | str) -> str: base_path += "/" return urllib_join(base_path, path) + def log_download_request(self, url: URL, query_params: dict[str, str]) -> int: + req_id = _next_global_req_id() + self._log_request(Method.GET, url, None, None, query_params, {}, req_id, False) + return req_id + + def log_download_request_done( + self, url: URL, req_id: int, duration: float, status: int + ) -> None: + self._log_request_done(url.path.removeprefix("/_matrix/media/"), req_id, duration, status) + async def request( self, method: Method, diff --git a/mautrix/client/api/modules/media_repository.py b/mautrix/client/api/modules/media_repository.py index 0d51d9f6..fe52252a 100644 --- a/mautrix/client/api/modules/media_repository.py +++ b/mautrix/client/api/modules/media_repository.py @@ -182,8 +182,16 @@ async def download_media(self, url: ContentURI, timeout_ms: int | None = None) - query_params: dict[str, Any] = {"allow_redirect": "true"} if timeout_ms is not None: query_params["timeout_ms"] = timeout_ms + req_id = self.api.log_download_request(url, query_params) + start = time.monotonic() async with self.api.session.get(url, params=query_params) as response: - return await response.read() + try: + response.raise_for_status() + return await response.read() + finally: + self.api.log_download_request_done( + url, req_id, time.monotonic() - start, response.status + ) async def download_thumbnail( self, @@ -227,8 +235,16 @@ async def download_thumbnail( query_params["allow_remote"] = allow_remote if timeout_ms is not None: query_params["timeout_ms"] = timeout_ms + req_id = self.api.log_download_request(url, query_params) + start = time.monotonic() async with self.api.session.get(url, params=query_params) as response: - return await response.read() + try: + response.raise_for_status() + return await response.read() + finally: + self.api.log_download_request_done( + url, req_id, time.monotonic() - start, response.status + ) async def get_url_preview(self, url: str, timestamp: int | None = None) -> MXOpenGraph: """ From 7a43c4203de10c1cfeb7206be69dd62108b3955d Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 8 Jan 2024 14:41:46 +0200 Subject: [PATCH 147/218] Drop Python 3.9 support --- .github/workflows/python-package.yml | 11 +++++------ CHANGELOG.md | 1 + setup.py | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index cde25b45..41ebfd8f 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -4,17 +4,16 @@ on: [push, pull_request] jobs: build: - runs-on: ubuntu-latest strategy: fail-fast: false matrix: - python-version: ["3.9", "3.10", "3.11", "3.12"] + python-version: ["3.10", "3.11", "3.12"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install libolm @@ -47,8 +46,8 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v3 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: python-version: "3.12" - uses: isort/isort-action@master diff --git a/CHANGELOG.md b/CHANGELOG.md index e4a28618..295e06aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ ## v0.20.4 (unreleased) +* Dropped Python 3.9 support. * *(client)* Changed media download methods to log requests and to raise exceptions on non-successful status codes. diff --git a/setup.py b/setup.py index e81d6c73..51e99270 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,7 @@ "encryption": encryption_dependencies, }, tests_require=test_dependencies, - python_requires="~=3.9", + python_requires="~=3.10", classifiers=[ "Development Status :: 4 - Beta", From ad6c04e48da3a1b2e065d5a9989c0fa15a7234db Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 8 Jan 2024 14:46:24 +0200 Subject: [PATCH 148/218] Install ruamel.yaml for tests --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 51e99270..b56b987f 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ from mautrix import __version__ encryption_dependencies = ["python-olm", "unpaddedbase64", "pycryptodome"] -test_dependencies = ["aiosqlite", "asyncpg", *encryption_dependencies] +test_dependencies = ["ruamel.yaml", "aiosqlite", "asyncpg", *encryption_dependencies] setuptools.setup( name="mautrix", From 450c31564ec73a27b80ae2dc29beb0d0d87578da Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 8 Jan 2024 14:48:24 +0200 Subject: [PATCH 149/218] Also install commonmark --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b56b987f..dcef751b 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,8 @@ from mautrix import __version__ encryption_dependencies = ["python-olm", "unpaddedbase64", "pycryptodome"] -test_dependencies = ["ruamel.yaml", "aiosqlite", "asyncpg", *encryption_dependencies] +bridge_dependencies = ["ruamel.yaml", "commonmark"] +test_dependencies = ["aiosqlite", "asyncpg", *encryption_dependencies, *bridge_dependencies] setuptools.setup( name="mautrix", From 68b0d71b19cb8989ef11ff482610d72ea2ae5a7d Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 8 Jan 2024 14:51:57 +0200 Subject: [PATCH 150/218] Add ignore paths for pytest --- pyproject.toml | 1 + setup.py | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c0e41313..95426e1e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,3 +11,4 @@ target-version = ["py38"] [tool.pytest.ini_options] asyncio_mode = "auto" +addopts = "--ignore mautrix/util/db/ --ignore mautrix/bridge/" diff --git a/setup.py b/setup.py index dcef751b..51e99270 100644 --- a/setup.py +++ b/setup.py @@ -3,8 +3,7 @@ from mautrix import __version__ encryption_dependencies = ["python-olm", "unpaddedbase64", "pycryptodome"] -bridge_dependencies = ["ruamel.yaml", "commonmark"] -test_dependencies = ["aiosqlite", "asyncpg", *encryption_dependencies, *bridge_dependencies] +test_dependencies = ["aiosqlite", "asyncpg", *encryption_dependencies] setuptools.setup( name="mautrix", From f7b018c501e132d69ec2b3bbe3192efc0f56688f Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 8 Jan 2024 14:54:03 +0200 Subject: [PATCH 151/218] Add ruamel.yaml for tests again --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 51e99270..3f52625b 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ from mautrix import __version__ encryption_dependencies = ["python-olm", "unpaddedbase64", "pycryptodome"] -test_dependencies = ["aiosqlite", "asyncpg", *encryption_dependencies] +test_dependencies = ["aiosqlite", "asyncpg", "ruamel.yaml", *encryption_dependencies] setuptools.setup( name="mautrix", From 61bb33ffb17cf13ae134637a700befe2416a2d73 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 9 Jan 2024 14:14:58 +0200 Subject: [PATCH 152/218] Bump version to 0.20.4 --- CHANGELOG.md | 2 +- mautrix/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 295e06aa..aad156fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -## v0.20.4 (unreleased) +## v0.20.4 (2024-01-09) * Dropped Python 3.9 support. * *(client)* Changed media download methods to log requests and to raise diff --git a/mautrix/__init__.py b/mautrix/__init__.py index c5e20954..5d4ce561 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.20.3" +__version__ = "0.20.4" __author__ = "Tulir Asokan " __all__ = [ "api", From 889996a4e543272ec7426e666db58ff5ca0931de Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 29 Jan 2024 18:48:39 +0200 Subject: [PATCH 153/218] Update Black to 2024 style and Python 3.10 target --- .github/workflows/python-package.yml | 2 +- .pre-commit-config.yaml | 4 ++-- dev-requirements.txt | 2 +- mautrix/bridge/config.py | 20 ++++++++++++-------- mautrix/client/api/filtering.py | 8 +++++--- mautrix/client/api/rooms.py | 9 ++++++--- mautrix/crypto/encrypt_megolm.py | 6 +++--- mautrix/types/event/type.pyi | 1 + mautrix/util/bridge_state.py | 7 ++++--- mautrix/util/message_send_checkpoint.py | 15 +++++++++------ mautrix/util/utf16_surrogate.py | 8 +++++--- pyproject.toml | 2 +- 12 files changed, 50 insertions(+), 34 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 41ebfd8f..5a71c970 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -56,7 +56,7 @@ jobs: - uses: psf/black@stable with: src: "./mautrix" - version: "23.11.0" + version: "24.1.1" - name: pre-commit run: | pip install pre-commit diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7bed3e1e..a056ffdb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,13 +8,13 @@ repos: - id: check-yaml - id: check-added-large-files - repo: https://github.com/psf/black - rev: 23.11.0 + rev: 24.1.1 hooks: - id: black language_version: python3 files: ^mautrix/.*\.pyi?$ - repo: https://github.com/PyCQA/isort - rev: 5.12.0 + rev: 5.13.2 hooks: - id: isort files: ^mautrix/.*\.pyi?$ diff --git a/dev-requirements.txt b/dev-requirements.txt index 5cd14c23..bb8c2a0a 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,3 +1,3 @@ pre-commit>=2.10.1,<3 isort>=5.10.1,<6 -black>=23,<24 +black>=24,<25 diff --git a/mautrix/bridge/config.py b/mautrix/bridge/config.py index 1705e914..b98ebc52 100644 --- a/mautrix/bridge/config.py +++ b/mautrix/bridge/config.py @@ -200,14 +200,18 @@ def namespaces(self) -> dict[str, list[dict[str, Any]]]: "regex": re.escape(f"@{username_format}:{homeserver}").replace(regex_ph, ".*"), } ], - "aliases": [ - { - "exclusive": True, - "regex": re.escape(f"#{alias_format}:{homeserver}").replace(regex_ph, ".*"), - } - ] - if alias_format - else [], + "aliases": ( + [ + { + "exclusive": True, + "regex": re.escape(f"#{alias_format}:{homeserver}").replace( + regex_ph, ".*" + ), + } + ] + if alias_format + else [] + ), } def generate_registration(self) -> None: diff --git a/mautrix/client/api/filtering.py b/mautrix/client/api/filtering.py index 09889ec5..b8df1f36 100644 --- a/mautrix/client/api/filtering.py +++ b/mautrix/client/api/filtering.py @@ -50,9 +50,11 @@ async def create_filter(self, filter_params: Filter) -> FilterID: resp = await self.api.request( Method.POST, Path.v3.user[self.mxid].filter, - filter_params.serialize() - if isinstance(filter_params, Serializable) - else filter_params, + ( + filter_params.serialize() + if isinstance(filter_params, Serializable) + else filter_params + ), ) try: return resp["filter_id"] diff --git a/mautrix/client/api/rooms.py b/mautrix/client/api/rooms.py index 9f935430..a488fba6 100644 --- a/mautrix/client/api/rooms.py +++ b/mautrix/client/api/rooms.py @@ -354,9 +354,12 @@ async def join_room( except KeyError: raise MatrixResponseError("`room_id` not in response.") - fill_member_event_callback: Callable[ - [RoomID, UserID, MemberStateEventContent], Awaitable[MemberStateEventContent | None] - ] | None + fill_member_event_callback: ( + Callable[ + [RoomID, UserID, MemberStateEventContent], Awaitable[MemberStateEventContent | None] + ] + | None + ) async def fill_member_event( self, room_id: RoomID, user_id: UserID, content: MemberStateEventContent diff --git a/mautrix/crypto/encrypt_megolm.py b/mautrix/crypto/encrypt_megolm.py index 227fd3fe..459bbf1f 100644 --- a/mautrix/crypto/encrypt_megolm.py +++ b/mautrix/crypto/encrypt_megolm.py @@ -95,9 +95,9 @@ async def _encrypt_megolm_event( { "room_id": room_id, "type": event_type.serialize(), - "content": content.serialize() - if isinstance(content, Serializable) - else content, + "content": ( + content.serialize() if isinstance(content, Serializable) else content + ), } ) ) diff --git a/mautrix/types/event/type.pyi b/mautrix/types/event/type.pyi index bee0fde0..22922288 100644 --- a/mautrix/types/event/type.pyi +++ b/mautrix/types/event/type.pyi @@ -18,6 +18,7 @@ class EventType(Serializable): ACCOUNT_DATA = "account_data" EPHEMERAL = "ephemeral" TO_DEVICE = "to_device" + _by_event_type: ClassVar[dict[str, EventType]] ROOM_CANONICAL_ALIAS: "EventType" diff --git a/mautrix/util/bridge_state.py b/mautrix/util/bridge_state.py index dcb42977..d28448bf 100644 --- a/mautrix/util/bridge_state.py +++ b/mautrix/util/bridge_state.py @@ -115,9 +115,10 @@ async def send(self, url: str, token: str, log: logging.Logger, log_sent: bool = self.send_attempts_ += 1 headers = {"Authorization": f"Bearer {token}", "User-Agent": HTTPAPI.default_ua} try: - async with aiohttp.ClientSession() as sess, sess.post( - url, json=self.serialize(), headers=headers - ) as resp: + async with ( + aiohttp.ClientSession() as sess, + sess.post(url, json=self.serialize(), headers=headers) as resp, + ): if not 200 <= resp.status < 300: text = await resp.text() text = text.replace("\n", "\\n") diff --git a/mautrix/util/message_send_checkpoint.py b/mautrix/util/message_send_checkpoint.py index 5dd023b5..ee0c17f3 100644 --- a/mautrix/util/message_send_checkpoint.py +++ b/mautrix/util/message_send_checkpoint.py @@ -57,12 +57,15 @@ async def send(self, endpoint: str, as_token: str, log: logging.Logger) -> None: return try: headers = {"Authorization": f"Bearer {as_token}", "User-Agent": HTTPAPI.default_ua} - async with aiohttp.ClientSession() as sess, sess.post( - endpoint, - json={"checkpoints": [self.serialize()]}, - headers=headers, - timeout=ClientTimeout(30), - ) as resp: + async with ( + aiohttp.ClientSession() as sess, + sess.post( + endpoint, + json={"checkpoints": [self.serialize()]}, + headers=headers, + timeout=ClientTimeout(30), + ) as resp, + ): if not 200 <= resp.status < 300: text = await resp.text() text = text.replace("\n", "\\n") diff --git a/mautrix/util/utf16_surrogate.py b/mautrix/util/utf16_surrogate.py index d202e15e..92dc49c2 100644 --- a/mautrix/util/utf16_surrogate.py +++ b/mautrix/util/utf16_surrogate.py @@ -16,9 +16,11 @@ def add(text: str) -> str: The text with surrogate pairs. """ return "".join( - "".join(chr(y) for y in struct.unpack(" Date: Mon, 29 Jan 2024 18:49:28 +0200 Subject: [PATCH 154/218] Update missed version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 3f52625b..b01c50a1 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ ], extras_require={ "detect_mimetype": ["python-magic>=0.4.15,<0.5"], - "lint": ["black~=23.1", "isort"], + "lint": ["black~=24.1", "isort"], "test": ["pytest", "pytest-asyncio", *test_dependencies], "encryption": encryption_dependencies, }, From 9b4198a4c667e41ca25b657c8dc3305a978bd446 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 24 Feb 2024 22:10:37 +0200 Subject: [PATCH 155/218] Fix type error in download_thumbnail --- mautrix/client/api/modules/media_repository.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mautrix/client/api/modules/media_repository.py b/mautrix/client/api/modules/media_repository.py index fe52252a..779ec814 100644 --- a/mautrix/client/api/modules/media_repository.py +++ b/mautrix/client/api/modules/media_repository.py @@ -199,7 +199,7 @@ async def download_thumbnail( width: int | None = None, height: int | None = None, resize_method: Literal["crop", "scale"] = None, - allow_remote: bool = True, + allow_remote: bool | None = None, timeout_ms: int | None = None, ): """ @@ -232,7 +232,7 @@ async def download_thumbnail( if resize_method is not None: query_params["method"] = resize_method if allow_remote is not None: - query_params["allow_remote"] = allow_remote + query_params["allow_remote"] = str(allow_remote).lower() if timeout_ms is not None: query_params["timeout_ms"] = timeout_ms req_id = self.api.log_download_request(url, query_params) From 47e2157dd43e4854ef23a087e200a557868cd794 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 24 Feb 2024 22:11:18 +0200 Subject: [PATCH 156/218] Fix checking OTK count on server when no keys have been uploaded --- mautrix/crypto/machine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/crypto/machine.py b/mautrix/crypto/machine.py index 0cfe6ea3..3dc84856 100644 --- a/mautrix/crypto/machine.py +++ b/mautrix/crypto/machine.py @@ -292,7 +292,7 @@ async def _share_keys(self, current_otk_count: int | None) -> None: ): self.log.debug("Checking OTK count on server") current_otk_count = (await self.client.upload_keys()).get( - EncryptionKeyAlgorithm.SIGNED_CURVE25519 + EncryptionKeyAlgorithm.SIGNED_CURVE25519, 0 ) device_keys = ( self.account.get_device_keys(self.client.mxid, self.client.device_id) From 8eca64ed1dfd84acb40b36f9db950eb5b3e0ebd7 Mon Sep 17 00:00:00 2001 From: Joe Groocock Date: Thu, 28 Mar 2024 08:34:17 +0000 Subject: [PATCH 157/218] Support MSC2530 fields in media event types (#170) Add `filename`, `format` and `formatted_body` to media event content types. Signed-off-by: Joe Groocock --- mautrix/types/event/message.py | 55 +++++++++++++++++----------------- 1 file changed, 28 insertions(+), 27 deletions(-) diff --git a/mautrix/types/event/message.py b/mautrix/types/event/message.py index 8d5f1bba..30b1a153 100644 --- a/mautrix/types/event/message.py +++ b/mautrix/types/event/message.py @@ -271,33 +271,6 @@ class LocationInfo(SerializableAttrs): # region Event content -@dataclass -class MediaMessageEventContent(BaseMessageEventContent, SerializableAttrs): - """The content of a media message event (m.image, m.audio, m.video, m.file)""" - - url: Optional[ContentURI] = None - info: Optional[MediaInfo] = None - file: Optional[EncryptedFile] = None - - @staticmethod - @deserializer(MediaInfo) - @deserializer(Optional[MediaInfo]) - def deserialize_info(data: JSON) -> MediaInfo: - if not isinstance(data, dict): - return Obj() - msgtype = data.pop("__mautrix_msgtype", None) - if msgtype == "m.image" or msgtype == "m.sticker": - return ImageInfo.deserialize(data) - elif msgtype == "m.video": - return VideoInfo.deserialize(data) - elif msgtype == "m.audio": - return AudioInfo.deserialize(data) - elif msgtype == "m.file": - return FileInfo.deserialize(data) - else: - return Obj(**data) - - @dataclass class LocationMessageEventContent(BaseMessageEventContent, SerializableAttrs): geo_uri: str = None @@ -364,6 +337,34 @@ def _trim_reply_fallback_html(self) -> None: self.formatted_body = html_reply_fallback_regex.sub("", self.formatted_body) +@dataclass +class MediaMessageEventContent(TextMessageEventContent, SerializableAttrs): + """The content of a media message event (m.image, m.audio, m.video, m.file)""" + + url: Optional[ContentURI] = None + info: Optional[MediaInfo] = None + file: Optional[EncryptedFile] = None + filename: Optional[str] = None + + @staticmethod + @deserializer(MediaInfo) + @deserializer(Optional[MediaInfo]) + def deserialize_info(data: JSON) -> MediaInfo: + if not isinstance(data, dict): + return Obj() + msgtype = data.pop("__mautrix_msgtype", None) + if msgtype == "m.image" or msgtype == "m.sticker": + return ImageInfo.deserialize(data) + elif msgtype == "m.video": + return VideoInfo.deserialize(data) + elif msgtype == "m.audio": + return AudioInfo.deserialize(data) + elif msgtype == "m.file": + return FileInfo.deserialize(data) + else: + return Obj(**data) + + MessageEventContent = Union[ TextMessageEventContent, MediaMessageEventContent, LocationMessageEventContent, Obj ] From 55c53e06a204c7cf2a9277456d7615b7744e1f56 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 9 Jul 2024 10:47:08 +0300 Subject: [PATCH 158/218] Don't use cached base URL for double puppeting if one is configured --- mautrix/bridge/custom_puppet.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/mautrix/bridge/custom_puppet.py b/mautrix/bridge/custom_puppet.py index 9057877c..03034d55 100644 --- a/mautrix/bridge/custom_puppet.py +++ b/mautrix/bridge/custom_puppet.py @@ -132,8 +132,13 @@ def is_real_user(self) -> bool: return bool(self.custom_mxid and self.access_token) def _fresh_intent(self) -> IntentAPI: + _, server = self.az.intent.parse_user_id(self.custom_mxid) + try: + self.base_url = self.homeserver_url_map[server] + except KeyError: + if server == self.az.domain: + self.base_url = self.az.intent.api.base_url if self.access_token == "appservice-config" and self.custom_mxid: - _, server = self.az.intent.parse_user_id(self.custom_mxid) try: secret = self.login_shared_secret_map[server] except KeyError: From 1dbdc3fe3c91479a1b6fd0b30e9ed1bb848d4184 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 9 Jul 2024 11:10:32 +0300 Subject: [PATCH 159/218] Add support for authenticated media downloads --- mautrix/api.py | 8 ++++++- mautrix/appservice/api/intent.py | 2 ++ mautrix/appservice/appservice.py | 2 +- .../client/api/modules/media_repository.py | 21 +++++++++++++++---- mautrix/types/versions.py | 4 ++++ 5 files changed, 31 insertions(+), 6 deletions(-) diff --git a/mautrix/api.py b/mautrix/api.py index 39871bb5..1adde9ec 100644 --- a/mautrix/api.py +++ b/mautrix/api.py @@ -462,6 +462,7 @@ def get_download_url( mxc_uri: str, download_type: Literal["download", "thumbnail"] = "download", file_name: str | None = None, + authenticated: bool = False, ) -> URL: """ Get the full HTTP URL to download a ``mxc://`` URI. @@ -470,6 +471,7 @@ def get_download_url( mxc_uri: The MXC URI whose full URL to get. download_type: The type of download ("download" or "thumbnail"). file_name: Optionally, a file name to include in the download URL. + authenticated: Whether to use the new authenticated download endpoint in Matrix v1.11. Returns: The full HTTP URL. @@ -485,7 +487,11 @@ def get_download_url( "https://matrix-client.matrix.org/_matrix/media/v3/download/matrix.org/pqjkOuKZ1ZKRULWXgz2IVZV6/hello.png" """ server_name, media_id = self.parse_mxc_uri(mxc_uri) - url = self.base_url / str(APIPath.MEDIA) / "v3" / download_type / server_name / media_id + if authenticated: + url = self.base_url / str(APIPath.CLIENT) / "v1" / "media" + else: + url = self.base_url / str(APIPath.MEDIA) / "v3" + url = url / download_type / server_name / media_id if file_name: url /= file_name return url diff --git a/mautrix/appservice/api/intent.py b/mautrix/appservice/api/intent.py index 00990671..6e8e9fea 100644 --- a/mautrix/appservice/api/intent.py +++ b/mautrix/appservice/api/intent.py @@ -118,6 +118,8 @@ def __init__( ) -> None: super().__init__(mxid=mxid, api=api, state_store=state_store) self.bot = bot + if bot is not None: + self.versions_cache = bot.versions_cache self.log = api.base_log.getChild("intent") for method in ENSURE_REGISTERED_METHODS: diff --git a/mautrix/appservice/appservice.py b/mautrix/appservice/appservice.py index 892db195..65e202ae 100644 --- a/mautrix/appservice/appservice.py +++ b/mautrix/appservice/appservice.py @@ -13,7 +13,7 @@ from aiohttp import web import aiohttp -from mautrix.types import JSON, RoomAlias, UserID +from mautrix.types import JSON, RoomAlias, UserID, VersionsResponse from mautrix.util.logging import TraceLogger from ..api import HTTPAPI diff --git a/mautrix/client/api/modules/media_repository.py b/mautrix/client/api/modules/media_repository.py index 779ec814..80f7eb7d 100644 --- a/mautrix/client/api/modules/media_repository.py +++ b/mautrix/client/api/modules/media_repository.py @@ -10,6 +10,8 @@ import asyncio import time +from yarl import URL + from mautrix import __optional_imports__ from mautrix.api import MediaPath, Method from mautrix.errors import MatrixResponseError, make_request_error @@ -19,6 +21,7 @@ MediaRepoConfig, MXOpenGraph, SerializerError, + SpecVersions, ) from mautrix.util import background_task from mautrix.util.async_body import async_iter_bytes @@ -178,13 +181,17 @@ async def download_media(self, url: ContentURI, timeout_ms: int | None = None) - Returns: The raw downloaded data. """ - url = self.api.get_download_url(url) + authenticated = (await self.versions()).supports(SpecVersions.V111) + url = self.api.get_download_url(url, authenticated=authenticated) query_params: dict[str, Any] = {"allow_redirect": "true"} if timeout_ms is not None: query_params["timeout_ms"] = timeout_ms + headers: dict[str, str] = {} + if authenticated: + headers["Authorization"] = f"Bearer {self.api.token}" req_id = self.api.log_download_request(url, query_params) start = time.monotonic() - async with self.api.session.get(url, params=query_params) as response: + async with self.api.session.get(url, params=query_params, headers=headers) as response: try: response.raise_for_status() return await response.read() @@ -223,7 +230,10 @@ async def download_thumbnail( Returns: The raw downloaded data. """ - url = self.api.get_download_url(url, download_type="thumbnail") + authenticated = (await self.versions()).supports(SpecVersions.V111) + url = self.api.get_download_url( + url, download_type="thumbnail", authenticated=authenticated + ) query_params: dict[str, Any] = {"allow_redirect": "true"} if width is not None: query_params["width"] = width @@ -235,9 +245,12 @@ async def download_thumbnail( query_params["allow_remote"] = str(allow_remote).lower() if timeout_ms is not None: query_params["timeout_ms"] = timeout_ms + headers: dict[str, str] = {} + if authenticated: + headers["Authorization"] = f"Bearer {self.api.token}" req_id = self.api.log_download_request(url, query_params) start = time.monotonic() - async with self.api.session.get(url, params=query_params) as response: + async with self.api.session.get(url, params=query_params, headers=headers) as response: try: response.raise_for_status() return await response.read() diff --git a/mautrix/types/versions.py b/mautrix/types/versions.py index 9ad81710..52a62f59 100644 --- a/mautrix/types/versions.py +++ b/mautrix/types/versions.py @@ -74,6 +74,10 @@ class SpecVersions: V15 = Version.deserialize("v1.5") V16 = Version.deserialize("v1.6") V17 = Version.deserialize("v1.7") + V18 = Version.deserialize("v1.8") + V19 = Version.deserialize("v1.9") + V110 = Version.deserialize("v1.10") + V111 = Version.deserialize("v1.11") @dataclass From 3e89f1f55843634a5b9cad6d12620fa37d14588f Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 9 Jul 2024 11:44:02 +0300 Subject: [PATCH 160/218] Fix server URL overriding in _fresh_intent --- mautrix/bridge/custom_puppet.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/mautrix/bridge/custom_puppet.py b/mautrix/bridge/custom_puppet.py index 03034d55..f5befd5f 100644 --- a/mautrix/bridge/custom_puppet.py +++ b/mautrix/bridge/custom_puppet.py @@ -132,12 +132,13 @@ def is_real_user(self) -> bool: return bool(self.custom_mxid and self.access_token) def _fresh_intent(self) -> IntentAPI: - _, server = self.az.intent.parse_user_id(self.custom_mxid) - try: - self.base_url = self.homeserver_url_map[server] - except KeyError: - if server == self.az.domain: - self.base_url = self.az.intent.api.base_url + if self.custom_mxid: + _, server = self.az.intent.parse_user_id(self.custom_mxid) + try: + self.base_url = self.homeserver_url_map[server] + except KeyError: + if server == self.az.domain: + self.base_url = self.az.intent.api.base_url if self.access_token == "appservice-config" and self.custom_mxid: try: secret = self.login_shared_secret_map[server] From d645e208f70fef235420643c7110ca6d8109b51a Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 9 Jul 2024 11:46:42 +0300 Subject: [PATCH 161/218] Also set user_id for authenticated downloads --- mautrix/client/api/modules/media_repository.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mautrix/client/api/modules/media_repository.py b/mautrix/client/api/modules/media_repository.py index 80f7eb7d..b1d90cc1 100644 --- a/mautrix/client/api/modules/media_repository.py +++ b/mautrix/client/api/modules/media_repository.py @@ -189,6 +189,8 @@ async def download_media(self, url: ContentURI, timeout_ms: int | None = None) - headers: dict[str, str] = {} if authenticated: headers["Authorization"] = f"Bearer {self.api.token}" + if self.api.as_user_id: + query_params["user_id"] = self.api.as_user_id req_id = self.api.log_download_request(url, query_params) start = time.monotonic() async with self.api.session.get(url, params=query_params, headers=headers) as response: @@ -248,6 +250,8 @@ async def download_thumbnail( headers: dict[str, str] = {} if authenticated: headers["Authorization"] = f"Bearer {self.api.token}" + if self.api.as_user_id: + query_params["user_id"] = self.api.as_user_id req_id = self.api.log_download_request(url, query_params) start = time.monotonic() async with self.api.session.get(url, params=query_params, headers=headers) as response: From 6d1d944e32e2cf36dd30f7402dcda39769b649c2 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 9 Jul 2024 11:46:50 +0300 Subject: [PATCH 162/218] Bump version to 0.20.5 --- CHANGELOG.md | 11 +++++++++++ mautrix/__init__.py | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aad156fd..68e077bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,14 @@ +## v0.20.5 (2024-07-09) + +**Note:** The `bridge` module is deprecated as all bridges are being rewritten +in Go. See for more info. + +* *(client)* Added support for authenticated media downloads. +* *(bridge)* Stopped using cached homeserver URLs for double puppeting if one + is set in the config file. +* *(crypto)* Fixed error when checking OTK counts before uploading new keys. +* *(types)* Added MSC2530 (captions) fields to `MediaMessageEventContent`. + ## v0.20.4 (2024-01-09) * Dropped Python 3.9 support. diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 5d4ce561..8a0aa1ca 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.20.4" +__version__ = "0.20.5" __author__ = "Tulir Asokan " __all__ = [ "api", From 6fc1c6a1ca939652d3566daf87cc110276235ff0 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 12 Jul 2024 19:48:58 +0300 Subject: [PATCH 163/218] Register if /versions fails with M_FORBIDDEN --- mautrix/bridge/matrix.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/mautrix/bridge/matrix.py b/mautrix/bridge/matrix.py index da42a95e..e5399094 100644 --- a/mautrix/bridge/matrix.py +++ b/mautrix/bridge/matrix.py @@ -225,6 +225,12 @@ async def wait_for_connection(self) -> None: try: self.versions = await self.az.intent.versions() break + except MForbidden: + self.log.debug( + "/versions endpoint returned M_FORBIDDEN, " + "trying to register bridge bot before retrying..." + ) + await self.az.intent.ensure_registered() except Exception: self.log.exception("Connection to homeserver failed, retrying in 10 seconds") await asyncio.sleep(10) From c42da3deff9be83e740c4731f4e9381870899333 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 12 Jul 2024 19:49:05 +0300 Subject: [PATCH 164/218] Bump version to 0.20.6 --- CHANGELOG.md | 4 ++++ mautrix/__init__.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 68e077bb..b02d66ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## v0.20.6 (2024-07-12) + +* *(bridge)* Added `/register` call if `/versions` fails with `M_FORBIDDEN`. + ## v0.20.5 (2024-07-09) **Note:** The `bridge` module is deprecated as all bridges are being rewritten diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 8a0aa1ca..22035a89 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.20.5" +__version__ = "0.20.6" __author__ = "Tulir Asokan " __all__ = [ "api", From 6f25b62e80616fa8a8a57d12a03caca51b3c89b4 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 17 Nov 2024 21:25:44 +0200 Subject: [PATCH 165/218] Implement MSC2781 --- mautrix/types/event/message.py | 54 +--------------------------------- 1 file changed, 1 insertion(+), 53 deletions(-) diff --git a/mautrix/types/event/message.py b/mautrix/types/event/message.py index 30b1a153..32033581 100644 --- a/mautrix/types/event/message.py +++ b/mautrix/types/event/message.py @@ -119,7 +119,7 @@ def set_thread_parent( thread_parent.content.get_thread_parent() or self.relates_to.event_id ) if not disable_reply_fallback: - self.set_reply(last_event_in_thread or thread_parent, disable_fallback=True, **kwargs) + self.set_reply(last_event_in_thread or thread_parent, **kwargs) self.relates_to.is_falling_back = True def set_edit(self, edits: Union[EventID, "MessageEvent"]) -> None: @@ -287,25 +287,6 @@ class TextMessageEventContent(BaseMessageEventContent, SerializableAttrs): format: Format = None formatted_body: str = None - def set_reply( - self, - reply_to: Union["MessageEvent", EventID], - *, - displayname: Optional[str] = None, - disable_fallback: bool = False, - ) -> None: - super().set_reply(reply_to) - if isinstance(reply_to, str): - return - if isinstance(reply_to, MessageEvent) and not disable_fallback: - self.ensure_has_html() - if isinstance(reply_to.content, TextMessageEventContent): - reply_to.content.trim_reply_fallback() - self.formatted_body = ( - reply_to.make_reply_fallback_html(displayname) + self.formatted_body - ) - self.body = reply_to.make_reply_fallback_text(displayname) + self.body - def ensure_has_html(self) -> None: if not self.formatted_body or self.format != Format.HTML: self.format = Format.HTML @@ -424,36 +405,3 @@ def deserialize_content(data: JSON) -> MessageEventContent: return LocationMessageEventContent.deserialize(data) else: return Obj(**data) - - def make_reply_fallback_html(self, displayname: Optional[str] = None) -> str: - """Generate the HTML fallback for messages replying to this event.""" - if self.content.msgtype.is_text: - body = self.content.formatted_body or escape(self.content.body).replace("\n", "
") - else: - sent_type = media_reply_fallback_body_map[self.content.msgtype] or "a message" - body = f"sent {sent_type}" - displayname = escape(displayname) if displayname else self.sender - return html_reply_fallback_format.format( - room_id=self.room_id, - event_id=self.event_id, - sender=self.sender, - displayname=displayname, - content=body, - ) - - def make_reply_fallback_text(self, displayname: Optional[str] = None) -> str: - """Generate the plaintext fallback for messages replying to this event.""" - if self.content.msgtype.is_text: - body = self.content.body - else: - try: - body = media_reply_fallback_body_map[self.content.msgtype] - except KeyError: - body = "an unknown message type" - lines = body.strip().split("\n") - first_line, lines = lines[0], lines[1:] - fallback_text = f"> <{displayname or self.sender}> {first_line}" - for line in lines: - fallback_text += f"\n> {line}" - fallback_text += "\n\n" - return fallback_text From 7f2c0335cbc4a6ac9de6219b5ae552378db77660 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 3 Jan 2025 12:23:40 +0200 Subject: [PATCH 166/218] Update linters --- .github/workflows/python-package.yml | 6 +++--- .pre-commit-config.yaml | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 5a71c970..aef7eb74 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -8,7 +8,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.10", "3.11", "3.12"] + python-version: ["3.10", "3.11", "3.12", "3.13"] steps: - uses: actions/checkout@v4 @@ -49,14 +49,14 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: - python-version: "3.12" + python-version: "3.13" - uses: isort/isort-action@master with: sortPaths: "./mautrix" - uses: psf/black@stable with: src: "./mautrix" - version: "24.1.1" + version: "24.10.0" - name: pre-commit run: | pip install pre-commit diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a056ffdb..72cce792 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.5.0 + rev: v5.0.0 hooks: - id: trailing-whitespace exclude_types: [markdown] @@ -8,7 +8,7 @@ repos: - id: check-yaml - id: check-added-large-files - repo: https://github.com/psf/black - rev: 24.1.1 + rev: 24.10.0 hooks: - id: black language_version: python3 From c84730e2a4ea7f8a085924ed42d6e64de7d88246 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 3 Jan 2025 12:22:10 +0200 Subject: [PATCH 167/218] Bump version to 0.20.7 --- CHANGELOG.md | 7 +++++++ mautrix/__init__.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b02d66ad..08a4577d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## v0.20.7 (2025-01-03) + +* *(types)* Removed support for generating reply fallbacks to implement + [MSC2781]. Stripping fallbacks is still supported. + +[MSC2781]: https://github.com/matrix-org/matrix-spec-proposals/pull/2781 + ## v0.20.6 (2024-07-12) * *(bridge)* Added `/register` call if `/versions` fails with `M_FORBIDDEN`. diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 22035a89..d50477f4 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.20.6" +__version__ = "0.20.7" __author__ = "Tulir Asokan " __all__ = [ "api", From bba5542d8f11d1b9bbfc4e66a1ae3f37a1edbfb3 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 3 Jan 2025 12:31:48 +0200 Subject: [PATCH 168/218] Add Python 3.13 to classifiers --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index b01c50a1..d5c58eb0 100644 --- a/setup.py +++ b/setup.py @@ -45,6 +45,7 @@ "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", ], package_data={ From 33a034a95260286c1ce75cc7c381d997d1a2bb1f Mon Sep 17 00:00:00 2001 From: Marco Antonio Alvarez Date: Wed, 15 Jan 2025 17:19:02 +0100 Subject: [PATCH 169/218] Add support for MSC4190 (#175) --- mautrix/bridge/config.py | 4 ++++ mautrix/bridge/e2ee.py | 34 +++++++++++++++++++--------- mautrix/client/api/authentication.py | 15 ++++++++++++ 3 files changed, 42 insertions(+), 11 deletions(-) diff --git a/mautrix/bridge/config.py b/mautrix/bridge/config.py index b98ebc52..6127d5db 100644 --- a/mautrix/bridge/config.py +++ b/mautrix/bridge/config.py @@ -143,6 +143,7 @@ def do_update(self, helper: ConfigUpdateHelper) -> None: copy("bridge.encryption.default") copy("bridge.encryption.require") copy("bridge.encryption.appservice") + copy("bridge.encryption.msc4190") copy("bridge.encryption.delete_keys.delete_outbound_on_ack") copy("bridge.encryption.delete_keys.dont_store_outbound") copy("bridge.encryption.delete_keys.ratchet_on_decrypt") @@ -241,3 +242,6 @@ def generate_registration(self) -> None: if self["appservice.ephemeral_events"]: self._registration["de.sorunome.msc2409.push_ephemeral"] = True self._registration["push_ephemeral"] = True + + if self["bridge.encryption.msc4190"]: + self._registration["io.element.msc4190"] = True diff --git a/mautrix/bridge/e2ee.py b/mautrix/bridge/e2ee.py index 7ae66abf..266c8db9 100644 --- a/mautrix/bridge/e2ee.py +++ b/mautrix/bridge/e2ee.py @@ -57,6 +57,7 @@ class EncryptionManager: appservice_mode: bool periodically_delete_expired_keys: bool delete_outdated_inbound: bool + msc4190: bool bridge: br.Bridge az: AppService @@ -108,6 +109,7 @@ def __init__( self.crypto.send_keys_min_trust = TrustState.parse(verification_levels["receive"]) self.key_sharing_enabled = bridge.config["bridge.encryption.allow_key_sharing"] self.appservice_mode = bridge.config["bridge.encryption.appservice"] + self.msc4190 = bridge.config["bridge.encryption.msc4190"] if self.appservice_mode: self.az.otk_handler = self.crypto.handle_as_otk_counts self.az.device_list_handler = self.crypto.handle_as_device_lists @@ -246,7 +248,7 @@ async def decrypt(self, evt: EncryptedEvent, wait_session_timeout: int = 5) -> M async def start(self) -> None: flows = await self.client.get_login_flows() - if not flows.supports_type(LoginType.APPSERVICE): + if not self.msc4190 and not flows.supports_type(LoginType.APPSERVICE): self.log.critical( "Encryption enabled in config, but homeserver does not support appservice login" ) @@ -261,16 +263,26 @@ async def start(self) -> None: device_id = await self.crypto_store.get_device_id() if device_id: self.log.debug(f"Found device ID in database: {device_id}") - # We set the API token to the AS token here to authenticate the appservice login - # It'll get overridden after the login - self.client.api.token = self.az.as_token - await self.client.login( - login_type=LoginType.APPSERVICE, - device_name=self.device_name, - device_id=device_id, - store_access_token=True, - update_hs_url=False, - ) + + if self.msc4190: + if not device_id: + self.log.debug("Creating bot device with MSC4190") + self.client.api.token = self.az.as_token + await self.client.create_device_msc4190( + device_id=device_id, initial_display_name=self.device_name + ) + else: + # We set the API token to the AS token here to authenticate the appservice login + # It'll get overridden after the login + self.client.api.token = self.az.as_token + await self.client.login( + login_type=LoginType.APPSERVICE, + device_name=self.device_name, + device_id=device_id, + store_access_token=True, + update_hs_url=False, + ) + await self.crypto.load() if not device_id: await self.crypto_store.put_device_id(self.client.device_id) diff --git a/mautrix/client/api/authentication.py b/mautrix/client/api/authentication.py index cd77272d..0f6249ea 100644 --- a/mautrix/client/api/authentication.py +++ b/mautrix/client/api/authentication.py @@ -5,6 +5,8 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import annotations +import secrets + from mautrix.api import Method, Path from mautrix.errors import MatrixResponseError from mautrix.types import ( @@ -117,6 +119,19 @@ async def login( self.api.base_url = base_url.rstrip("/") return resp_data + async def create_device_msc4190(self, device_id: str, initial_display_name: str) -> None: + """ + Create a Device for a user of the homeserver using appservice interface defined in MSC4190 + """ + if len(device_id) == 0: + device_id = DeviceID(secrets.token_urlsafe(10)) + self.api.as_user_id = self.mxid + await self.api.request( + Method.PUT, Path.v3.devices[device_id], {"display_name": initial_display_name} + ) + self.api.as_device_id = device_id + self.device_id = device_id + async def logout(self, clear_access_token: bool = True) -> None: """ Invalidates an existing access token, so that it can no longer be used for authorization. From 626fcada6d82ee163afd888808b634df1b81eb2e Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 15 Jan 2025 18:02:12 +0200 Subject: [PATCH 170/218] Update push_ephemeral -> receive_ephemeral --- mautrix/bridge/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/bridge/config.py b/mautrix/bridge/config.py index 6127d5db..46fb88d2 100644 --- a/mautrix/bridge/config.py +++ b/mautrix/bridge/config.py @@ -241,7 +241,7 @@ def generate_registration(self) -> None: if self["appservice.ephemeral_events"]: self._registration["de.sorunome.msc2409.push_ephemeral"] = True - self._registration["push_ephemeral"] = True + self._registration["receive_ephemeral"] = True if self["bridge.encryption.msc4190"]: self._registration["io.element.msc4190"] = True From ca41430f91c545629a540cf05f57d82d987c066f Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 15 Jan 2025 18:52:44 +0200 Subject: [PATCH 171/218] Update changelog --- CHANGELOG.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 08a4577d..56f25916 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,13 @@ +## v0.20.8 (unreleased) + +* *(bridge)* Added support for [MSC4190] (thanks to [@surakin] in [#175]). +* *(appservice)* Renamed `push_ephemeral` in generated registrations to + `receive_ephemeral` to match the accepted version of [MSC2409]. + +[MSC4190]: https://github.com/matrix-org/matrix-spec-proposals/pull/2781 +[@surakin]: https://github.com/surakin +[#175]: https://github.com/mautrix/python/pull/175 + ## v0.20.7 (2025-01-03) * *(types)* Removed support for generating reply fallbacks to implement From 8eac9db01e2b5fd9a30620bcbc8ebbaa36c71ecb Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 15 Jan 2025 18:54:14 +0200 Subject: [PATCH 172/218] Bump version to 0.20.8b1 --- mautrix/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/__init__.py b/mautrix/__init__.py index d50477f4..91d7ec3c 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.20.7" +__version__ = "0.20.8b1" __author__ = "Tulir Asokan " __all__ = [ "api", From 4e3220c891c257caa4366283f1b48a47496a221e Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 1 Jun 2025 19:15:51 +0300 Subject: [PATCH 173/218] Start event loop in prepare step --- mautrix/util/program.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/mautrix/util/program.py b/mautrix/util/program.py index 2fb6b141..aa37b374 100644 --- a/mautrix/util/program.py +++ b/mautrix/util/program.py @@ -98,7 +98,7 @@ def _prepare(self) -> None: self.log.info(f"Initializing {self.name} {self.version}") try: - self.prepare() + self.loop.run_until_complete(self._async_prepare()) except Exception: self.log.critical("Unexpected error in initialization", exc_info=True) sys.exit(1) @@ -117,6 +117,7 @@ def preinit(self) -> None: self.prepare_config() self.prepare_log() self.check_config() + self.init_loop() @property def base_config_path(self) -> str: @@ -168,14 +169,16 @@ def prepare_log(self) -> None: logging.config.dictConfig(copy.deepcopy(self.config["logging"])) self.log = cast(TraceLogger, logging.getLogger("mau.init")) + async def _async_prepare(self) -> None: + self.prepare() + def prepare(self) -> None: """ Lifecycle method where the primary program initialization happens. Use this to fill startup_actions with async startup tasks. """ - self.prepare_loop() - def prepare_loop(self) -> None: + def init_loop(self) -> None: """Init lifecycle method where the asyncio event loop is created.""" if uvloop is not None: uvloop.install() From dbf642555de16374108ba3745fdbe3bbabfa0a3f Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 1 Jun 2025 19:21:22 +0300 Subject: [PATCH 174/218] Bump version to 0.20.8 --- CHANGELOG.md | 3 ++- mautrix/__init__.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 56f25916..8769162e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,9 @@ -## v0.20.8 (unreleased) +## v0.20.8 (2025-06-01) * *(bridge)* Added support for [MSC4190] (thanks to [@surakin] in [#175]). * *(appservice)* Renamed `push_ephemeral` in generated registrations to `receive_ephemeral` to match the accepted version of [MSC2409]. +* *(bridge)* Fixed compatibility with breaking change in aiohttp 3.12.6. [MSC4190]: https://github.com/matrix-org/matrix-spec-proposals/pull/2781 [@surakin]: https://github.com/surakin diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 91d7ec3c..69c8862a 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.20.8b1" +__version__ = "0.20.8" __author__ = "Tulir Asokan " __all__ = [ "api", From 0349445bd6992ac8f294582e85c3f61ce5c863b3 Mon Sep 17 00:00:00 2001 From: Christopher Johnstone Date: Mon, 11 Aug 2025 13:07:44 -0400 Subject: [PATCH 175/218] Check login flows only if MSC4190 is not enabled (#178) Currently, the homeserver login flows are checked even if MSC4190 is enabled. However, the `flows` variable is unused when MSC4190 is enabled. This is an unnecessary network call, and also e.g. requires a reverse proxy soley for this purpose if bridges would otherwise directly connect to a homeserver that is delegating OIDC authentication. Closes #177. --- mautrix/bridge/e2ee.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/mautrix/bridge/e2ee.py b/mautrix/bridge/e2ee.py index 266c8db9..56270942 100644 --- a/mautrix/bridge/e2ee.py +++ b/mautrix/bridge/e2ee.py @@ -247,12 +247,13 @@ async def decrypt(self, evt: EncryptedEvent, wait_session_timeout: int = 5) -> M return decrypted async def start(self) -> None: - flows = await self.client.get_login_flows() - if not self.msc4190 and not flows.supports_type(LoginType.APPSERVICE): - self.log.critical( - "Encryption enabled in config, but homeserver does not support appservice login" - ) - sys.exit(30) + if not self.msc4190: + flows = await self.client.get_login_flows() + if not flows.supports_type(LoginType.APPSERVICE): + self.log.critical( + "Encryption enabled in config, but homeserver does not support appservice login" + ) + sys.exit(30) self.log.debug("Logging in with bridge bot user") if self.crypto_db: try: From c95603784baa0f1c3036d447ed1fcda6fe5ac64f Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 11 Jul 2025 14:18:07 +0300 Subject: [PATCH 176/218] Add support for creator power --- mautrix/appservice/api/intent.py | 2 + mautrix/client/api/events.py | 46 +++++++++++++++++-- mautrix/client/state_store/abstract.py | 21 ++++++++- mautrix/client/state_store/asyncpg/store.py | 24 ++++++++++ mautrix/client/state_store/asyncpg/upgrade.py | 12 +++-- mautrix/client/state_store/file.py | 5 ++ mautrix/client/state_store/memory.py | 19 ++++++++ mautrix/client/store_updater.py | 35 ++++++++------ mautrix/types/event/state.py | 43 ++++++++++++++++- 9 files changed, 183 insertions(+), 24 deletions(-) diff --git a/mautrix/appservice/api/intent.py b/mautrix/appservice/api/intent.py index 6e8e9fea..626b34f1 100644 --- a/mautrix/appservice/api/intent.py +++ b/mautrix/appservice/api/intent.py @@ -710,6 +710,8 @@ async def _ensure_has_power_level_for( if not await self.state_store.has_power_levels_cached(room_id): # TODO add option to not try to fetch power levels from server await self.get_power_levels(room_id, ignore_cache=True, ensure_joined=False) + if not await self.state_store.has_create_cached(room_id): + await self.get_state_event(room_id, EventType.ROOM_CREATE, format="event") if not await self.state_store.has_power_level(room_id, self.mxid, event_type): # TODO implement something better raise IntentError( diff --git a/mautrix/client/api/events.py b/mautrix/client/api/events.py index 99e1c178..fd6b2c9d 100644 --- a/mautrix/client/api/events.py +++ b/mautrix/client/api/events.py @@ -5,7 +5,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import annotations -from typing import Awaitable +from typing import Awaitable, Literal, overload import json from mautrix.api import Method, Path @@ -168,12 +168,41 @@ async def get_event_context( ) return EventContext.deserialize(resp) + @overload async def get_state_event( self, room_id: RoomID, event_type: EventType, state_key: str = "", - ) -> StateEventContent: + *, + format: Literal["content"] = "content", + ) -> StateEventContent: ... + @overload + async def get_state_event( + self, + room_id: RoomID, + event_type: EventType, + state_key: str = "", + *, + format: Literal["event"], + ) -> StateEvent: ... + @overload + async def get_state_event( + self, + room_id: RoomID, + event_type: EventType, + state_key: str = "", + *, + format: str = "content", + ) -> StateEventContent | StateEvent: ... + async def get_state_event( + self, + room_id: RoomID, + event_type: EventType, + state_key: str = "", + *, + format: str = "content", + ) -> StateEventContent | StateEvent: """ Looks up the contents of a state event in a room. If the user is joined to the room then the state is taken from the current state of the room. If the user has left the room then the @@ -185,6 +214,9 @@ async def get_state_event( room_id: The ID of the room to look up the state in. event_type: The type of state to look up. state_key: The key of the state to look up. Defaults to empty string. + format: The format of the state event to return. Defaults to "content", which only returns + the content of the state event. If set to "event", the full event is returned. + See https://github.com/matrix-org/matrix-spec/issues/1047 for more info. Returns: The state event. @@ -192,11 +224,17 @@ async def get_state_event( content = await self.api.request( Method.GET, Path.v3.rooms[room_id].state[event_type][state_key], + query_params={"format": format} if format != "content" else None, metrics_method="getStateEvent", ) - content["__mautrix_event_type"] = event_type try: - return StateEvent.deserialize_content(content) + if format == "content": + content["__mautrix_event_type"] = event_type + return StateEvent.deserialize_content(content) + elif format == "event": + return StateEvent.deserialize(content) + else: + return content except SerializerError as e: raise MatrixResponseError("Invalid state event in response") from e diff --git a/mautrix/client/state_store/abstract.py b/mautrix/client/state_store/abstract.py index 3d37087d..d5b1f5f2 100644 --- a/mautrix/client/state_store/abstract.py +++ b/mautrix/client/state_store/abstract.py @@ -121,6 +121,18 @@ async def set_power_levels( ) -> None: pass + @abstractmethod + async def has_create_cached(self, room_id: RoomID) -> bool: + pass + + @abstractmethod + async def get_create(self, room_id: RoomID) -> StateEvent | None: + pass + + @abstractmethod + async def set_create(self, event: StateEvent) -> None: + pass + @abstractmethod async def has_encryption_info_cached(self, room_id: RoomID) -> bool: pass @@ -135,7 +147,7 @@ async def get_encryption_info(self, room_id: RoomID) -> RoomEncryptionStateEvent @abstractmethod async def set_encryption_info( - self, room_id: RoomID, content: RoomEncryptionStateEventContent | dict[str, any] + self, room_id: RoomID, content: RoomEncryptionStateEventContent | dict[str, Any] ) -> None: pass @@ -149,6 +161,8 @@ async def update_state(self, evt: StateEvent) -> None: await self.set_member(evt.room_id, UserID(evt.state_key), evt.content) elif evt.type == EventType.ROOM_ENCRYPTION: await self.set_encryption_info(evt.room_id, evt.content) + elif evt.type == EventType.ROOM_CREATE and evt.sender: + await self.set_create(evt) async def get_membership(self, room_id: RoomID, user_id: UserID) -> Membership: member = await self.get_member(room_id, user_id) @@ -172,4 +186,7 @@ async def has_power_level( room_levels = await self.get_power_levels(room_id) if not room_levels: return None - return room_levels.get_user_level(user_id) >= room_levels.get_event_level(event_type) + create_event = await self.get_create(room_id) + return room_levels.get_user_level(user_id, create_event) >= room_levels.get_event_level( + event_type + ) diff --git a/mautrix/client/state_store/asyncpg/store.py b/mautrix/client/state_store/asyncpg/store.py index d78c04d6..f4f8436f 100644 --- a/mautrix/client/state_store/asyncpg/store.py +++ b/mautrix/client/state_store/asyncpg/store.py @@ -16,6 +16,7 @@ RoomEncryptionStateEventContent, RoomID, Serializable, + StateEvent, UserID, ) from mautrix.util.async_db import Database, Scheme @@ -223,6 +224,29 @@ async def set_power_levels( json.dumps(content.serialize() if isinstance(content, Serializable) else content), ) + async def has_create_cached(self, room_id: RoomID) -> bool: + return bool( + await self.db.fetchval( + "SELECT create_event IS NOT NULL FROM mx_room_state WHERE room_id=$1", room_id + ) + ) + + async def get_create(self, room_id: RoomID) -> StateEvent | None: + create_event_json = await self.db.fetchval( + "SELECT create_event FROM mx_room_state WHERE room_id=$1", room_id + ) + if create_event_json is None: + return None + return StateEvent.parse_json(create_event_json) + + async def set_create(self, event: StateEvent) -> None: + await self.db.execute( + "INSERT INTO mx_room_state (room_id, create_event) VALUES ($1, $2) " + "ON CONFLICT (room_id) DO UPDATE SET create_event=$2", + event.room_id, + json.dumps(event.serialize() if isinstance(event, Serializable) else event), + ) + async def has_encryption_info_cached(self, room_id: RoomID) -> bool: return bool( await self.db.fetchval( diff --git a/mautrix/client/state_store/asyncpg/upgrade.py b/mautrix/client/state_store/asyncpg/upgrade.py index 88f115f2..0d722709 100644 --- a/mautrix/client/state_store/asyncpg/upgrade.py +++ b/mautrix/client/state_store/asyncpg/upgrade.py @@ -14,15 +14,16 @@ ) -@upgrade_table.register(description="Latest revision", upgrades_to=3) -async def upgrade_blank_to_v3(conn: Connection, scheme: Scheme) -> None: +@upgrade_table.register(description="Latest revision", upgrades_to=4) +async def upgrade_blank_to_v4(conn: Connection, scheme: Scheme) -> None: await conn.execute( """CREATE TABLE mx_room_state ( room_id TEXT PRIMARY KEY, is_encrypted BOOLEAN, has_full_member_list BOOLEAN, encryption TEXT, - power_levels TEXT + power_levels TEXT, + create_event TEXT, )""" ) membership_check = "" @@ -69,3 +70,8 @@ async def upgrade_v3(conn: Connection) -> None: WHERE mx_room_state.encryption IS NULL """ ) + + +@upgrade_table.register(description="Add create event to room state cache") +async def upgrade_v4(conn: Connection) -> None: + await conn.execute("ALTER TABLE mx_room_state ADD COLUMN create_event TYPE TEXT") diff --git a/mautrix/client/state_store/file.py b/mautrix/client/state_store/file.py index d567c853..a5d53663 100644 --- a/mautrix/client/state_store/file.py +++ b/mautrix/client/state_store/file.py @@ -15,6 +15,7 @@ PowerLevelStateEventContent, RoomEncryptionStateEventContent, RoomID, + StateEvent, UserID, ) from mautrix.util.file_store import Filer, FileStore @@ -65,3 +66,7 @@ async def set_power_levels( ) -> None: await super().set_power_levels(room_id, content) self._time_limited_flush() + + async def set_create(self, event: StateEvent) -> None: + await super().set_create(event) + self._time_limited_flush() diff --git a/mautrix/client/state_store/memory.py b/mautrix/client/state_store/memory.py index 8f2edac5..2b010a75 100644 --- a/mautrix/client/state_store/memory.py +++ b/mautrix/client/state_store/memory.py @@ -14,6 +14,7 @@ PowerLevelStateEventContent, RoomEncryptionStateEventContent, RoomID, + StateEvent, UserID, ) @@ -25,6 +26,7 @@ class SerializedStateStore(TypedDict): full_member_list: dict[RoomID, bool] power_levels: dict[RoomID, Any] encryption: dict[RoomID, Any] + create: dict[RoomID, Any] class MemoryStateStore(StateStore): @@ -32,12 +34,14 @@ class MemoryStateStore(StateStore): full_member_list: dict[RoomID, bool] power_levels: dict[RoomID, PowerLevelStateEventContent] encryption: dict[RoomID, RoomEncryptionStateEventContent | None] + create: dict[RoomID, StateEvent] def __init__(self) -> None: self.members = {} self.full_member_list = {} self.power_levels = {} self.encryption = {} + self.create = {} def serialize(self) -> SerializedStateStore: """ @@ -58,6 +62,7 @@ def serialize(self) -> SerializedStateStore: room_id: (content.serialize() if content is not None else None) for room_id, content in self.encryption.items() }, + "create": {room_id: evt.serialize() for room_id, evt in self.create.items()}, } def deserialize(self, data: SerializedStateStore) -> None: @@ -84,6 +89,9 @@ def deserialize(self, data: SerializedStateStore) -> None: ) for room_id, content in data["encryption"].items() } + self.create = { + room_id: StateEvent.deserialize(evt) for room_id, evt in data["create"].items() + } async def get_member(self, room_id: RoomID, user_id: UserID) -> Member | None: try: @@ -176,6 +184,17 @@ async def set_power_levels( content = PowerLevelStateEventContent.deserialize(content) self.power_levels[room_id] = content + async def has_create_cached(self, room_id: RoomID) -> bool: + return room_id in self.create + + async def get_create(self, room_id: RoomID) -> StateEvent | None: + return self.create.get(room_id) + + async def set_create(self, event: StateEvent | dict[str, Any]) -> None: + if not isinstance(event, StateEvent): + event = StateEvent.deserialize(event) + self.create[event.room_id] = event + async def has_encryption_info_cached(self, room_id: RoomID) -> bool: return room_id in self.encryption diff --git a/mautrix/client/store_updater.py b/mautrix/client/store_updater.py index a5530bde..35280324 100644 --- a/mautrix/client/store_updater.py +++ b/mautrix/client/store_updater.py @@ -5,6 +5,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import annotations +from typing import Literal import asyncio from mautrix.errors import MForbidden, MNotFound @@ -196,20 +197,28 @@ async def send_state_event( return event_id async def get_state_event( - self, room_id: RoomID, event_type: EventType, state_key: str = "" - ) -> StateEventContent: - event = await super().get_state_event(room_id, event_type, state_key) + self, + room_id: RoomID, + event_type: EventType, + state_key: str = "", + *, + format: str = "content", + ) -> StateEventContent | StateEvent: + event = await super().get_state_event(room_id, event_type, state_key, format=format) if self.state_store: - fake_event = StateEvent( - type=event_type, - room_id=room_id, - event_id=EventID(""), - sender=UserID(""), - state_key=state_key, - timestamp=0, - content=event, - ) - await self.state_store.update_state(fake_event) + if isinstance(event, StateEvent): + await self.state_store.update_state(event) + else: + fake_event = StateEvent( + type=event_type, + room_id=room_id, + event_id=EventID(""), + sender=UserID(""), + state_key=state_key, + timestamp=0, + content=event, + ) + await self.state_store.update_state(fake_event) return event async def get_joined_members(self, room_id: RoomID) -> dict[UserID, Member]: diff --git a/mautrix/types/event/state.py b/mautrix/types/event/state.py index c6b351c6..ef51cff4 100644 --- a/mautrix/types/event/state.py +++ b/mautrix/types/event/state.py @@ -41,7 +41,19 @@ class PowerLevelStateEventContent(SerializableAttrs): ban: int = 50 redact: int = 50 - def get_user_level(self, user_id: UserID) -> int: + def get_user_level( + self, + user_id: UserID, + create: Optional["StateEvent"] = None, + ) -> int: + if ( + create + and create.content.supports_creator_power + and (user_id == create.sender or user_id in (create.content.additional_creators or [])) + ): + # This is really meant to be infinity, but involving floats would be annoying, + # so we use an integer larger than the maximum power level (2^53-1) instead. + return 2**60 - 1 return int(self.users.get(user_id, self.users_default)) def set_user_level(self, user_id: UserID, level: int) -> None: @@ -50,7 +62,16 @@ def set_user_level(self, user_id: UserID, level: int) -> None: else: self.users[user_id] = level - def ensure_user_level(self, user_id: UserID, level: int) -> bool: + def ensure_user_level( + self, user_id: UserID, level: int, create: Optional["StateEvent"] = None + ) -> bool: + if ( + create + and create.content.supports_creator_power + and (user_id == create.sender or user_id in (create.content.additional_creators or [])) + ): + # Don't try to set creator power levels + return False if self.get_user_level(user_id) != level: self.set_user_level(user_id, level) return True @@ -193,6 +214,24 @@ class RoomCreateStateEventContent(SerializableAttrs): federate: bool = field(json="m.federate", omit_default=True, default=True) predecessor: Optional[RoomPredecessor] = None type: Optional[RoomType] = None + additional_creators: Optional[List[UserID]] = None + + @property + def supports_creator_power(self) -> bool: + return self.room_version not in ( + "", + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10", + "11", + ) @dataclass From 3b1690bd626b40219bfbdfdc63602a7b08e6a481 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 11 Aug 2025 22:27:42 +0300 Subject: [PATCH 177/218] Bump version to 0.20.9rc1 --- CHANGELOG.md | 9 +++++++++ mautrix/__init__.py | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8769162e..6bbb268b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,12 @@ +## v0.20.9 (unreleased) + +* *(event)* Added support for creator power in room v12+. +* *(bridge)* Removed check for login flows when using MSC4190 + (thanks to [@meson800] in [#178]). + +[@meson800]: https://github.com/meson800 +[#178]: https://github.com/mautrix/python/pull/178 + ## v0.20.8 (2025-06-01) * *(bridge)* Added support for [MSC4190] (thanks to [@surakin] in [#175]). diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 69c8862a..44635a05 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.20.8" +__version__ = "0.20.9rc1" __author__ = "Tulir Asokan " __all__ = [ "api", From 6acd0bebc8dd17a565bc7f1e95f0091e4eeb6cc0 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 11 Aug 2025 22:49:22 +0300 Subject: [PATCH 178/218] Fix typo in sql store upgrade --- mautrix/__init__.py | 2 +- mautrix/client/state_store/asyncpg/upgrade.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 44635a05..deb5af97 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.20.9rc1" +__version__ = "0.20.9rc2" __author__ = "Tulir Asokan " __all__ = [ "api", diff --git a/mautrix/client/state_store/asyncpg/upgrade.py b/mautrix/client/state_store/asyncpg/upgrade.py index 0d722709..72bc6d68 100644 --- a/mautrix/client/state_store/asyncpg/upgrade.py +++ b/mautrix/client/state_store/asyncpg/upgrade.py @@ -23,7 +23,7 @@ async def upgrade_blank_to_v4(conn: Connection, scheme: Scheme) -> None: has_full_member_list BOOLEAN, encryption TEXT, power_levels TEXT, - create_event TEXT, + create_event TEXT )""" ) membership_check = "" From a8c0ad51339e0d68d70565cba63150c26e18cc74 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 12 Aug 2025 10:09:30 +0300 Subject: [PATCH 179/218] Fix another typo in sql store upgrade --- mautrix/__init__.py | 2 +- mautrix/client/state_store/asyncpg/upgrade.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mautrix/__init__.py b/mautrix/__init__.py index deb5af97..beb96400 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.20.9rc2" +__version__ = "0.20.9rc3" __author__ = "Tulir Asokan " __all__ = [ "api", diff --git a/mautrix/client/state_store/asyncpg/upgrade.py b/mautrix/client/state_store/asyncpg/upgrade.py index 72bc6d68..c1f32664 100644 --- a/mautrix/client/state_store/asyncpg/upgrade.py +++ b/mautrix/client/state_store/asyncpg/upgrade.py @@ -74,4 +74,4 @@ async def upgrade_v3(conn: Connection) -> None: @upgrade_table.register(description="Add create event to room state cache") async def upgrade_v4(conn: Connection) -> None: - await conn.execute("ALTER TABLE mx_room_state ADD COLUMN create_event TYPE TEXT") + await conn.execute("ALTER TABLE mx_room_state ADD COLUMN create_event TEXT") From 970f371b033b39fd00135792fa3b190fe43d4e27 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 13 Aug 2025 22:14:11 +0300 Subject: [PATCH 180/218] Update link to C-S spec --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index e03596d9..f1342c19 100644 --- a/README.rst +++ b/README.rst @@ -49,7 +49,7 @@ Components .. _#maunium:maunium.net: https://matrix.to/#/#maunium:maunium.net .. _python-appservice-framework: https://github.com/Cadair/python-appservice-framework/ -.. _Client API: https://matrix.org/docs/spec/client_server/r0.6.1.html +.. _Client API: https://spec.matrix.org/latest/client-server-api/ .. _mautrix.api: https://docs.mau.fi/python/latest/api/mautrix.api.html .. _mautrix.client.api: https://docs.mau.fi/python/latest/api/mautrix.client.api.html From ba883ea18ae7af80a83ee5203b21dde17a993aa0 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 20 Aug 2025 10:07:15 +0300 Subject: [PATCH 181/218] Allow filtering event handlers by sync source --- mautrix/client/syncer.py | 52 ++++++++++++++++++++++++++-------------- 1 file changed, 34 insertions(+), 18 deletions(-) diff --git a/mautrix/client/syncer.py b/mautrix/client/syncer.py index 8e115bbb..8183e142 100644 --- a/mautrix/client/syncer.py +++ b/mautrix/client/syncer.py @@ -5,11 +5,11 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import annotations -from typing import Any, Awaitable, Callable, Type, TypeVar +from typing import Any, Awaitable, Callable, NamedTuple, Optional, Type, TypeVar from abc import ABC, abstractmethod -from contextlib import suppress from enum import Enum, Flag, auto import asyncio +import itertools import time from mautrix.errors import MUnknownToken @@ -25,7 +25,6 @@ Filter, FilterID, GenericEvent, - MessageEvent, PresenceState, SerializerError, StateEvent, @@ -79,13 +78,18 @@ class InternalEventType(Enum): DEVICE_OTK_COUNT = auto() +class EventHandlerProps(NamedTuple): + wait_sync: bool + sync_stream: Optional[SyncStream] + + class Syncer(ABC): loop: asyncio.AbstractEventLoop log: TraceLogger mxid: UserID - global_event_handlers: list[tuple[EventHandler, bool]] - event_handlers: dict[EventType | InternalEventType, list[tuple[EventHandler, bool]]] + global_event_handlers: dict[EventHandler, EventHandlerProps] + event_handlers: dict[EventType | InternalEventType, dict[EventHandler, EventHandlerProps]] dispatchers: dict[Type[dispatcher.Dispatcher], dispatcher.Dispatcher] syncing_task: asyncio.Task | None ignore_initial_sync: bool @@ -95,7 +99,7 @@ class Syncer(ABC): sync_store: SyncStore def __init__(self, sync_store: SyncStore) -> None: - self.global_event_handlers = [] + self.global_event_handlers = {} self.event_handlers = {} self.dispatchers = {} self.syncing_task = None @@ -158,6 +162,7 @@ def add_event_handler( event_type: InternalEventType | EventType, handler: EventHandler, wait_sync: bool = False, + sync_stream: Optional[SyncStream] = None, ) -> None: """ Add a new event handler. @@ -167,13 +172,15 @@ def add_event_handler( event types. handler: The handler function to add. wait_sync: Whether or not the handler should be awaited before the next sync request. + sync_stream: The sync streams to listen to. Defaults to all. """ if not isinstance(event_type, (EventType, InternalEventType)): raise ValueError("Invalid event type") + props = EventHandlerProps(wait_sync=wait_sync, sync_stream=sync_stream) if event_type == EventType.ALL: - self.global_event_handlers.append((handler, wait_sync)) + self.global_event_handlers[handler] = props else: - self.event_handlers.setdefault(event_type, []).append((handler, wait_sync)) + self.event_handlers.setdefault(event_type, {})[handler] = props def remove_event_handler( self, event_type: EventType | InternalEventType, handler: EventHandler @@ -197,11 +204,7 @@ def remove_event_handler( # No handlers for this event type registered return - # FIXME this is a bit hacky - with suppress(ValueError): - handler_list.remove((handler, True)) - with suppress(ValueError): - handler_list.remove((handler, False)) + handler_list.pop(handler, None) if len(handler_list) == 0 and event_type != EventType.ALL: del self.event_handlers[event_type] @@ -229,7 +232,9 @@ def dispatch_event(self, event: Event | None, source: SyncStream) -> list[asynci else: event.type = event.type.with_class(EventType.Class.MESSAGE) setattr(event, "source", source) - return self.dispatch_manual_event(event.type, event, include_global_handlers=True) + return self.dispatch_manual_event( + event.type, event, include_global_handlers=True, source=source + ) async def _catch_errors(self, handler: EventHandler, data: Any) -> None: try: @@ -243,13 +248,22 @@ def dispatch_manual_event( data: Any, include_global_handlers: bool = False, force_synchronous: bool = False, + source: Optional[SyncStream] = None, ) -> list[asyncio.Task]: - handlers = self.event_handlers.get(event_type, []) + handlers = self.event_handlers.get(event_type, {}).items() if include_global_handlers: - handlers = self.global_event_handlers + handlers + handlers = itertools.chain(self.global_event_handlers.items(), handlers) tasks = [] - for handler, wait_sync in handlers: - if force_synchronous or wait_sync: + if source is None: + source = getattr(data, "source", None) + for handler, props in handlers: + if ( + props.sync_stream is not None + and source is not None + and not props.sync_stream & source + ): + continue + if force_synchronous or props.wait_sync: tasks.append(asyncio.create_task(self._catch_errors(handler, data))) else: background_task.create(self._catch_errors(handler, data)) @@ -263,6 +277,7 @@ async def run_internal_event( event_type, custom_type if custom_type is not None else kwargs, include_global_handlers=False, + source=SyncStream.INTERNAL, ) await asyncio.gather(*tasks) @@ -274,6 +289,7 @@ def dispatch_internal_event( event_type, custom_type if custom_type is not None else kwargs, include_global_handlers=False, + source=SyncStream.INTERNAL, ) def _try_deserialize(self, type: Type[T], data: JSON) -> T | GenericEvent: From ecd10f57bf13da3508f5ca98294c1ca2167bbf39 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 13 Sep 2025 19:45:17 +0300 Subject: [PATCH 182/218] Fix style in variation selector code --- mautrix/util/variation_selector.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mautrix/util/variation_selector.py b/mautrix/util/variation_selector.py index 498f1abe..dec0554d 100644 --- a/mautrix/util/variation_selector.py +++ b/mautrix/util/variation_selector.py @@ -59,11 +59,11 @@ async def fetch_data() -> dict[str, str]: ADD_VARIATION_TRANSLATION = str.maketrans( {ord(emoji): f"{emoji}{VARIATION_SELECTOR_16}" for emoji in read_data().values()} ) -SKIN_TONE_MODIFIERS = ("\U0001F3FB", "\U0001F3FC", "\U0001F3FD", "\U0001F3FE", "\U0001F3FF") +SKIN_TONE_MODIFIERS = ("\U0001f3fb", "\U0001f3fc", "\U0001f3fd", "\U0001f3fe", "\U0001f3ff") SKIN_TONE_REPLACEMENTS = {f"{VARIATION_SELECTOR_16}{mod}": mod for mod in SKIN_TONE_MODIFIERS} VARIATION_SELECTOR_REPLACEMENTS = { **SKIN_TONE_REPLACEMENTS, - "\U0001F408\ufe0f\u200d\u2b1b\ufe0f": "\U0001F408\u200d\u2b1b", + "\U0001f408\ufe0f\u200d\u2b1b\ufe0f": "\U0001f408\u200d\u2b1b", } From 799cb27267d0addb02e28fe7eb33e38987dc20d7 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 13 Sep 2025 19:23:43 +0300 Subject: [PATCH 183/218] Add utilities for SSSS --- mautrix/crypto/ssss/__init__.py | 7 + mautrix/crypto/ssss/key.py | 136 +++++++++++++++++++ mautrix/crypto/ssss/key_test.py | 199 ++++++++++++++++++++++++++++ mautrix/crypto/ssss/types.py | 51 +++++++ mautrix/crypto/ssss/util.py | 79 +++++++++++ mautrix/types/__init__.py | 5 + mautrix/types/event/__init__.py | 2 + mautrix/types/event/account_data.py | 20 ++- mautrix/types/event/type.py | 5 + mautrix/types/event/type.pyi | 5 + optional-requirements.txt | 1 + 11 files changed, 508 insertions(+), 2 deletions(-) create mode 100644 mautrix/crypto/ssss/__init__.py create mode 100644 mautrix/crypto/ssss/key.py create mode 100644 mautrix/crypto/ssss/key_test.py create mode 100644 mautrix/crypto/ssss/types.py create mode 100644 mautrix/crypto/ssss/util.py diff --git a/mautrix/crypto/ssss/__init__.py b/mautrix/crypto/ssss/__init__.py new file mode 100644 index 00000000..77389424 --- /dev/null +++ b/mautrix/crypto/ssss/__init__.py @@ -0,0 +1,7 @@ +from .key import Key, KeyMetadata, PassphraseMetadata +from .types import ( + Algorithm, + EncryptedAccountDataEventContent, + EncryptedKeyData, + PassphraseAlgorithm, +) diff --git a/mautrix/crypto/ssss/key.py b/mautrix/crypto/ssss/key.py new file mode 100644 index 00000000..0a159b3e --- /dev/null +++ b/mautrix/crypto/ssss/key.py @@ -0,0 +1,136 @@ +# Copyright (c) 2025 Tulir Asokan +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +from typing import Optional +import base64 +import hashlib +import hmac + +from attr import dataclass +import unpaddedbase64 + +from mautrix.types import EventType, SerializableAttrs + +from .types import Algorithm, EncryptedKeyData, PassphraseAlgorithm +from .util import ( + calculate_hash, + cryptorand, + decode_base58_recovery_key, + derive_keys, + encode_base58_recovery_key, + prepare_aes, +) + +try: + from Crypto.Cipher import AES + from Crypto.Util import Counter +except ImportError: + from Cryptodome.Cipher import AES + from Cryptodome.Util import Counter + + +@dataclass +class PassphraseMetadata(SerializableAttrs): + algorithm: PassphraseAlgorithm + iterations: int + salt: str + bits: int = 256 + + def get_key(self, passphrase: str) -> bytes: + if self.algorithm != PassphraseAlgorithm.PBKDF2: + raise ValueError(f"Unsupported passphrase algorithm {self.algorithm}") + return hashlib.pbkdf2_hmac( + "sha512", + passphrase.encode("utf-8"), + self.salt.encode("utf-8"), + self.iterations, + self.bits // 8, + ) + + +@dataclass +class KeyMetadata(SerializableAttrs): + algorithm: Algorithm + + iv: str | None = None + mac: str | None = None + + name: str | None = None + passphrase: Optional[PassphraseMetadata] = None + + def verify_passphrase(self, key_id: str, phrase: str) -> "Key": + if not self.passphrase: + raise ValueError("Passphrase not set on this key") + return self.verify_raw_key(key_id, self.passphrase.get_key(phrase)) + + def verify_recovery_key(self, key_id: str, recovery_key: str) -> "Key": + return self.verify_raw_key(key_id, decode_base58_recovery_key(recovery_key)) + + def verify_raw_key(self, key_id: str, key: bytes) -> "Key": + if self.mac.rstrip("=") != calculate_hash(key, self.iv): + raise ValueError("Key MAC does not match") + return Key(id=key_id, key=key, metadata=self) + + +@dataclass +class Key: + id: str + key: bytes + metadata: KeyMetadata + + @classmethod + def generate(cls, passphrase: str | None = None) -> "Key": + passphrase_meta = ( + PassphraseMetadata( + algorithm=PassphraseAlgorithm.PBKDF2, + iterations=500_000, + salt=base64.b64encode(cryptorand.read(24)).decode("utf-8"), + bits=256, + ) + if passphrase + else None + ) + key = passphrase_meta.get_key(passphrase) if passphrase else cryptorand.read(32) + iv = unpaddedbase64.encode_base64(cryptorand.read(16)) + metadata = KeyMetadata( + algorithm=Algorithm.AES_HMAC_SHA2, + passphrase=passphrase_meta, + mac=calculate_hash(key, iv), + iv=iv, + ) + key_id = unpaddedbase64.encode_base64(cryptorand.read(24)) + return cls(key=key, id=key_id, metadata=metadata) + + @property + def recovery_key(self) -> str: + return encode_base58_recovery_key(self.key) + + def encrypt(self, event_type: str | EventType, data: str | bytes) -> EncryptedKeyData: + if isinstance(data, str): + data = data.encode("utf-8") + data = base64.b64encode(data).rstrip(b"=") + + aes_key, hmac_key = derive_keys(self.key, event_type) + iv = bytearray(cryptorand.read(16)) + iv[8] &= 0x7F + ciphertext = prepare_aes(aes_key, iv).encrypt(data) + digest = hmac.digest(hmac_key, ciphertext, hashlib.sha256) + return EncryptedKeyData( + ciphertext=unpaddedbase64.encode_base64(ciphertext), + iv=unpaddedbase64.encode_base64(iv), + mac=unpaddedbase64.encode_base64(digest), + ) + + def decrypt(self, event_type: str | EventType, data: EncryptedKeyData) -> bytes: + aes_key, hmac_key = derive_keys(self.key, event_type) + ciphertext = unpaddedbase64.decode_base64(data.ciphertext) + mac = unpaddedbase64.decode_base64(data.mac) + + expected_mac = hmac.digest(hmac_key, ciphertext, hashlib.sha256) + if not hmac.compare_digest(mac, expected_mac): + raise ValueError("Invalid MAC") + + plaintext = prepare_aes(aes_key, data.iv).decrypt(ciphertext) + return unpaddedbase64.decode_base64(plaintext.decode("utf-8")) diff --git a/mautrix/crypto/ssss/key_test.py b/mautrix/crypto/ssss/key_test.py new file mode 100644 index 00000000..e60f1e3c --- /dev/null +++ b/mautrix/crypto/ssss/key_test.py @@ -0,0 +1,199 @@ +# Copyright (c) 2025 Tulir Asokan +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +import pytest + +from ...types.event.type import EventType +from .key import Key, KeyMetadata +from .types import EncryptedAccountDataEventContent + +KEY1_CROSS_SIGNING_MASTER_KEY = """{ + "encrypted": { + "gEJqbfSEMnP5JXXcukpXEX1l0aI3MDs0": { + "iv": "BpKP9nQJTE9jrsAssoxPqQ==", + "ciphertext": "fNRiiiidezjerTgV+G6pUtmeF3izzj5re/mVvY0hO2kM6kYGrxLuIu2ej80=", + "mac": "/gWGDGMyOLmbJp+aoSLh5JxCs0AdS6nAhjzpe+9G2Q0=" + } + } +}""" + +KEY1_CROSS_SIGNING_MASTER_KEY_DECRYPTED = bytes( + [ + 0x68, + 0xF9, + 0x7F, + 0xD1, + 0x92, + 0x2E, + 0xEC, + 0xF6, + 0xB8, + 0x2B, + 0xB8, + 0x90, + 0xD2, + 0x4D, + 0x06, + 0x52, + 0x98, + 0x4E, + 0x7A, + 0x1D, + 0x70, + 0x3B, + 0x9E, + 0x86, + 0x7B, + 0x7E, + 0xBA, + 0xF7, + 0xFE, + 0xB9, + 0x5B, + 0x6F, + ] +) + +KEY1_META = """{ + "algorithm": "m.secret_storage.v1.aes-hmac-sha2", + "passphrase": { + "algorithm": "m.pbkdf2", + "iterations": 500000, + "salt": "y863BOoqOadgDp8S3FtHXikDJEalsQ7d" + }, + "iv": "xxkTK0L4UzxgAFkQ6XPwsw", + "mac": "MEhooO0ZhFJNxUhvRMSxBnJfL20wkLgle3ocY0ee/eA" +}""" +KEY1_ID = "gEJqbfSEMnP5JXXcukpXEX1l0aI3MDs0" +KEY1_RECOVERY_KEY = "EsTE s92N EtaX s2h6 VQYF 9Kao tHYL mkyL GKMh isZb KJ4E tvoC" +KEY1_PASSPHRASE = "correct horse battery staple" + +KEY2_META = """{ + "algorithm": "m.secret_storage.v1.aes-hmac-sha2", + "iv": "O0BOvTqiIAYjC+RMcyHfWw==", + "mac": "7k6OruQlWg0UmQjxGZ0ad4Q6DdwkgnoI7G6X3IjBYtI=" +}""" +KEY2_ID = "NVe5vK6lZS9gEMQLJw0yqkzmE5Mr7dLv" +KEY2_RECOVERY_KEY = "EsUC xSxt XJgQ dz19 8WBZ rHdE GZo7 ybsn EFmG Y5HY MDAG GNWe" + +KEY2_META_BROKEN_IV = """{ + "algorithm": "m.secret_storage.v1.aes-hmac-sha2", + "iv": "O0BOvTqiIAYjC+RMcyHfWwMeowMeowMeow", + "mac": "7k6OruQlWg0UmQjxGZ0ad4Q6DdwkgnoI7G6X3IjBYtI=" +}""" + +KEY2_META_BROKEN_MAC = """{ + "algorithm": "m.secret_storage.v1.aes-hmac-sha2", + "iv": "O0BOvTqiIAYjC+RMcyHfWw==", + "mac": "7k6OruQlWg0UmQjxGZ0ad4Q6DdwkgnoI7G6X3IjBYtIMeowMeowMeow" +}""" + + +def get_key_meta(meta: str) -> KeyMetadata: + return KeyMetadata.parse_json(meta) + + +def get_key1() -> Key: + return get_key_meta(KEY1_META).verify_recovery_key(KEY1_ID, KEY1_RECOVERY_KEY) + + +def get_key2() -> Key: + return get_key_meta(KEY2_META).verify_recovery_key(KEY2_ID, KEY2_RECOVERY_KEY) + + +def get_encrypted_master_key() -> EncryptedAccountDataEventContent: + return EncryptedAccountDataEventContent.parse_json(KEY1_CROSS_SIGNING_MASTER_KEY) + + +def test_decrypt_success() -> None: + key = get_key1() + emk = get_encrypted_master_key() + assert ( + emk.decrypt(EventType.CROSS_SIGNING_MASTER, key) == KEY1_CROSS_SIGNING_MASTER_KEY_DECRYPTED + ) + + +def test_decrypt_fail_wrong_key() -> None: + key = get_key2() + emk = get_encrypted_master_key() + with pytest.raises(ValueError): + emk.decrypt(EventType.CROSS_SIGNING_MASTER, key) + + +def test_decrypt_fail_fake_key() -> None: + key = get_key2() + key.id = KEY1_ID + emk = get_encrypted_master_key() + with pytest.raises(ValueError): + emk.decrypt(EventType.CROSS_SIGNING_MASTER, key) + + +def test_decrypt_fail_wrong_type() -> None: + key = get_key1() + emk = get_encrypted_master_key() + with pytest.raises(ValueError): + emk.decrypt(EventType.CROSS_SIGNING_SELF_SIGNING, key) + + +def test_encrypt_roundtrip() -> None: + key = get_key1() + data = bytes([0xDE, 0xAD, 0xBE, 0xEF]) + ciphertext = key.encrypt("net.maunium.data", data) + plaintext = key.decrypt("net.maunium.data", ciphertext) + assert plaintext == data + + +def test_verify_recovery_key_correct() -> None: + meta = get_key_meta(KEY1_META) + key = meta.verify_recovery_key(KEY1_ID, KEY1_RECOVERY_KEY) + assert key.recovery_key == KEY1_RECOVERY_KEY + + +def test_verify_recovery_key_correct2() -> None: + meta = get_key_meta(KEY2_META) + key = meta.verify_recovery_key(KEY2_ID, KEY2_RECOVERY_KEY) + assert key.recovery_key == KEY2_RECOVERY_KEY + + +def test_verify_recovery_key_invalid() -> None: + meta = get_key_meta(KEY1_META) + with pytest.raises(ValueError): + meta.verify_recovery_key(KEY1_ID, "foo") + + +def test_verify_recovery_key_incorrect() -> None: + meta = get_key_meta(KEY1_META) + with pytest.raises(ValueError): + meta.verify_recovery_key(KEY2_ID, KEY2_RECOVERY_KEY) + + +def test_verify_recovery_key_broken_iv() -> None: + meta = get_key_meta(KEY2_META_BROKEN_IV) + with pytest.raises(ValueError): + meta.verify_recovery_key(KEY2_ID, KEY2_RECOVERY_KEY) + + +def test_verify_recovery_key_broken_mac() -> None: + meta = get_key_meta(KEY2_META_BROKEN_MAC) + with pytest.raises(ValueError): + meta.verify_recovery_key(KEY2_ID, KEY2_RECOVERY_KEY) + + +def test_verify_passphrase_correct() -> None: + meta = get_key_meta(KEY1_META) + key = meta.verify_passphrase(KEY1_ID, KEY1_PASSPHRASE) + assert key.recovery_key == KEY1_RECOVERY_KEY + + +def test_verify_passphrase_incorrect() -> None: + meta = get_key_meta(KEY1_META) + with pytest.raises(ValueError): + meta.verify_passphrase(KEY1_ID, "incorrect horse battery staple") + + +def test_verify_passphrase_notset() -> None: + meta = get_key_meta(KEY2_META) + with pytest.raises(ValueError): + meta.verify_passphrase(KEY2_ID, "hmm") diff --git a/mautrix/crypto/ssss/types.py b/mautrix/crypto/ssss/types.py new file mode 100644 index 00000000..4a47f743 --- /dev/null +++ b/mautrix/crypto/ssss/types.py @@ -0,0 +1,51 @@ +# Copyright (c) 2025 Tulir Asokan +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +from typing import TYPE_CHECKING + +from attr import dataclass + +from mautrix.types import EventType, SerializableAttrs, SerializableEnum +from mautrix.types.event.account_data import account_data_event_content_map + +if TYPE_CHECKING: + from .key import Key + + +class Algorithm(SerializableEnum): + AES_HMAC_SHA2 = "m.secret_storage.v1.aes-hmac-sha2" + CURVE25519_AES_SHA2 = "m.secret_storage.v1.curve25519-aes-sha2" + + +class PassphraseAlgorithm(SerializableEnum): + PBKDF2 = "m.pbkdf2" + + +@dataclass +class EncryptedKeyData(SerializableAttrs): + ciphertext: str + iv: str + mac: str + + +@dataclass +class EncryptedAccountDataEventContent(SerializableAttrs): + encrypted: dict[str, EncryptedKeyData] + + def decrypt(self, event_type: str | EventType, key: "Key") -> bytes: + try: + encrypted_data = self.encrypted[key.id] + except KeyError as e: + raise ValueError(f"Event not encrypted for provided key") from e + return key.decrypt(event_type, encrypted_data) + + +for encrypted_account_data_type in ( + EventType.CROSS_SIGNING_MASTER, + EventType.CROSS_SIGNING_USER_SIGNING, + EventType.CROSS_SIGNING_SELF_SIGNING, + EventType.MEGOLM_BACKUP_V1, +): + account_data_event_content_map[encrypted_account_data_type] = EncryptedAccountDataEventContent diff --git a/mautrix/crypto/ssss/util.py b/mautrix/crypto/ssss/util.py new file mode 100644 index 00000000..9ebf6dc9 --- /dev/null +++ b/mautrix/crypto/ssss/util.py @@ -0,0 +1,79 @@ +# Copyright (c) 2025 Tulir Asokan +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +import hashlib +import hmac +import struct + +import base58 +import unpaddedbase64 + +from mautrix.types import EventType + +try: + from Crypto import Random + from Crypto.Cipher import AES + from Crypto.Hash import SHA256 + from Crypto.Protocol.KDF import HKDF + from Crypto.Util import Counter +except ImportError: + from Cryptodome import Random + from Cryptodome.Cipher import AES + from Cryptodome.Hash import SHA256 + from Cryptodome.Protocol.KDF import HKDF + from Cryptodome.Util import Counter + +cryptorand = Random.new() + + +def decode_base58_recovery_key(key: str) -> bytes | None: + key_bytes = base58.b58decode(key.replace(" ", "")) + if len(key_bytes) != 35 or key_bytes[0] != 0x8B or key_bytes[1] != 1: + return None + parity = 0 + for byte in key_bytes[:34]: + parity ^= byte + return key_bytes[2:34] if parity == key_bytes[34] else None + + +def encode_base58_recovery_key(key: bytes) -> str: + key_bytes = bytearray(35) + key_bytes[0] = 0x8B + key_bytes[1] = 1 + key_bytes[2:34] = key + parity = 0 + for byte in key_bytes: + parity ^= byte + key_bytes[34] = parity + encoded_key = base58.b58encode(key_bytes).decode("utf-8") + return " ".join(encoded_key[i : i + 4] for i in range(0, len(encoded_key), 4)) + + +def derive_keys(key: bytes, name: str | EventType = "") -> tuple[bytes, bytes]: + aes_key, hmac_key = HKDF( + master=key, + key_len=32, + salt=b"\x00" * 32, + hashmod=SHA256, + num_keys=2, + context=str(name).encode("utf-8"), + ) + return aes_key, hmac_key + + +def prepare_aes(key: bytes, iv: str | bytes) -> AES: + if isinstance(iv, str): + iv = unpaddedbase64.decode_base64(iv) + # initial_value = struct.unpack(">Q", iv[8:])[0] + # counter = Counter.new(64, prefix=iv[:8], initial_value=initial_value) + counter = Counter.new(128, initial_value=int.from_bytes(iv)) + return AES.new(key=key, mode=AES.MODE_CTR, counter=counter) + + +def calculate_hash(key: bytes, iv: str | bytes) -> str: + aes_key, hmac_key = derive_keys(key) + cipher = prepare_aes(aes_key, iv).decrypt(b"\x00" * 32) + digest = hmac.digest(hmac_key, cipher, hashlib.sha256) + return unpaddedbase64.encode_base64(digest) diff --git a/mautrix/types/__init__.py b/mautrix/types/__init__.py index 42b9068c..ceceeaca 100644 --- a/mautrix/types/__init__.py +++ b/mautrix/types/__init__.py @@ -57,6 +57,7 @@ CallRejectEventContent, CallSelectAnswerEventContent, CanonicalAliasStateEventContent, + DirectAccountDataEventContent, EncryptedEvent, EncryptedEventContent, EncryptedFile, @@ -122,6 +123,7 @@ RoomTombstoneStateEventContent, RoomTopicStateEventContent, RoomType, + SecretStorageDefaultKeyEventContent, SingleReceiptEventContent, SpaceChildStateEventContent, SpaceParentStateEventContent, @@ -259,6 +261,7 @@ "CallRejectEventContent", "CallSelectAnswerEventContent", "CanonicalAliasStateEventContent", + "DirectAccountDataEventContent", "EncryptedEvent", "EncryptedEventContent", "EncryptedFile", @@ -324,6 +327,7 @@ "RoomTombstoneStateEventContent", "RoomTopicStateEventContent", "RoomType", + "SecretStorageDefaultKeyEventContent", "SingleReceiptEventContent", "SpaceChildStateEventContent", "SpaceParentStateEventContent", @@ -354,6 +358,7 @@ "OpenGraphImage", "OpenGraphVideo", "BatchSendResponse", + "BeeperBatchSendResponse", "DeviceLists", "DeviceOTKCount", "DirectoryPaginationToken", diff --git a/mautrix/types/event/__init__.py b/mautrix/types/event/__init__.py index b391e912..db0658db 100644 --- a/mautrix/types/event/__init__.py +++ b/mautrix/types/event/__init__.py @@ -6,8 +6,10 @@ from .account_data import ( AccountDataEvent, AccountDataEventContent, + DirectAccountDataEventContent, RoomTagAccountDataEventContent, RoomTagInfo, + SecretStorageDefaultKeyEventContent, ) from .base import BaseEvent, BaseRoomEvent, BaseUnsigned, GenericEvent from .batch import BatchSendEvent, BatchSendStateEvent diff --git a/mautrix/types/event/account_data.py b/mautrix/types/event/account_data.py index 3c144a36..fdfa0a30 100644 --- a/mautrix/types/event/account_data.py +++ b/mautrix/types/event/account_data.py @@ -3,7 +3,7 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from typing import Dict, List, Union +from typing import TYPE_CHECKING, Dict, List, Union from attr import dataclass import attr @@ -12,6 +12,9 @@ from ..util import Obj, SerializableAttrs, deserializer from .base import BaseEvent, EventType +if TYPE_CHECKING: + from mautrix.crypto.ssss import EncryptedAccountDataEventContent, KeyMetadata + @dataclass class RoomTagInfo(SerializableAttrs): @@ -23,11 +26,24 @@ class RoomTagAccountDataEventContent(SerializableAttrs): tags: Dict[str, RoomTagInfo] = attr.ib(default=None, metadata={"json": "tags"}) +@dataclass +class SecretStorageDefaultKeyEventContent(SerializableAttrs): + key: str + + DirectAccountDataEventContent = Dict[UserID, List[RoomID]] -AccountDataEventContent = Union[RoomTagAccountDataEventContent, DirectAccountDataEventContent, Obj] +AccountDataEventContent = Union[ + RoomTagAccountDataEventContent, + DirectAccountDataEventContent, + SecretStorageDefaultKeyEventContent, + "EncryptedAccountDataEventContent", + "KeyMetadata", + Obj, +] account_data_event_content_map = { EventType.TAG: RoomTagAccountDataEventContent, + EventType.SECRET_STORAGE_DEFAULT_KEY: SecretStorageDefaultKeyEventContent, # m.direct doesn't really need deserializing # EventType.DIRECT: DirectAccountDataEventContent, } diff --git a/mautrix/types/event/type.py b/mautrix/types/event/type.py index 509d6857..5faf3785 100644 --- a/mautrix/types/event/type.py +++ b/mautrix/types/event/type.py @@ -207,6 +207,11 @@ def is_to_device(self) -> bool: "m.push_rules": "PUSH_RULES", "m.tag": "TAG", "m.ignored_user_list": "IGNORED_USER_LIST", + "m.secret_storage.default_key": "SECRET_STORAGE_DEFAULT_KEY", + "m.cross_signing.master": "CROSS_SIGNING_MASTER", + "m.cross_signing.self_signing": "CROSS_SIGNING_SELF_SIGNING", + "m.cross_signing.user_signing": "CROSS_SIGNING_USER_SIGNING", + "m.megolm_backup.v1": "MEGOLM_BACKUP_V1", }, EventType.Class.TO_DEVICE: { "m.room.encrypted": "TO_DEVICE_ENCRYPTED", diff --git a/mautrix/types/event/type.pyi b/mautrix/types/event/type.pyi index 22922288..a2788d6f 100644 --- a/mautrix/types/event/type.pyi +++ b/mautrix/types/event/type.pyi @@ -61,6 +61,11 @@ class EventType(Serializable): PUSH_RULES: "EventType" TAG: "EventType" IGNORED_USER_LIST: "EventType" + SECRET_STORAGE_DEFAULT_KEY: "EventType" + CROSS_SIGNING_MASTER: "EventType" + CROSS_SIGNING_SELF_SIGNING: "EventType" + CROSS_SIGNING_USER_SIGNING: "EventType" + MEGOLM_BACKUP_V1: "EventType" TO_DEVICE_ENCRYPTED: "EventType" TO_DEVICE_DUMMY: "EventType" diff --git a/optional-requirements.txt b/optional-requirements.txt index a5660f4a..a6e0227d 100644 --- a/optional-requirements.txt +++ b/optional-requirements.txt @@ -11,3 +11,4 @@ uvloop python-olm unpaddedbase64 pycryptodome +base58 From 5e3dbf5c6e82a74960f9a905bb7be2393675a18e Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 13 Sep 2025 19:45:02 +0300 Subject: [PATCH 184/218] Add API calls for SSSS --- mautrix/crypto/base.py | 2 + mautrix/crypto/machine.py | 3 ++ mautrix/crypto/ssss/__init__.py | 1 + mautrix/crypto/ssss/machine.py | 65 +++++++++++++++++++++++++++++++++ 4 files changed, 71 insertions(+) create mode 100644 mautrix/crypto/ssss/machine.py diff --git a/mautrix/crypto/base.py b/mautrix/crypto/base.py index 1f8e9a62..b7b2171f 100644 --- a/mautrix/crypto/base.py +++ b/mautrix/crypto/base.py @@ -31,6 +31,7 @@ from mautrix.util.logging import TraceLogger from .. import client as cli, crypto +from .ssss import Machine as SSSSMachine class SignedObject(TypedDict): @@ -40,6 +41,7 @@ class SignedObject(TypedDict): class BaseOlmMachine: client: cli.Client + ssss: SSSSMachine log: TraceLogger crypto_store: crypto.CryptoStore state_store: crypto.StateStore diff --git a/mautrix/crypto/machine.py b/mautrix/crypto/machine.py index 3dc84856..adca95b2 100644 --- a/mautrix/crypto/machine.py +++ b/mautrix/crypto/machine.py @@ -36,6 +36,7 @@ from .encrypt_megolm import MegolmEncryptionMachine from .key_request import KeyRequestingMachine from .key_share import KeySharingMachine +from .ssss import Machine as SSSSMachine from .store import CryptoStore, StateStore from .unwedge import OlmUnwedgingMachine @@ -58,6 +59,7 @@ class OlmMachine( log: TraceLogger crypto_store: CryptoStore state_store: StateStore + ssss: SSSSMachine account: Optional[OlmAccount] @@ -70,6 +72,7 @@ def __init__( ) -> None: super().__init__() self.client = client + self.ssss = SSSSMachine(client) self.log = log or logging.getLogger("mau.crypto") self.crypto_store = crypto_store self.state_store = state_store diff --git a/mautrix/crypto/ssss/__init__.py b/mautrix/crypto/ssss/__init__.py index 77389424..9224418d 100644 --- a/mautrix/crypto/ssss/__init__.py +++ b/mautrix/crypto/ssss/__init__.py @@ -1,4 +1,5 @@ from .key import Key, KeyMetadata, PassphraseMetadata +from .machine import Machine from .types import ( Algorithm, EncryptedAccountDataEventContent, diff --git a/mautrix/crypto/ssss/machine.py b/mautrix/crypto/ssss/machine.py new file mode 100644 index 00000000..c43e25e3 --- /dev/null +++ b/mautrix/crypto/ssss/machine.py @@ -0,0 +1,65 @@ +# Copyright (c) 2025 Tulir Asokan +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +from mautrix import client as cli +from mautrix.errors import MNotFound +from mautrix.types import EventType, SecretStorageDefaultKeyEventContent + +from .key import Key, KeyMetadata +from .types import EncryptedAccountDataEventContent + + +class Machine: + client: cli.Client + + def __init__(self, client: cli.Client) -> None: + self.client = client + + async def get_default_key_id(self) -> str | None: + try: + data = await self.client.get_account_data(EventType.SECRET_STORAGE_DEFAULT_KEY) + return SecretStorageDefaultKeyEventContent.deserialize(data).key + except (MNotFound, ValueError): + return None + + async def set_default_key_id(self, key_id: str) -> None: + await self.client.set_account_data( + EventType.SECRET_STORAGE_DEFAULT_KEY, + SecretStorageDefaultKeyEventContent(key=key_id), + ) + + async def get_key_data(self, key_id: str) -> KeyMetadata: + data = await self.client.get_account_data(f"m.secret_storage.key.{key_id}") + return KeyMetadata.deserialize(data) + + async def set_key_data(self, key_id: str, data: KeyMetadata) -> None: + await self.client.set_account_data(f"m.secret_storage.key.{key_id}", data) + + async def get_default_key_data(self) -> tuple[str, KeyMetadata]: + key_id = await self.get_default_key_id() + if not key_id: + raise ValueError("No default key ID set") + return key_id, await self.get_key_data(key_id) + + async def get_decrypted_account_data(self, event_type: EventType | str, key: Key) -> bytes: + data = await self.client.get_account_data(event_type) + parsed = EncryptedAccountDataEventContent.deserialize(data) + return parsed.decrypt(event_type, key) + + async def set_encrypted_account_data( + self, event_type: EventType | str, data: bytes, *keys: Key + ) -> None: + encrypted_data = {} + for key in keys: + encrypted_data[key.id] = key.encrypt(event_type, data) + await self.client.set_account_data( + event_type, + EncryptedAccountDataEventContent(encrypted=encrypted_data), + ) + + async def generate_and_upload_key(self, passphrase: str | None = None) -> Key: + key = Key.generate(passphrase) + await self.set_key_data(key.id, key.metadata) + return key From ec318c0406d7ab7368d70fd77669afe0ce348d1d Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 13 Sep 2025 20:32:53 +0300 Subject: [PATCH 185/218] Move verify_signature_json to new file and add test --- mautrix/crypto/base.py | 39 +------------------------- mautrix/crypto/device_lists.py | 3 +- mautrix/crypto/encrypt_olm.py | 3 +- mautrix/crypto/signature.py | 47 ++++++++++++++++++++++++++++++++ mautrix/crypto/signature_test.py | 39 ++++++++++++++++++++++++++ mautrix/crypto/ssss/util.py | 1 - 6 files changed, 91 insertions(+), 41 deletions(-) create mode 100644 mautrix/crypto/signature.py create mode 100644 mautrix/crypto/signature_test.py diff --git a/mautrix/crypto/base.py b/mautrix/crypto/base.py index b7b2171f..f7015ca1 100644 --- a/mautrix/crypto/base.py +++ b/mautrix/crypto/base.py @@ -5,26 +5,18 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import annotations -from typing import Any, Awaitable, Callable, TypedDict +from typing import Awaitable, Callable import asyncio -import functools -import json - -import olm from mautrix.errors import MForbidden, MNotFound from mautrix.types import ( - DeviceID, - EncryptionKeyAlgorithm, EventType, IdentityKey, - KeyID, RequestedKeyInfo, RoomEncryptionStateEventContent, RoomID, RoomKeyEventContent, SessionID, - SigningKey, TrustState, UserID, ) @@ -34,11 +26,6 @@ from .ssss import Machine as SSSSMachine -class SignedObject(TypedDict): - signatures: dict[UserID, dict[str, str]] - unsigned: Any - - class BaseOlmMachine: client: cli.Client ssss: SSSSMachine @@ -118,27 +105,3 @@ async def _fill_encryption_info(self, evt: RoomKeyEventContent) -> None: evt.beeper_max_age_ms = encryption_info.rotation_period_ms if not evt.beeper_max_messages: evt.beeper_max_messages = encryption_info.rotation_period_msgs - - -canonical_json = functools.partial( - json.dumps, ensure_ascii=False, separators=(",", ":"), sort_keys=True -) - - -def verify_signature_json( - data: "SignedObject", user_id: UserID, key_name: DeviceID | str, key: SigningKey -) -> bool: - data_copy = {**data} - data_copy.pop("unsigned", None) - signatures = data_copy.pop("signatures") - key_id = str(KeyID(EncryptionKeyAlgorithm.ED25519, key_name)) - try: - signature = signatures[user_id][key_id] - except KeyError: - return False - signed_data = canonical_json(data_copy) - try: - olm.ed25519_verify(key, signed_data, signature) - return True - except olm.OlmVerifyError: - return False diff --git a/mautrix/crypto/device_lists.py b/mautrix/crypto/device_lists.py index 1b5e0dbd..93648889 100644 --- a/mautrix/crypto/device_lists.py +++ b/mautrix/crypto/device_lists.py @@ -23,7 +23,8 @@ UserID, ) -from .base import BaseOlmMachine, verify_signature_json +from .base import BaseOlmMachine +from .signature import verify_signature_json class DeviceListMachine(BaseOlmMachine): diff --git a/mautrix/crypto/encrypt_olm.py b/mautrix/crypto/encrypt_olm.py index 620cbbc8..029ad1e5 100644 --- a/mautrix/crypto/encrypt_olm.py +++ b/mautrix/crypto/encrypt_olm.py @@ -18,8 +18,9 @@ UserID, ) -from .base import BaseOlmMachine, verify_signature_json +from .base import BaseOlmMachine from .sessions import Session +from .signature import verify_signature_json ClaimKeysList = Dict[UserID, Dict[DeviceID, DeviceIdentity]] diff --git a/mautrix/crypto/signature.py b/mautrix/crypto/signature.py new file mode 100644 index 00000000..32f4342a --- /dev/null +++ b/mautrix/crypto/signature.py @@ -0,0 +1,47 @@ +# Copyright (c) 2025 Tulir Asokan +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +from typing import Any, TypedDict +import functools +import json + +import olm + +from mautrix.types import DeviceID, EncryptionKeyAlgorithm, KeyID, SigningKey, UserID + +try: + from Crypto.PublicKey import ECC + from Crypto.Signature import eddsa +except ImportError: + from Cryptodome.PublicKey import ECC + from Cryptodome.Signature import eddsa + +canonical_json = functools.partial( + json.dumps, ensure_ascii=False, separators=(",", ":"), sort_keys=True +) + + +class SignedObject(TypedDict): + signatures: dict[UserID, dict[str, str]] + unsigned: Any + + +def verify_signature_json( + data: "SignedObject", user_id: UserID, key_name: DeviceID | str, key: SigningKey +) -> bool: + data_copy = {**data} + data_copy.pop("unsigned", None) + signatures = data_copy.pop("signatures") + key_id = str(KeyID(EncryptionKeyAlgorithm.ED25519, key_name)) + try: + signature = signatures[user_id][key_id] + except KeyError: + return False + signed_data = canonical_json(data_copy) + try: + olm.ed25519_verify(key, signed_data, signature) + return True + except olm.OlmVerifyError: + return False diff --git a/mautrix/crypto/signature_test.py b/mautrix/crypto/signature_test.py new file mode 100644 index 00000000..115e4836 --- /dev/null +++ b/mautrix/crypto/signature_test.py @@ -0,0 +1,39 @@ +# Copyright (c) 2025 Tulir Asokan +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +from mautrix.types import SigningKey, UserID + +from .signature import verify_signature_json + + +def test_verify_signature_json() -> None: + assert verify_signature_json( + # This is actually a federation PDU rather than a device signature, + # but they're both 25519 curves so it doesn't make a difference. + { + "auth_events": [ + "$L8Ak6A939llTRIsZrytMlLDXQhI4uLEjx-wb1zSg-Bw", + "$QJmr7mmGeXGD4Tof0ZYSPW2oRGklseyHTKtZXnF-YNM", + "$7bkKK_Z-cGQ6Ae4HXWGBwXyZi3YjC6rIcQzGfVyl3Eo", + ], + "content": {}, + "depth": 3212, + "hashes": {"sha256": "K549YdTnv62Jn84Y7sS5ZN3+AdmhleZHbenbhUpR2R8"}, + "origin_server_ts": 1754242687127, + "prev_events": ["$DAhJg4jVsqk5FRatE2hbT1dSA8D2ASy5DbjEHIMSHwY"], + "room_id": "!offtopic-2:continuwuity.org", + "sender": "@tulir:maunium.net", + "type": "m.room.message", + "signatures": { + UserID("maunium.net"): { + "ed25519:a_xxeS": "SkzZdZ+rH22kzCBBIAErTdB0Vg6vkFmzvwjlOarGul72EnufgtE/tJcd3a8szAdK7f1ZovRyQxDgVm/Ib2u0Aw" + } + }, + "unsigned": {"age_ts": 1754242687146}, + }, + UserID("maunium.net"), + "a_xxeS", + SigningKey("lVt/CC3tv74OH6xTph2JrUmeRj/j+1q0HVa0Xf4QlCg"), + ) diff --git a/mautrix/crypto/ssss/util.py b/mautrix/crypto/ssss/util.py index 9ebf6dc9..bc7dfb81 100644 --- a/mautrix/crypto/ssss/util.py +++ b/mautrix/crypto/ssss/util.py @@ -5,7 +5,6 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. import hashlib import hmac -import struct import base58 import unpaddedbase64 From 71f4fae6cd1d32f12e09a04af7199da70ff4e4d1 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 13 Sep 2025 20:33:14 +0300 Subject: [PATCH 186/218] Switch to pycryptodome for ed25519 verification --- mautrix/crypto/signature.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/mautrix/crypto/signature.py b/mautrix/crypto/signature.py index 32f4342a..ca5220f8 100644 --- a/mautrix/crypto/signature.py +++ b/mautrix/crypto/signature.py @@ -7,7 +7,7 @@ import functools import json -import olm +import unpaddedbase64 from mautrix.types import DeviceID, EncryptionKeyAlgorithm, KeyID, SigningKey, UserID @@ -37,11 +37,13 @@ def verify_signature_json( key_id = str(KeyID(EncryptionKeyAlgorithm.ED25519, key_name)) try: signature = signatures[user_id][key_id] - except KeyError: - return False - signed_data = canonical_json(data_copy) - try: - olm.ed25519_verify(key, signed_data, signature) + decoded_key = unpaddedbase64.decode_base64(key) + # pycryptodome doesn't accept raw keys, so wrap it in a DER structure + der_key = b"\x30\x2a\x30\x05\x06\x03\x2b\x65\x70\x03\x21\x00" + decoded_key + decoded_signature = unpaddedbase64.decode_base64(signature) + parsed_key = ECC.import_key(der_key) + verifier = eddsa.new(parsed_key, "rfc8032") + verifier.verify(canonical_json(data_copy).encode("utf-8"), decoded_signature) return True - except olm.OlmVerifyError: + except (KeyError, ValueError): return False From 0ea9586cc2a7317e1d679503137a2bc4c2c3cbef Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 15 Sep 2025 02:22:06 +0300 Subject: [PATCH 187/218] Use more dataclasses for uploading e2ee device keys --- mautrix/client/api/modules/crypto.py | 10 ++++++-- mautrix/crypto/account.py | 35 ++++++++++++++-------------- mautrix/crypto/machine.py | 1 + mautrix/crypto/signature.py | 20 +++++++++++++++- mautrix/types/event/encrypted.py | 14 ++++++++++- 5 files changed, 59 insertions(+), 21 deletions(-) diff --git a/mautrix/client/api/modules/crypto.py b/mautrix/client/api/modules/crypto.py index af656916..065bffda 100644 --- a/mautrix/client/api/modules/crypto.py +++ b/mautrix/client/api/modules/crypto.py @@ -5,13 +5,15 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import annotations -from typing import Any +from typing import Any, Union from mautrix.api import Method, Path from mautrix.errors import MatrixResponseError from mautrix.types import ( + JSON, ClaimKeysResponse, DeviceID, + DeviceKeys, EncryptionKeyAlgorithm, EventType, QueryKeysResponse, @@ -82,7 +84,7 @@ async def send_to_one_device( async def upload_keys( self, one_time_keys: dict[str, Any] | None = None, - device_keys: dict[str, Any] | None = None, + device_keys: DeviceKeys | dict[str, Any] | None = None, ) -> dict[EncryptionKeyAlgorithm, int]: """ Publishes end-to-end encryption keys for the device. @@ -102,8 +104,12 @@ async def upload_keys( """ data = {} if device_keys: + if isinstance(device_keys, Serializable): + device_keys = device_keys.serialize() data["device_keys"] = device_keys if one_time_keys: + if isinstance(one_time_keys, Serializable): + one_time_keys = one_time_keys.serialize() data["one_time_keys"] = one_time_keys resp = await self.api.request(Method.POST, Path.v3.keys.upload, data) try: diff --git a/mautrix/crypto/account.py b/mautrix/crypto/account.py index db508262..63f83f93 100644 --- a/mautrix/crypto/account.py +++ b/mautrix/crypto/account.py @@ -10,15 +10,17 @@ from mautrix.types import ( DeviceID, + DeviceKeys, EncryptionAlgorithm, EncryptionKeyAlgorithm, IdentityKey, + KeyID, SigningKey, UserID, ) -from . import base from .sessions import Session +from .signature import sign_olm class OlmAccount(olm.Account): @@ -74,19 +76,18 @@ def new_outbound_session(self, target_key: IdentityKey, one_time_key: IdentityKe session.pickle("roundtrip"), passphrase="roundtrip", creation_time=datetime.now() ) - def get_device_keys(self, user_id: UserID, device_id: DeviceID) -> Dict[str, Any]: - device_keys = { - "user_id": user_id, - "device_id": device_id, - "algorithms": [EncryptionAlgorithm.OLM_V1.value, EncryptionAlgorithm.MEGOLM_V1.value], - "keys": { - f"{algorithm}:{device_id}": key for algorithm, key in self.identity_keys.items() + def get_device_keys(self, user_id: UserID, device_id: DeviceID) -> DeviceKeys: + device_keys = DeviceKeys( + user_id=user_id, + device_id=device_id, + algorithms=[EncryptionAlgorithm.OLM_V1, EncryptionAlgorithm.MEGOLM_V1], + keys={ + KeyID(algorithm=EncryptionKeyAlgorithm(algorithm), key_id=key): key + for algorithm, key in self.identity_keys.items() }, - } - signature = self.sign(base.canonical_json(device_keys)) - device_keys["signatures"] = { - user_id: {f"{EncryptionKeyAlgorithm.ED25519}:{device_id}": signature} - } + signatures={}, + ) + device_keys.signatures[user_id] = {KeyID.ed25519(device_id): sign_olm(device_keys, self)} return device_keys def get_one_time_keys( @@ -97,12 +98,12 @@ def get_one_time_keys( self.generate_one_time_keys(new_count) keys = {} for key_id, key in self.one_time_keys.get("curve25519", {}).items(): - signature = self.sign(base.canonical_json({"key": key})) - keys[f"{EncryptionKeyAlgorithm.SIGNED_CURVE25519}:{key_id}"] = { + keys[str(KeyID.signed_curve25519(IdentityKey(key_id)))] = { "key": key, "signatures": { - user_id: {f"{EncryptionKeyAlgorithm.ED25519}:{device_id}": signature} + user_id: { + str(KeyID.ed25519(device_id)): sign_olm({"key": key}, self), + } }, } - self.mark_keys_as_published() return keys diff --git a/mautrix/crypto/machine.py b/mautrix/crypto/machine.py index adca95b2..9088662d 100644 --- a/mautrix/crypto/machine.py +++ b/mautrix/crypto/machine.py @@ -313,6 +313,7 @@ async def _share_keys(self, current_otk_count: int | None) -> None: self.log.debug(f"Uploading {len(one_time_keys)} one-time keys") resp = await self.client.upload_keys(one_time_keys=one_time_keys, device_keys=device_keys) self.account.shared = True + self.account.mark_keys_as_published() self._last_key_share = time.monotonic() await self.crypto_store.put_account(self.account) self.log.debug(f"Shared keys and saved account, new keys: {resp}") diff --git a/mautrix/crypto/signature.py b/mautrix/crypto/signature.py index ca5220f8..6dc13e65 100644 --- a/mautrix/crypto/signature.py +++ b/mautrix/crypto/signature.py @@ -7,9 +7,19 @@ import functools import json +import olm import unpaddedbase64 -from mautrix.types import DeviceID, EncryptionKeyAlgorithm, KeyID, SigningKey, UserID +from mautrix.types import ( + JSON, + DeviceID, + EncryptionKeyAlgorithm, + KeyID, + Serializable, + Signature, + SigningKey, + UserID, +) try: from Crypto.PublicKey import ECC @@ -28,6 +38,14 @@ class SignedObject(TypedDict): unsigned: Any +def sign_olm(data: dict[str, JSON] | Serializable, key: olm.PkSigning | olm.Account) -> Signature: + if isinstance(data, Serializable): + data = data.serialize() + data.pop("signatures", None) + data.pop("unsigned", None) + return Signature(key.sign(canonical_json(data))) + + def verify_signature_json( data: "SignedObject", user_id: UserID, key_name: DeviceID | str, key: SigningKey ) -> bool: diff --git a/mautrix/types/event/encrypted.py b/mautrix/types/event/encrypted.py index cd08fc94..735e481a 100644 --- a/mautrix/types/event/encrypted.py +++ b/mautrix/types/event/encrypted.py @@ -9,7 +9,7 @@ from attr import dataclass -from ..primitive import JSON, DeviceID, IdentityKey, SessionID +from ..primitive import JSON, DeviceID, IdentityKey, SessionID, SigningKey from ..util import ExtensibleEnum, Obj, Serializable, SerializableAttrs, deserializer, field from .base import BaseRoomEvent, BaseUnsigned from .message import RelatesTo @@ -43,6 +43,18 @@ def deserialize(cls, raw: JSON) -> "KeyID": def __str__(self) -> str: return f"{self.algorithm.value}:{self.key_id}" + @classmethod + def ed25519(cls, key_id: SigningKey | DeviceID) -> "KeyID": + return cls(EncryptionKeyAlgorithm.ED25519, key_id) + + @classmethod + def curve25519(cls, key_id: IdentityKey) -> "KeyID": + return cls(EncryptionKeyAlgorithm.CURVE25519, key_id) + + @classmethod + def signed_curve25519(cls, key_id: IdentityKey) -> "KeyID": + return cls(EncryptionKeyAlgorithm.SIGNED_CURVE25519, key_id) + class OlmMsgType(Serializable, IntEnum): PREKEY = 0 From e496c2f5a2bd74458758c1f214101364c9483f64 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 15 Sep 2025 02:22:52 +0300 Subject: [PATCH 188/218] Add utilities for generating and using recovery keys --- mautrix/client/api/modules/crypto.py | 39 +++++++ mautrix/crypto/cross_signing.py | 169 +++++++++++++++++++++++++++ mautrix/crypto/cross_signing_key.py | 52 +++++++++ mautrix/crypto/device_lists.py | 18 +++ mautrix/crypto/machine.py | 6 + mautrix/types/crypto.py | 6 +- 6 files changed, 287 insertions(+), 3 deletions(-) create mode 100644 mautrix/crypto/cross_signing.py create mode 100644 mautrix/crypto/cross_signing_key.py diff --git a/mautrix/client/api/modules/crypto.py b/mautrix/client/api/modules/crypto.py index 065bffda..2d879a63 100644 --- a/mautrix/client/api/modules/crypto.py +++ b/mautrix/client/api/modules/crypto.py @@ -12,6 +12,8 @@ from mautrix.types import ( JSON, ClaimKeysResponse, + CrossSigningKeys, + CrossSigningUsage, DeviceID, DeviceKeys, EncryptionKeyAlgorithm, @@ -122,6 +124,43 @@ async def upload_keys( except AttributeError as e: raise MatrixResponseError("Invalid `one_time_key_counts` field in response.") from e + async def upload_cross_signing_keys( + self, + keys: dict[CrossSigningUsage, CrossSigningKeys], + auth: dict[str, JSON] | None = None, + ) -> None: + await self.api.request( + Method.POST, + Path.v3.keys.device_signing.upload, + {f"{usage}_key": key.serialize() for usage, key in keys.items()} + | ({"auth": auth} if auth else {}), + ) + + async def upload_one_signature( + self, + user_id: UserID, + device_id: DeviceID, + keys: Union[DeviceKeys, CrossSigningKeys], + ) -> None: + await self.api.request( + Method.POST, Path.v3.keys.signatures.upload, {user_id: {device_id: keys.serialize()}} + ) + # TODO check failures + + async def upload_many_signatures( + self, + signatures: dict[UserID, dict[DeviceID, Union[DeviceKeys, CrossSigningKeys]]], + ) -> None: + await self.api.request( + Method.POST, + Path.v3.keys.signatures.upload, + { + user_id: {device_id: keys.serialize() for device_id, keys in devices.items()} + for user_id, devices in signatures.items() + }, + ) + # TODO check failures + async def query_keys( self, device_keys: list[UserID] | set[UserID] | dict[UserID, list[DeviceID]], diff --git a/mautrix/crypto/cross_signing.py b/mautrix/crypto/cross_signing.py new file mode 100644 index 00000000..b768bc6f --- /dev/null +++ b/mautrix/crypto/cross_signing.py @@ -0,0 +1,169 @@ +# Copyright (c) 2025 Tulir Asokan +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +from ..types import ( + JSON, + CrossSigner, + CrossSigningKeys, + CrossSigningUsage, + DeviceIdentity, + EventType, + KeyID, + UserID, +) +from .cross_signing_key import CrossSigningPrivateKeys, CrossSigningPublicKeys, CrossSigningSeeds +from .device_lists import DeviceListMachine +from .signature import sign_olm +from .ssss import Key as SSSSKey + + +class CrossSigningMachine(DeviceListMachine): + _cross_signing_public_keys: CrossSigningPublicKeys | None + _cross_signing_public_keys_fetched: bool + _cross_signing_private_keys: CrossSigningPrivateKeys | None + + async def verify_with_recovery_key(self, recovery_key: str) -> None: + key_id, key_data = await self.ssss.get_default_key_data() + ssss_key = key_data.verify_recovery_key(key_id, recovery_key) + seeds = await self._fetch_cross_signing_keys_from_ssss(ssss_key) + self._import_cross_signing_keys(seeds) + await self.sign_own_device(self.own_identity) + + def _import_cross_signing_keys(self, seeds: CrossSigningSeeds) -> None: + self._cross_signing_private_keys = seeds.to_keys() + self._cross_signing_public_keys = self._cross_signing_private_keys.public_keys + + async def generate_recovery_key( + self, passphrase: str | None = None, seeds: CrossSigningSeeds | None = None + ) -> str: + seeds = seeds or CrossSigningSeeds.generate() + ssss_key = await self.ssss.generate_and_upload_key(passphrase) + await self._upload_cross_signing_keys_to_ssss(ssss_key, seeds) + await self._publish_cross_signing_keys(seeds.to_keys()) + await self.ssss.set_default_key_id(ssss_key.id) + await self.sign_own_device(self.own_identity) + return ssss_key.recovery_key + + async def _fetch_cross_signing_keys_from_ssss(self, key: SSSSKey) -> CrossSigningSeeds: + return CrossSigningSeeds( + master_key=await self.ssss.get_decrypted_account_data( + EventType.CROSS_SIGNING_MASTER, key + ), + user_signing_key=await self.ssss.get_decrypted_account_data( + EventType.CROSS_SIGNING_USER_SIGNING, key + ), + self_signing_key=await self.ssss.get_decrypted_account_data( + EventType.CROSS_SIGNING_SELF_SIGNING, key + ), + ) + + async def _upload_cross_signing_keys_to_ssss( + self, key: SSSSKey, seeds: CrossSigningSeeds + ) -> None: + await self.ssss.set_encrypted_account_data( + EventType.CROSS_SIGNING_MASTER, seeds.master_key, key + ) + await self.ssss.set_encrypted_account_data( + EventType.CROSS_SIGNING_USER_SIGNING, seeds.user_signing_key, key + ) + await self.ssss.set_encrypted_account_data( + EventType.CROSS_SIGNING_SELF_SIGNING, seeds.self_signing_key, key + ) + + async def get_own_cross_signing_public_keys(self) -> CrossSigningPublicKeys | None: + if self._cross_signing_public_keys or self._cross_signing_public_keys_fetched: + return self._cross_signing_public_keys + keys = await self.get_cross_signing_public_keys(self.client.mxid) + self._cross_signing_public_keys_fetched = True + if keys: + self._cross_signing_public_keys = keys + return keys + + async def get_cross_signing_public_keys( + self, user_id: UserID + ) -> CrossSigningPublicKeys | None: + db_keys = await self.crypto_store.get_cross_signing_keys(user_id) + if CrossSigningUsage.MASTER not in db_keys: + await self._fetch_keys([user_id], include_untracked=True) + db_keys = await self.crypto_store.get_cross_signing_keys(user_id) + if CrossSigningUsage.MASTER not in db_keys: + return None + return CrossSigningPublicKeys( + master_key=db_keys[CrossSigningUsage.MASTER].key, + self_signing_key=( + db_keys[CrossSigningUsage.SELF].key if CrossSigningUsage.SELF in db_keys else None + ), + user_signing_key=( + db_keys[CrossSigningUsage.USER].key if CrossSigningUsage.USER in db_keys else None + ), + ) + + async def sign_own_device(self, device: DeviceIdentity) -> None: + full_keys = await self._get_full_device_keys(device) + ssk = self._cross_signing_private_keys.self_signing_key + signature = sign_olm(full_keys, ssk) + full_keys.signatures = {self.client.mxid: {KeyID.ed25519(ssk.public_key): signature}} + await self.client.upload_one_signature(device.user_id, device.device_id, full_keys) + await self.crypto_store.put_signature( + CrossSigner(device.user_id, device.signing_key), + CrossSigner(self.client.mxid, ssk.public_key), + signature, + ) + + async def _publish_cross_signing_keys( + self, + keys: CrossSigningPrivateKeys, + auth: dict[str, JSON] | None = None, + ) -> None: + public = keys.public_keys + master_key = CrossSigningKeys( + user_id=self.client.mxid, + usage=[CrossSigningUsage.MASTER], + keys={KeyID.ed25519(public.master_key): public.master_key}, + ) + master_key.signatures = { + self.client.mxid: { + KeyID.ed25519(self.client.device_id): sign_olm(master_key, self.account), + } + } + self_key = CrossSigningKeys( + user_id=self.client.mxid, + usage=[CrossSigningUsage.SELF], + keys={KeyID.ed25519(public.self_signing_key): public.self_signing_key}, + ) + self_key.signatures = { + self.client.mxid: { + KeyID.ed25519(public.master_key): sign_olm(self_key, keys.master_key), + } + } + user_key = CrossSigningKeys( + user_id=self.client.mxid, + usage=[CrossSigningUsage.USER], + keys={KeyID.ed25519(public.user_signing_key): public.user_signing_key}, + ) + user_key.signatures = { + self.client.mxid: { + KeyID.ed25519(public.master_key): sign_olm(user_key, keys.master_key), + } + } + await self.client.upload_cross_signing_keys( + keys={ + CrossSigningUsage.MASTER: master_key, + CrossSigningUsage.SELF: self_key, + CrossSigningUsage.USER: user_key, + }, + auth=auth, + ) + await self.crypto_store.put_cross_signing_key( + self.client.mxid, CrossSigningUsage.MASTER, public.master_key + ) + await self.crypto_store.put_cross_signing_key( + self.client.mxid, CrossSigningUsage.SELF, public.self_signing_key + ) + await self.crypto_store.put_cross_signing_key( + self.client.mxid, CrossSigningUsage.USER, public.user_signing_key + ) + self._cross_signing_private_keys = keys + self._cross_signing_public_keys = public diff --git a/mautrix/crypto/cross_signing_key.py b/mautrix/crypto/cross_signing_key.py new file mode 100644 index 00000000..f4e3c1c1 --- /dev/null +++ b/mautrix/crypto/cross_signing_key.py @@ -0,0 +1,52 @@ +# Copyright (c) 2025 Tulir Asokan +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +from typing import NamedTuple + +import olm + +from mautrix.crypto.ssss.util import cryptorand +from mautrix.types import SigningKey + + +class CrossSigningPublicKeys(NamedTuple): + master_key: SigningKey + self_signing_key: SigningKey + user_signing_key: SigningKey + + +class CrossSigningPrivateKeys(NamedTuple): + master_key: olm.PkSigning + self_signing_key: olm.PkSigning + user_signing_key: olm.PkSigning + + @property + def public_keys(self) -> CrossSigningPublicKeys: + return CrossSigningPublicKeys( + master_key=self.master_key.public_key, + self_signing_key=self.self_signing_key.public_key, + user_signing_key=self.user_signing_key.public_key, + ) + + +class CrossSigningSeeds(NamedTuple): + master_key: bytes + self_signing_key: bytes + user_signing_key: bytes + + def to_keys(self) -> CrossSigningPrivateKeys: + return CrossSigningPrivateKeys( + master_key=olm.PkSigning(self.master_key), + self_signing_key=olm.PkSigning(self.self_signing_key), + user_signing_key=olm.PkSigning(self.user_signing_key), + ) + + @classmethod + def generate(cls) -> "CrossSigningSeeds": + return cls( + master_key=cryptorand.read(32), + self_signing_key=cryptorand.read(32), + user_signing_key=cryptorand.read(32), + ) diff --git a/mautrix/crypto/device_lists.py b/mautrix/crypto/device_lists.py index 93648889..96b4a64a 100644 --- a/mautrix/crypto/device_lists.py +++ b/mautrix/crypto/device_lists.py @@ -28,6 +28,18 @@ class DeviceListMachine(BaseOlmMachine): + @property + def own_identity(self) -> DeviceIdentity: + return DeviceIdentity( + user_id=self.client.mxid, + device_id=self.client.device_id, + identity_key=self.account.identity_key, + signing_key=self.account.signing_key, + trust=TrustState.VERIFIED, + deleted=False, + name="", + ) + async def _fetch_keys( self, users: list[UserID], since: SyncToken = "", include_untracked: bool = False ) -> dict[UserID, dict[DeviceID, DeviceIdentity]]: @@ -220,6 +232,12 @@ async def _store_cross_signing_keys(self, resp: QueryKeysResponse, user_id: User else: self.log.warning(f"Invalid signature from {signing_key_log} for {key_id}") + async def _get_full_device_keys(self, device: DeviceIdentity) -> DeviceKeys: + resp = await self.client.query_keys({device.user_id: [device.device_id]}) + keys = resp.device_keys[device.user_id][device.device_id] + await self._validate_device(device.user_id, device.device_id, keys, device) + return keys + async def get_or_fetch_device( self, user_id: UserID, device_id: DeviceID ) -> DeviceIdentity | None: diff --git a/mautrix/crypto/machine.py b/mautrix/crypto/machine.py index 9088662d..7585776d 100644 --- a/mautrix/crypto/machine.py +++ b/mautrix/crypto/machine.py @@ -32,6 +32,7 @@ from mautrix.util.logging import TraceLogger from .account import OlmAccount +from .cross_signing import CrossSigningMachine from .decrypt_megolm import MegolmDecryptionMachine from .encrypt_megolm import MegolmEncryptionMachine from .key_request import KeyRequestingMachine @@ -47,6 +48,7 @@ class OlmMachine( OlmUnwedgingMachine, KeySharingMachine, KeyRequestingMachine, + CrossSigningMachine, ): """ OlmMachine is the main class for handling things related to Matrix end-to-end encryption with @@ -99,6 +101,10 @@ def __init__( self._prev_unwedge = {} self._cs_fetch_attempted = set() + self._cross_signing_public_keys = None + self._cross_signing_public_keys_fetched = False + self._cross_signing_private_keys = None + self.client.add_event_handler( cli.InternalEventType.DEVICE_OTK_COUNT, self.handle_otk_count, wait_sync=True ) diff --git a/mautrix/types/crypto.py b/mautrix/types/crypto.py index 3bf7e96b..fe4ab742 100644 --- a/mautrix/types/crypto.py +++ b/mautrix/types/crypto.py @@ -47,9 +47,9 @@ def curve25519(self) -> Optional[IdentityKey]: class CrossSigningUsage(ExtensibleEnum): - MASTER = "master" - SELF = "self_signing" - USER = "user_signing" + MASTER: "CrossSigningUsage" = "master" + SELF: "CrossSigningUsage" = "self_signing" + USER: "CrossSigningUsage" = "user_signing" @dataclass From eccb1c5943d360185a255b69337f382940963e90 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 15 Sep 2025 03:07:46 +0300 Subject: [PATCH 189/218] Adjust cross-signing things --- mautrix/crypto/cross_signing.py | 8 ++++++-- mautrix/crypto/device_lists.py | 18 ++++++++++++------ 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/mautrix/crypto/cross_signing.py b/mautrix/crypto/cross_signing.py index b768bc6f..e3e3ead1 100644 --- a/mautrix/crypto/cross_signing.py +++ b/mautrix/crypto/cross_signing.py @@ -85,8 +85,12 @@ async def get_cross_signing_public_keys( self, user_id: UserID ) -> CrossSigningPublicKeys | None: db_keys = await self.crypto_store.get_cross_signing_keys(user_id) - if CrossSigningUsage.MASTER not in db_keys: - await self._fetch_keys([user_id], include_untracked=True) + if CrossSigningUsage.MASTER not in db_keys and user_id not in self._cs_fetch_attempted: + self.log.debug(f"Didn't find any cross-signing keys for {user_id}, fetching...") + async with self._fetch_keys_lock: + if user_id not in self._cs_fetch_attempted: + self._cs_fetch_attempted.add(user_id) + await self._fetch_keys([user_id], include_untracked=True) db_keys = await self.crypto_store.get_cross_signing_keys(user_id) if CrossSigningUsage.MASTER not in db_keys: return None diff --git a/mautrix/crypto/device_lists.py b/mautrix/crypto/device_lists.py index 96b4a64a..e2ff411e 100644 --- a/mautrix/crypto/device_lists.py +++ b/mautrix/crypto/device_lists.py @@ -315,18 +315,23 @@ async def _validate_device( deleted=False, ) - async def resolve_trust(self, device: DeviceIdentity) -> TrustState: + async def resolve_trust(self, device: DeviceIdentity, allow_fetch: bool = True) -> TrustState: try: - return await self._try_resolve_trust(device) + return await self._try_resolve_trust(device, allow_fetch) except Exception: self.log.exception(f"Failed to resolve trust of {device.user_id}/{device.device_id}") return TrustState.UNVERIFIED - async def _try_resolve_trust(self, device: DeviceIdentity) -> TrustState: - if device.trust in (TrustState.VERIFIED, TrustState.BLACKLISTED): + async def _try_resolve_trust( + self, device: DeviceIdentity, allow_fetch: bool = True + ) -> TrustState: + if device.device_id != self.client.device_id and device.trust in ( + TrustState.VERIFIED, + TrustState.BLACKLISTED, + ): return device.trust their_keys = await self.crypto_store.get_cross_signing_keys(device.user_id) - if len(their_keys) == 0 and device.user_id not in self._cs_fetch_attempted: + if len(their_keys) == 0 and allow_fetch and device.user_id not in self._cs_fetch_attempted: self.log.debug(f"Didn't find any cross-signing keys for {device.user_id}, fetching...") async with self._fetch_keys_lock: if device.user_id not in self._cs_fetch_attempted: @@ -337,7 +342,8 @@ async def _try_resolve_trust(self, device: DeviceIdentity) -> TrustState: msk = their_keys[CrossSigningUsage.MASTER] ssk = their_keys[CrossSigningUsage.SELF] except KeyError as e: - self.log.error(f"Didn't find cross-signing key {e.args[0]} of {device.user_id}") + if allow_fetch: + self.log.error(f"Didn't find cross-signing key {e.args[0]} of {device.user_id}") return TrustState.UNVERIFIED ssk_signed = await self.crypto_store.is_key_signed_by( target=CrossSigner(device.user_id, ssk.key), From afc780408d6e4d3a06577467bf0a0a5e603c4437 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 15 Sep 2025 03:13:31 +0300 Subject: [PATCH 190/218] Bump version to 0.21.0b1 --- CHANGELOG.md | 4 +++- mautrix/__init__.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6bbb268b..7e663680 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,8 @@ -## v0.20.9 (unreleased) +## v0.21.0 (unreleased) * *(event)* Added support for creator power in room v12+. +* *(crypto)* Added support for generating and using recovery keys for verifying + the active device. * *(bridge)* Removed check for login flows when using MSC4190 (thanks to [@meson800] in [#178]). diff --git a/mautrix/__init__.py b/mautrix/__init__.py index beb96400..8a95f01e 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.20.9rc3" +__version__ = "0.21.0b1" __author__ = "Tulir Asokan " __all__ = [ "api", From 9cb168138eafc9c2047df1edf1e02cf0c6f034de Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 15 Sep 2025 03:16:03 +0300 Subject: [PATCH 191/218] Update doc builder requirements --- docs/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/requirements.txt b/docs/requirements.txt index 798e0f1b..4b0fdff7 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -17,3 +17,4 @@ prometheus_client python-olm unpaddedbase64 pycryptodome +base58 From 8a201445fcc3b0b3d3b332e61e00e468994025c3 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 15 Sep 2025 03:18:49 +0300 Subject: [PATCH 192/218] Fix sqlalchemy version --- docs/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 4b0fdff7..7269d3a5 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -9,7 +9,7 @@ yarl # that aren't used for anything that's in the docs python-magic ruamel.yaml -SQLAlchemy +SQLAlchemy<2 commonmark asyncpg aiosqlite From 7aadbc53ae254179d88472dc7d06275bc969b9ea Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 15 Sep 2025 03:19:36 +0300 Subject: [PATCH 193/218] Update test requirements --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d5c58eb0..20b51098 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from mautrix import __version__ -encryption_dependencies = ["python-olm", "unpaddedbase64", "pycryptodome"] +encryption_dependencies = ["python-olm", "unpaddedbase64", "pycryptodome", "base58"] test_dependencies = ["aiosqlite", "asyncpg", "ruamel.yaml", *encryption_dependencies] setuptools.setup( From bcb597733f0421fdbf0808389a857880aa29a10c Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 15 Sep 2025 03:25:16 +0300 Subject: [PATCH 194/218] Fix Python 3.10 compatibility --- mautrix/crypto/ssss/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/crypto/ssss/util.py b/mautrix/crypto/ssss/util.py index bc7dfb81..b58c941a 100644 --- a/mautrix/crypto/ssss/util.py +++ b/mautrix/crypto/ssss/util.py @@ -67,7 +67,7 @@ def prepare_aes(key: bytes, iv: str | bytes) -> AES: iv = unpaddedbase64.decode_base64(iv) # initial_value = struct.unpack(">Q", iv[8:])[0] # counter = Counter.new(64, prefix=iv[:8], initial_value=initial_value) - counter = Counter.new(128, initial_value=int.from_bytes(iv)) + counter = Counter.new(128, initial_value=int.from_bytes(iv, byteorder="big")) return AES.new(key=key, mode=AES.MODE_CTR, counter=counter) From 481cb8330739e93df1dfc0b239bb3bbc875ab149 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 15 Sep 2025 03:26:58 +0300 Subject: [PATCH 195/218] Bump version to 0.21.0b2 --- mautrix/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 8a95f01e..15596f17 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.21.0b1" +__version__ = "0.21.0b2" __author__ = "Tulir Asokan " __all__ = [ "api", From 03258b1c511722aed6a74649a0fb1d56e8ebe461 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 15 Sep 2025 15:18:43 +0300 Subject: [PATCH 196/218] Throw error if recovery key is malformed --- mautrix/__init__.py | 2 +- mautrix/crypto/ssss/key.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 15596f17..9f500b4c 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.21.0b2" +__version__ = "0.21.0b3" __author__ = "Tulir Asokan " __all__ = [ "api", diff --git a/mautrix/crypto/ssss/key.py b/mautrix/crypto/ssss/key.py index 0a159b3e..691ded71 100644 --- a/mautrix/crypto/ssss/key.py +++ b/mautrix/crypto/ssss/key.py @@ -66,7 +66,10 @@ def verify_passphrase(self, key_id: str, phrase: str) -> "Key": return self.verify_raw_key(key_id, self.passphrase.get_key(phrase)) def verify_recovery_key(self, key_id: str, recovery_key: str) -> "Key": - return self.verify_raw_key(key_id, decode_base58_recovery_key(recovery_key)) + decoded_key = decode_base58_recovery_key(recovery_key) + if not decoded_key: + raise ValueError("Invalid recovery key syntax") + return self.verify_raw_key(key_id, decoded_key) def verify_raw_key(self, key_id: str, key: bytes) -> "Key": if self.mac.rstrip("=") != calculate_hash(key, self.iv): From c4028be095c092623068819f174671f40ccb8a1a Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 15 Sep 2025 15:24:50 +0300 Subject: [PATCH 197/218] Don't allow verifying before sharing account keys --- mautrix/crypto/cross_signing.py | 4 ++++ mautrix/crypto/device_lists.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/mautrix/crypto/cross_signing.py b/mautrix/crypto/cross_signing.py index e3e3ead1..7577cd5d 100644 --- a/mautrix/crypto/cross_signing.py +++ b/mautrix/crypto/cross_signing.py @@ -25,6 +25,8 @@ class CrossSigningMachine(DeviceListMachine): _cross_signing_private_keys: CrossSigningPrivateKeys | None async def verify_with_recovery_key(self, recovery_key: str) -> None: + if not self.account.shared: + raise ValueError("Device keys must be shared before verifying with recovery key") key_id, key_data = await self.ssss.get_default_key_data() ssss_key = key_data.verify_recovery_key(key_id, recovery_key) seeds = await self._fetch_cross_signing_keys_from_ssss(ssss_key) @@ -38,6 +40,8 @@ def _import_cross_signing_keys(self, seeds: CrossSigningSeeds) -> None: async def generate_recovery_key( self, passphrase: str | None = None, seeds: CrossSigningSeeds | None = None ) -> str: + if not self.account.shared: + raise ValueError("Device keys must be shared before generating recovery key") seeds = seeds or CrossSigningSeeds.generate() ssss_key = await self.ssss.generate_and_upload_key(passphrase) await self._upload_cross_signing_keys_to_ssss(ssss_key, seeds) diff --git a/mautrix/crypto/device_lists.py b/mautrix/crypto/device_lists.py index e2ff411e..4314501a 100644 --- a/mautrix/crypto/device_lists.py +++ b/mautrix/crypto/device_lists.py @@ -206,7 +206,7 @@ async def _store_cross_signing_keys(self, resp: QueryKeysResponse, user_id: User signing_key = device.ed25519 except KeyError: pass - if len(signing_key) != 43: + if not signing_key or len(signing_key) != 43: self.log.debug( f"Cross-signing key {user_id}/{actual_key} has a signature from " f"an unknown key {key_id}" From f5492167d9b832390a9869a9c98f65d2949e7071 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 15 Sep 2025 15:26:01 +0300 Subject: [PATCH 198/218] Fix sharing device keys --- mautrix/crypto/account.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/crypto/account.py b/mautrix/crypto/account.py index 63f83f93..a00ada71 100644 --- a/mautrix/crypto/account.py +++ b/mautrix/crypto/account.py @@ -82,7 +82,7 @@ def get_device_keys(self, user_id: UserID, device_id: DeviceID) -> DeviceKeys: device_id=device_id, algorithms=[EncryptionAlgorithm.OLM_V1, EncryptionAlgorithm.MEGOLM_V1], keys={ - KeyID(algorithm=EncryptionKeyAlgorithm(algorithm), key_id=key): key + KeyID(algorithm=EncryptionKeyAlgorithm(algorithm), key_id=device_id): key for algorithm, key in self.identity_keys.items() }, signatures={}, From 96b5f20b44ef1258741f8ba193021d56066c1fc0 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 15 Sep 2025 15:26:41 +0300 Subject: [PATCH 199/218] Bump version to 0.21.0b4 --- mautrix/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 9f500b4c..41283ebb 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.21.0b3" +__version__ = "0.21.0b4" __author__ = "Tulir Asokan " __all__ = [ "api", From d122a7a9ba3be32c5fb9d782fea6e509a3952317 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 23 Sep 2025 23:33:28 +0300 Subject: [PATCH 200/218] Fix CI image --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8f3efe97..b0d7ab3f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,6 +1,6 @@ build docs builder: stage: build - image: docker:stable + image: docker:latest tags: - amd64 only: From 8adeea8c576f9db03aab8dd00619e99fc1a06ea0 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 23 Sep 2025 23:33:50 +0300 Subject: [PATCH 201/218] Store acquired connection in context variable --- mautrix/util/async_db/aiosqlite.py | 2 +- mautrix/util/async_db/asyncpg.py | 2 +- mautrix/util/async_db/database.py | 19 ++++++++++++++++++- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/mautrix/util/async_db/aiosqlite.py b/mautrix/util/async_db/aiosqlite.py index 80fa3cf8..4a2ff826 100644 --- a/mautrix/util/async_db/aiosqlite.py +++ b/mautrix/util/async_db/aiosqlite.py @@ -181,7 +181,7 @@ async def stop(self) -> None: self._conns -= 1 await conn.close() - def acquire(self) -> AsyncContextManager[LoggingConnection]: + def acquire_direct(self) -> AsyncContextManager[LoggingConnection]: if self._parent: return self._parent.acquire() return self._acquire() diff --git a/mautrix/util/async_db/asyncpg.py b/mautrix/util/async_db/asyncpg.py index 07ef7ad7..97b49f6c 100644 --- a/mautrix/util/async_db/asyncpg.py +++ b/mautrix/util/async_db/asyncpg.py @@ -96,7 +96,7 @@ async def _handle_exception(self, err: Exception) -> None: sys.exit(26) @asynccontextmanager - async def acquire(self) -> LoggingConnection: + async def acquire_direct(self) -> LoggingConnection: async with self.pool.acquire() as conn: yield LoggingConnection( self.scheme, conn, self.log, handle_exception=self._handle_exception diff --git a/mautrix/util/async_db/database.py b/mautrix/util/async_db/database.py index 0f23b02d..ff73a8b2 100644 --- a/mautrix/util/async_db/database.py +++ b/mautrix/util/async_db/database.py @@ -7,6 +7,8 @@ from typing import Any, AsyncContextManager, Type from abc import ABC, abstractmethod +from contextlib import asynccontextmanager +from contextvars import ContextVar import logging from yarl import URL @@ -23,6 +25,8 @@ from aiosqlite import Cursor from asyncpg import Record +conn_var: ContextVar[LoggingConnection | None] = ContextVar("db_connection", default=None) + class Database(ABC): schemes: dict[str, Type[Database]] = {} @@ -128,9 +132,22 @@ async def stop(self) -> None: pass @abstractmethod - def acquire(self) -> AsyncContextManager[LoggingConnection]: + def acquire_direct(self) -> AsyncContextManager[LoggingConnection]: pass + @asynccontextmanager + async def acquire(self) -> LoggingConnection: + conn = conn_var.get(None) + if conn is not None: + yield conn + return + async with self.acquire_direct() as conn: + token = conn_var.set(conn) + try: + yield conn + finally: + conn_var.reset(token) + async def execute(self, query: str, *args: Any, timeout: float | None = None) -> str | Cursor: async with self.acquire() as conn: return await conn.execute(query, *args, timeout=timeout) From d675cf3f9729b20bc024737b0bd1285cd847dd6e Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 23 Sep 2025 23:50:01 +0300 Subject: [PATCH 202/218] Make nested transactions no-ops on SQLite --- mautrix/util/async_db/aiosqlite.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/mautrix/util/async_db/aiosqlite.py b/mautrix/util/async_db/aiosqlite.py index 4a2ff826..934379a8 100644 --- a/mautrix/util/async_db/aiosqlite.py +++ b/mautrix/util/async_db/aiosqlite.py @@ -7,6 +7,7 @@ from typing import Any, AsyncContextManager from contextlib import asynccontextmanager +from contextvars import ContextVar import asyncio import logging import os @@ -24,6 +25,9 @@ POSITIONAL_PARAM_PATTERN = re.compile(r"\$(\d+)") +in_transaction = ContextVar("in_transaction", default=False) + + class TxnConnection(aiosqlite.Connection): def __init__(self, path: str, **kwargs) -> None: def connector() -> sqlite3.Connection: @@ -35,7 +39,11 @@ def connector() -> sqlite3.Connection: @asynccontextmanager async def transaction(self) -> None: + if in_transaction.get(): + yield + return await self.execute("BEGIN TRANSACTION") + token = in_transaction.set(True) try: yield except Exception: @@ -43,6 +51,8 @@ async def transaction(self) -> None: raise else: await self.commit() + finally: + in_transaction.reset(token) def __execute(self, query: str, *args: Any): query = POSITIONAL_PARAM_PATTERN.sub(r"?\1", query) From 513e9256f4c11c2b24a648bab79c244835f5cbf7 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 23 Sep 2025 23:56:01 +0300 Subject: [PATCH 203/218] Use transaction for entire fetch keys step --- mautrix/crypto/device_lists.py | 106 ++++++++++++++------------ mautrix/crypto/store/abstract.py | 6 +- mautrix/crypto/store/asyncpg/store.py | 6 ++ 3 files changed, 67 insertions(+), 51 deletions(-) diff --git a/mautrix/crypto/device_lists.py b/mautrix/crypto/device_lists.py index 4314501a..b00b104c 100644 --- a/mautrix/crypto/device_lists.py +++ b/mautrix/crypto/device_lists.py @@ -59,61 +59,67 @@ async def _fetch_keys( data = {} for user_id, devices in resp.device_keys.items(): missing_users.remove(user_id) - - new_devices = {} - existing_devices = (await self.crypto_store.get_devices(user_id)) or {} - - self.log.trace( - f"Updating devices for {user_id}, got {len(devices)}, " - f"have {len(existing_devices)} in store" - ) - changed = False - ssks = resp.self_signing_keys.get(user_id) - ssk = ssks.first_ed25519_key if ssks else None - for device_id, device_keys in devices.items(): - try: - existing = existing_devices[device_id] - except KeyError: - existing = None - changed = True - self.log.trace(f"Validating device {device_keys} of {user_id}") - try: - new_device = await self._validate_device( - user_id, device_id, device_keys, existing - ) - except DeviceValidationError as e: - self.log.warning(f"Failed to validate device {device_id} of {user_id}: {e}") - else: - if new_device: - new_devices[device_id] = new_device - await self._store_device_self_signatures(device_keys, ssk) - self.log.debug( - f"Storing new device list for {user_id} containing {len(new_devices)} devices" - ) - await self.crypto_store.put_devices(user_id, new_devices) - data[user_id] = new_devices - - if changed or len(new_devices) != len(existing_devices): - if self.delete_keys_on_device_delete: - for device_id in existing_devices.keys() - new_devices.keys(): - device = existing_devices[device_id] - removed_ids = await self.crypto_store.redact_group_sessions( - room_id=None, sender_key=device.identity_key, reason="device removed" - ) - self.log.info( - "Redacted megolm sessions sent by removed device " - f"{device.user_id}/{device.device_id}: {removed_ids}" - ) - await self.on_devices_changed(user_id) + async with self.crypto_store.transaction(): + data[user_id] = await self._process_fetched_keys(user_id, devices, resp) for user_id in missing_users: self.log.warning(f"Didn't get any devices for user {user_id}") - for user_id in users: - await self._store_cross_signing_keys(resp, user_id) - return data + async def _process_fetched_keys( + self, + user_id: UserID, + devices: dict[DeviceID, DeviceKeys], + resp: QueryKeysResponse, + ) -> dict[DeviceID, DeviceIdentity]: + new_devices = {} + existing_devices = (await self.crypto_store.get_devices(user_id)) or {} + + self.log.trace( + f"Updating devices for {user_id}, got {len(devices)}, " + f"have {len(existing_devices)} in store" + ) + changed = False + ssks = resp.self_signing_keys.get(user_id) + ssk = ssks.first_ed25519_key if ssks else None + for device_id, device_keys in devices.items(): + try: + existing = existing_devices[device_id] + except KeyError: + existing = None + changed = True + self.log.trace(f"Validating device {device_keys} of {user_id}") + try: + new_device = await self._validate_device(user_id, device_id, device_keys, existing) + except DeviceValidationError as e: + self.log.warning(f"Failed to validate device {device_id} of {user_id}: {e}") + else: + if new_device: + new_devices[device_id] = new_device + await self._store_device_self_signatures(device_keys, ssk) + self.log.debug( + f"Storing new device list for {user_id} containing {len(new_devices)} devices" + ) + await self.crypto_store.put_devices(user_id, new_devices) + + if changed or len(new_devices) != len(existing_devices): + if self.delete_keys_on_device_delete: + for device_id in existing_devices.keys() - new_devices.keys(): + device = existing_devices[device_id] + removed_ids = await self.crypto_store.redact_group_sessions( + room_id=None, sender_key=device.identity_key, reason="device removed" + ) + self.log.info( + "Redacted megolm sessions sent by removed device " + f"{device.user_id}/{device.device_id}: {removed_ids}" + ) + await self.on_devices_changed(user_id) + + await self._store_cross_signing_keys(resp, user_id) + + return new_devices + async def _store_device_self_signatures( self, device_keys: DeviceKeys, self_signing_key: SigningKey | None ) -> None: @@ -343,7 +349,7 @@ async def _try_resolve_trust( ssk = their_keys[CrossSigningUsage.SELF] except KeyError as e: if allow_fetch: - self.log.error(f"Didn't find cross-signing key {e.args[0]} of {device.user_id}") + self.log.warning(f"Didn't find cross-signing key {e.args[0]} of {device.user_id}") return TrustState.UNVERIFIED ssk_signed = await self.crypto_store.is_key_signed_by( target=CrossSigner(device.user_id, ssk.key), diff --git a/mautrix/crypto/store/abstract.py b/mautrix/crypto/store/abstract.py index 787d5225..1bae832f 100644 --- a/mautrix/crypto/store/abstract.py +++ b/mautrix/crypto/store/abstract.py @@ -5,7 +5,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import annotations -from typing import NamedTuple +from typing import AsyncContextManager, NamedTuple from abc import ABC, abstractmethod from mautrix.types import ( @@ -87,6 +87,10 @@ async def close(self) -> None: async def flush(self) -> None: """Flush the store. If all the methods persist data immediately, this can be a no-op.""" + async def transaction(self) -> AsyncContextManager[None]: + """Run a database transaction. If the store doesn't support transactions, this can be a no-op.""" + pass + @abstractmethod async def delete(self) -> None: """Delete the data in the store.""" diff --git a/mautrix/crypto/store/asyncpg/store.py b/mautrix/crypto/store/asyncpg/store.py index a29f7737..bdc37ddd 100644 --- a/mautrix/crypto/store/asyncpg/store.py +++ b/mautrix/crypto/store/asyncpg/store.py @@ -6,6 +6,7 @@ from __future__ import annotations from collections import defaultdict +from contextlib import asynccontextmanager from datetime import timedelta from asyncpg import UniqueViolationError @@ -79,6 +80,11 @@ def __init__(self, account_id: str, pickle_key: str, db: Database) -> None: self._account = None self._olm_cache = defaultdict(lambda: {}) + @asynccontextmanager + async def transaction(self) -> None: + async with self.db.acquire() as conn, conn.transaction(): + yield + async def delete(self) -> None: tables = ("crypto_account", "crypto_olm_session", "crypto_megolm_outbound_session") async with self.db.acquire() as conn, conn.transaction(): From 1443f53ef58ee3dc5b50d377d31b845aee989c41 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 23 Sep 2025 23:58:50 +0300 Subject: [PATCH 204/218] Add option to self-sign bridge bot device --- mautrix/bridge/config.py | 1 + mautrix/bridge/e2ee.py | 12 ++++++++++++ 2 files changed, 13 insertions(+) diff --git a/mautrix/bridge/config.py b/mautrix/bridge/config.py index 46fb88d2..defed222 100644 --- a/mautrix/bridge/config.py +++ b/mautrix/bridge/config.py @@ -144,6 +144,7 @@ def do_update(self, helper: ConfigUpdateHelper) -> None: copy("bridge.encryption.require") copy("bridge.encryption.appservice") copy("bridge.encryption.msc4190") + copy("bridge.encryption.self_sign") copy("bridge.encryption.delete_keys.delete_outbound_on_ack") copy("bridge.encryption.delete_keys.dont_store_outbound") copy("bridge.encryption.delete_keys.ratchet_on_decrypt") diff --git a/mautrix/bridge/e2ee.py b/mautrix/bridge/e2ee.py index 56270942..1525b388 100644 --- a/mautrix/bridge/e2ee.py +++ b/mautrix/bridge/e2ee.py @@ -58,6 +58,7 @@ class EncryptionManager: periodically_delete_expired_keys: bool delete_outdated_inbound: bool msc4190: bool + self_sign: bool bridge: br.Bridge az: AppService @@ -110,6 +111,7 @@ def __init__( self.key_sharing_enabled = bridge.config["bridge.encryption.allow_key_sharing"] self.appservice_mode = bridge.config["bridge.encryption.appservice"] self.msc4190 = bridge.config["bridge.encryption.msc4190"] + self.self_sign = bridge.config["bridge.encryption.self_sign"] if self.appservice_mode: self.az.otk_handler = self.crypto.handle_as_otk_counts self.az.device_list_handler = self.crypto.handle_as_device_lists @@ -288,8 +290,18 @@ async def start(self) -> None: if not device_id: await self.crypto_store.put_device_id(self.client.device_id) self.log.debug(f"Logged in with new device ID {self.client.device_id}") + await self.crypto.share_keys() elif self.crypto.account.shared: await self._verify_keys_are_on_server() + else: + await self.crypto.share_keys() + if self.self_sign: + trust_state = await self.crypto.resolve_trust(self.crypto.own_identity) + if trust_state < TrustState.CROSS_SIGNED_UNTRUSTED: + recovery_key = await self.crypto.generate_recovery_key() + self.log.info(f"Generated recovery key and signed own device: {recovery_key}") + else: + self.log.debug(f"Own device is already verified ({trust_state})") if self.appservice_mode: self.log.info("End-to-bridge encryption support is enabled (appservice mode)") else: From 36b29aff83e479c657f161f4fb65c0cacdcc7e78 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 24 Sep 2025 00:07:23 +0300 Subject: [PATCH 205/218] Bump version to 0.21.0b5 --- mautrix/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 41283ebb..0fb5e1e6 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.21.0b4" +__version__ = "0.21.0b5" __author__ = "Tulir Asokan " __all__ = [ "api", From 9aced5042aac119e7204f9c7ac6077c1ebef5143 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 24 Sep 2025 00:20:35 +0300 Subject: [PATCH 206/218] Fix inserting database owner --- mautrix/util/async_db/database.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mautrix/util/async_db/database.py b/mautrix/util/async_db/database.py index ff73a8b2..6a17dc68 100644 --- a/mautrix/util/async_db/database.py +++ b/mautrix/util/async_db/database.py @@ -123,7 +123,9 @@ async def _check_owner(self) -> None: ) owner = await self.fetchval("SELECT owner FROM database_owner WHERE key=0") if not owner: - await self.execute("INSERT INTO database_owner (owner) VALUES ($1)", self.owner_name) + await self.execute( + "INSERT INTO database_owner (key, owner) VALUES (0, $1)", self.owner_name + ) elif owner != self.owner_name: raise DatabaseNotOwned(owner) From 1dd7d49f3cdf24c0b96c39859e412b4145b7de8d Mon Sep 17 00:00:00 2001 From: L0ric0 Date: Fri, 24 Oct 2025 13:54:11 +0200 Subject: [PATCH 207/218] Add option to change listening network interface of prometheus client (#181) --- mautrix/util/program.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mautrix/util/program.py b/mautrix/util/program.py index aa37b374..74752ca5 100644 --- a/mautrix/util/program.py +++ b/mautrix/util/program.py @@ -191,6 +191,7 @@ def start_prometheus(self) -> None: try: enabled = self.config["metrics.enabled"] listen_port = self.config["metrics.listen_port"] + hostname = self.config.get("metrics.hostname", "0.0.0.0") except KeyError: return if not enabled: @@ -200,7 +201,7 @@ def start_prometheus(self) -> None: "Metrics are enabled in config, but prometheus_client is not installed" ) return - prometheus.start_http_server(listen_port) + prometheus.start_http_server(listen_port, addr=hostname) def _run(self) -> None: signal.signal(signal.SIGINT, signal.default_int_handler) From e4ab162eb3848c7b522b715298d5ba249b130923 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 17 Nov 2025 00:23:09 +0200 Subject: [PATCH 208/218] Update variation selectors to Unicode 17 --- mautrix/util/variation_selector.json | 17 +++++++++++++++++ mautrix/util/variation_selector.py | 2 +- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/mautrix/util/variation_selector.json b/mautrix/util/variation_selector.json index 0205ce94..f27acb7b 100644 --- a/mautrix/util/variation_selector.json +++ b/mautrix/util/variation_selector.json @@ -31,9 +31,12 @@ "23CF": "⏏", "23E9": "⏩", "23EA": "⏪", + "23EB": "⏫", + "23EC": "⏬", "23ED": "⏭", "23EE": "⏮", "23EF": "⏯", + "23F0": "⏰", "23F1": "⏱", "23F2": "⏲", "23F3": "⏳", @@ -114,6 +117,7 @@ "26C4": "⛄", "26C5": "⛅", "26C8": "⛈", + "26CE": "⛎", "26CF": "⛏", "26D1": "⛑", "26D3": "⛓", @@ -132,8 +136,11 @@ "26FA": "⛺", "26FD": "⛽", "2702": "✂", + "2705": "✅", "2708": "✈", "2709": "✉", + "270A": "✊", + "270B": "✋", "270C": "✌", "270D": "✍", "270F": "✏", @@ -142,15 +149,25 @@ "2716": "✖", "271D": "✝", "2721": "✡", + "2728": "✨", "2733": "✳", "2734": "✴", "2744": "❄", "2747": "❇", + "274C": "❌", + "274E": "❎", "2753": "❓", + "2754": "❔", + "2755": "❕", "2757": "❗", "2763": "❣", "2764": "❤", + "2795": "➕", + "2796": "➖", + "2797": "➗", "27A1": "➡", + "27B0": "➰", + "27BF": "➿", "2934": "⤴", "2935": "⤵", "2B05": "⬅", diff --git a/mautrix/util/variation_selector.py b/mautrix/util/variation_selector.py index dec0554d..7ef2ad24 100644 --- a/mautrix/util/variation_selector.py +++ b/mautrix/util/variation_selector.py @@ -10,7 +10,7 @@ import aiohttp -EMOJI_VAR_URL = "https://www.unicode.org/Public/14.0.0/ucd/emoji/emoji-variation-sequences.txt" +EMOJI_VAR_URL = "https://www.unicode.org/Public/17.0.0/ucd/emoji/emoji-variation-sequences.txt" def read_data() -> dict[str, str]: From 8bfde09694b7ffa7fd61ccb2bc2031370d01ab72 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 17 Nov 2025 00:23:30 +0200 Subject: [PATCH 209/218] Update variation selector generator to use importlib.resources --- mautrix/util/variation_selector.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/mautrix/util/variation_selector.py b/mautrix/util/variation_selector.py index 7ef2ad24..e1430293 100644 --- a/mautrix/util/variation_selector.py +++ b/mautrix/util/variation_selector.py @@ -43,11 +43,12 @@ async def fetch_data() -> dict[str, str]: if __name__ == "__main__": import asyncio + import importlib.resources + import pathlib import sys - import pkg_resources - - path = pkg_resources.resource_filename("mautrix.util", "variation_selector.json") + path = importlib.resources.files("mautrix.util").joinpath("variation_selector.json") + assert isinstance(path, pathlib.Path) emojis = asyncio.run(fetch_data()) with open(path, "w") as file: json.dump(emojis, file, indent=" ", ensure_ascii=False) From a4198a6ec46a5ca049a88214b799f65e8cb7f04e Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 17 Nov 2025 00:23:50 +0200 Subject: [PATCH 210/218] Update linters, CI and python version --- .github/workflows/python-package.yml | 14 +++++++------- .pre-commit-config.yaml | 6 +++--- setup.py | 3 ++- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index aef7eb74..e6f77c9f 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -8,12 +8,12 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.10", "3.11", "3.12", "3.13"] + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} - name: Install libolm @@ -46,17 +46,17 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@v5 + - uses: actions/setup-python@v6 with: - python-version: "3.13" + python-version: "3.14" - uses: isort/isort-action@master with: sortPaths: "./mautrix" - uses: psf/black@stable with: src: "./mautrix" - version: "24.10.0" + version: "25.11.0" - name: pre-commit run: | pip install pre-commit diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 72cce792..b3bd2047 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v5.0.0 + rev: v6.0.0 hooks: - id: trailing-whitespace exclude_types: [markdown] @@ -8,13 +8,13 @@ repos: - id: check-yaml - id: check-added-large-files - repo: https://github.com/psf/black - rev: 24.10.0 + rev: 25.11.0 hooks: - id: black language_version: python3 files: ^mautrix/.*\.pyi?$ - repo: https://github.com/PyCQA/isort - rev: 5.13.2 + rev: 7.0.0 hooks: - id: isort files: ^mautrix/.*\.pyi?$ diff --git a/setup.py b/setup.py index 20b51098..23ac1cb1 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ ], extras_require={ "detect_mimetype": ["python-magic>=0.4.15,<0.5"], - "lint": ["black~=24.1", "isort"], + "lint": ["black~=25.1", "isort"], "test": ["pytest", "pytest-asyncio", *test_dependencies], "encryption": encryption_dependencies, }, @@ -46,6 +46,7 @@ "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", ], package_data={ From ada10a3f762f6a04fbccbf88554a1ff561cc332f Mon Sep 17 00:00:00 2001 From: Joe Groocock Date: Sun, 16 Nov 2025 23:49:42 +0100 Subject: [PATCH 211/218] Avoid setting avatar_url to "" (#171) Fixes an edge-case bug where calling `set_avatar_url("")` when the user doesn't have an avatar(`get_avatar_url()` returns `None`) caused a redundant PUT request to be made to nullify the already-empty avatar_url field in the user profile. This also fixes the errant `$user made no change` state events appearing in every room they are in when `set_avatar_url("")` is called. Signed-off-by: Joe Groocock --- mautrix/client/api/user_data.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/mautrix/client/api/user_data.py b/mautrix/client/api/user_data.py index 4c3d437a..9c380335 100644 --- a/mautrix/client/api/user_data.py +++ b/mautrix/client/api/user_data.py @@ -71,7 +71,7 @@ async def search_users(self, search_query: str, limit: int | None = 10) -> UserS # region 10.2 Profiles # API reference: https://matrix.org/docs/spec/client_server/r0.4.0.html#profiles - async def set_displayname(self, displayname: str, check_current: bool = True) -> None: + async def set_displayname(self, displayname: str | None, check_current: bool = True) -> None: """ Set the display name of the current user. @@ -81,7 +81,9 @@ async def set_displayname(self, displayname: str, check_current: bool = True) -> displayname: The new display name for the user. check_current: Whether or not to check if the displayname is already set. """ - if check_current and await self.get_displayname(self.mxid) == displayname: + if check_current and str_or_none(await self.get_displayname(self.mxid)) == str_or_none( + displayname + ): return await self.api.request( Method.PUT, @@ -112,7 +114,9 @@ async def get_displayname(self, user_id: UserID) -> str | None: except KeyError: return None - async def set_avatar_url(self, avatar_url: ContentURI, check_current: bool = True) -> None: + async def set_avatar_url( + self, avatar_url: ContentURI | None, check_current: bool = True + ) -> None: """ Set the avatar of the current user. @@ -122,7 +126,9 @@ async def set_avatar_url(self, avatar_url: ContentURI, check_current: bool = Tru avatar_url: The ``mxc://`` URI to the new avatar. check_current: Whether or not to check if the avatar is already set. """ - if check_current and await self.get_avatar_url(self.mxid) == avatar_url: + if check_current and str_or_none(await self.get_avatar_url(self.mxid)) == str_or_none( + avatar_url + ): return await self.api.request( Method.PUT, @@ -185,3 +191,10 @@ async def beeper_update_profile(self, custom_fields: dict[str, Any]) -> None: await self.api.request(Method.PATCH, Path.v3.profile[self.mxid], custom_fields) # endregion + + +def str_or_none(v: str | None) -> str | None: + """ + str_or_none empty string values to None + """ + return None if v == "" else v From db075d702a4ee21664e1a4098ef040159fefb9e4 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 17 Nov 2025 15:51:55 +0200 Subject: [PATCH 212/218] Bump version to 0.21.0 --- CHANGELOG.md | 7 ++++++- mautrix/__init__.py | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e663680..c7179c86 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,12 +1,17 @@ -## v0.21.0 (unreleased) +## v0.21.0 (2025-11-17) * *(event)* Added support for creator power in room v12+. * *(crypto)* Added support for generating and using recovery keys for verifying the active device. +* *(bridge)* Added config option for self-signing bot device. * *(bridge)* Removed check for login flows when using MSC4190 (thanks to [@meson800] in [#178]). +* *(client)* Changed `set_displayname` and `set_avatar_url` to avoid setting + empty strings if the value is already unset (thanks to [@frebib] in [#171]). +[@frebib]: https://github.com/frebib [@meson800]: https://github.com/meson800 +[#171]: https://github.com/mautrix/python/pull/171 [#178]: https://github.com/mautrix/python/pull/178 ## v0.20.8 (2025-06-01) diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 0fb5e1e6..c94995f9 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.21.0b5" +__version__ = "0.21.0" __author__ = "Tulir Asokan " __all__ = [ "api", From 93dea5a78864f91de07a84c5df60b67c012429f4 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 4 Feb 2026 21:43:02 +0200 Subject: [PATCH 213/218] Dispatch unknown decrypted to-device events to event handlers --- mautrix/crypto/machine.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/mautrix/crypto/machine.py b/mautrix/crypto/machine.py index 7585776d..60c65677 100644 --- a/mautrix/crypto/machine.py +++ b/mautrix/crypto/machine.py @@ -222,6 +222,11 @@ async def handle_to_device_event(self, evt: ToDeviceEvent) -> None: passed to the OlmMachine is syncing. You shouldn't need to call this yourself unless you do syncing in some manual way. """ + if isinstance(evt, DecryptedOlmEvent): + self.log.warning( + f"Dropping unexpected nested encrypted to-device event from {evt.sender}" + ) + return self.log.trace( f"Handling encrypted to-device event from {evt.content.sender_key} ({evt.sender})" ) @@ -230,6 +235,8 @@ async def handle_to_device_event(self, evt: ToDeviceEvent) -> None: await self._receive_room_key(decrypted_evt) elif decrypted_evt.type == EventType.FORWARDED_ROOM_KEY: await self._receive_forwarded_room_key(decrypted_evt) + else: + self.client.dispatch_event(decrypted_evt, source=evt.source) async def _receive_room_key(self, evt: DecryptedOlmEvent) -> None: # TODO nio had a comment saying "handle this better" From d7cb10b3ae62534fc4f3eb38052b858fc625efc2 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 15 Feb 2026 15:40:26 +0200 Subject: [PATCH 214/218] Catch invalid invites in sync --- mautrix/client/syncer.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/mautrix/client/syncer.py b/mautrix/client/syncer.py index 8183e142..4b8661a2 100644 --- a/mautrix/client/syncer.py +++ b/mautrix/client/syncer.py @@ -369,12 +369,18 @@ def handle_sync(self, data: JSON) -> list[asyncio.Task]: events: list[dict[str, JSON]] = room_data.get("invite_state", {}).get("events", []) for raw_event in events: raw_event["room_id"] = room_id - raw_invite = next( - raw_event - for raw_event in events - if raw_event.get("type", "") == "m.room.member" - and raw_event.get("state_key", "") == self.mxid - ) + try: + raw_invite = next( + raw_event + for raw_event in events + if raw_event.get("type", "") == "m.room.member" + and raw_event.get("state_key", "") == self.mxid + ) + except StopIteration: + self.log.warning( + f"Corrupted invite section in sync: no invite event present for {room_id}" + ) + continue # These aren't required by the spec, so make sure they're set raw_invite.setdefault("event_id", None) raw_invite.setdefault("origin_server_ts", int(time.time() * 1000)) From 2296ac74a77940b2ceeaf4dbd3d82cfe3ca61731 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 13 Mar 2026 14:03:10 +0200 Subject: [PATCH 215/218] Fix join rule types Closes #182 --- mautrix/types/event/state.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/mautrix/types/event/state.py b/mautrix/types/event/state.py index ef51cff4..5ffc855f 100644 --- a/mautrix/types/event/state.py +++ b/mautrix/types/event/state.py @@ -9,7 +9,7 @@ import attr from ..primitive import JSON, ContentURI, EventID, RoomAlias, RoomID, UserID -from ..util import Obj, SerializableAttrs, SerializableEnum, deserializer, field +from ..util import ExtensibleEnum, Obj, SerializableAttrs, SerializableEnum, deserializer, field from .base import BaseRoomEvent, BaseUnsigned from .encrypted import EncryptionAlgorithm from .type import EventType, RoomType @@ -159,16 +159,15 @@ class RoomAvatarStateEventContent(SerializableAttrs): url: Optional[ContentURI] = None -class JoinRule(SerializableEnum): +class JoinRule(ExtensibleEnum): PUBLIC = "public" KNOCK = "knock" RESTRICTED = "restricted" INVITE = "invite" - PRIVATE = "private" KNOCK_RESTRICTED = "knock_restricted" -class JoinRestrictionType(SerializableEnum): +class JoinRestrictionType(ExtensibleEnum): ROOM_MEMBERSHIP = "m.room_membership" From 89ebd5fff50ca22a8c6e4f554ad9aae393c96760 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 23 Mar 2026 14:48:30 +0200 Subject: [PATCH 216/218] Fix parsing /messages response --- mautrix/client/api/events.py | 6 ++---- mautrix/types/misc.py | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/mautrix/client/api/events.py b/mautrix/client/api/events.py index fd6b2c9d..bd415b9b 100644 --- a/mautrix/client/api/events.py +++ b/mautrix/client/api/events.py @@ -395,15 +395,13 @@ async def get_messages( try: return PaginatedMessages( content["start"], - content["end"], + content.get("end"), [Event.deserialize(event) for event in content["chunk"]], ) except KeyError: if "start" not in content: raise MatrixResponseError("`start` not in response.") - elif "end" not in content: - raise MatrixResponseError("`start` not in response.") - raise MatrixResponseError("`content` not in response.") + raise MatrixResponseError("`chunk` not in response.") except SerializerError as e: raise MatrixResponseError("Invalid events in response") from e diff --git a/mautrix/types/misc.py b/mautrix/types/misc.py index cf576c2b..5a07699c 100644 --- a/mautrix/types/misc.py +++ b/mautrix/types/misc.py @@ -106,7 +106,7 @@ class RoomDirectoryResponse(SerializableAttrs): PaginatedMessages = NamedTuple( - "PaginatedMessages", start=SyncToken, end=SyncToken, events=List[Event] + "PaginatedMessages", start=SyncToken, end=Optional[SyncToken], events=List[Event] ) From 8e0dcefb9f4b68f30b58031272f1048e36980422 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 12 Apr 2026 14:04:31 +0300 Subject: [PATCH 217/218] Fix transaction in abstract crypto store --- mautrix/crypto/store/abstract.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mautrix/crypto/store/abstract.py b/mautrix/crypto/store/abstract.py index 1bae832f..0916a2d7 100644 --- a/mautrix/crypto/store/abstract.py +++ b/mautrix/crypto/store/abstract.py @@ -7,6 +7,7 @@ from typing import AsyncContextManager, NamedTuple from abc import ABC, abstractmethod +from contextlib import asynccontextmanager from mautrix.types import ( CrossSigner, @@ -87,9 +88,10 @@ async def close(self) -> None: async def flush(self) -> None: """Flush the store. If all the methods persist data immediately, this can be a no-op.""" - async def transaction(self) -> AsyncContextManager[None]: + @asynccontextmanager + async def transaction(self) -> None: """Run a database transaction. If the store doesn't support transactions, this can be a no-op.""" - pass + yield @abstractmethod async def delete(self) -> None: From bea9e63142c1fc89820be3344941cb57808456b6 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 12 Apr 2026 14:10:39 +0300 Subject: [PATCH 218/218] Update black and isort --- .github/workflows/python-package.yml | 6 +- .pre-commit-config.yaml | 4 +- dev-requirements.txt | 4 +- mautrix/client/state_store/asyncpg/upgrade.py | 22 +-- mautrix/crypto/store/asyncpg/upgrade.py | 176 +++++++++--------- mautrix/util/async_db/database.py | 8 +- mautrix/util/async_db/upgrade.py | 8 +- 7 files changed, 109 insertions(+), 119 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index e6f77c9f..3a845672 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -11,7 +11,7 @@ jobs: python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v6 with: @@ -46,7 +46,7 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - uses: actions/setup-python@v6 with: python-version: "3.14" @@ -56,7 +56,7 @@ jobs: - uses: psf/black@stable with: src: "./mautrix" - version: "25.11.0" + version: "26.3.1" - name: pre-commit run: | pip install pre-commit diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b3bd2047..66065033 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,13 +8,13 @@ repos: - id: check-yaml - id: check-added-large-files - repo: https://github.com/psf/black - rev: 25.11.0 + rev: 26.3.1 hooks: - id: black language_version: python3 files: ^mautrix/.*\.pyi?$ - repo: https://github.com/PyCQA/isort - rev: 7.0.0 + rev: 8.0.1 hooks: - id: isort files: ^mautrix/.*\.pyi?$ diff --git a/dev-requirements.txt b/dev-requirements.txt index bb8c2a0a..dff6ee9d 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,3 +1,3 @@ pre-commit>=2.10.1,<3 -isort>=5.10.1,<6 -black>=24,<25 +isort>=8,<9 +black>=26,<27 diff --git a/mautrix/client/state_store/asyncpg/upgrade.py b/mautrix/client/state_store/asyncpg/upgrade.py index c1f32664..20b2e5b2 100644 --- a/mautrix/client/state_store/asyncpg/upgrade.py +++ b/mautrix/client/state_store/asyncpg/upgrade.py @@ -16,16 +16,16 @@ @upgrade_table.register(description="Latest revision", upgrades_to=4) async def upgrade_blank_to_v4(conn: Connection, scheme: Scheme) -> None: - await conn.execute( - """CREATE TABLE mx_room_state ( + await conn.execute(""" + CREATE TABLE mx_room_state ( room_id TEXT PRIMARY KEY, is_encrypted BOOLEAN, has_full_member_list BOOLEAN, encryption TEXT, power_levels TEXT, create_event TEXT - )""" - ) + ) + """) membership_check = "" if scheme != Scheme.SQLITE: await conn.execute( @@ -33,16 +33,16 @@ async def upgrade_blank_to_v4(conn: Connection, scheme: Scheme) -> None: ) else: membership_check = "CHECK (membership IN ('join', 'leave', 'invite', 'ban', 'knock'))" - await conn.execute( - f"""CREATE TABLE mx_user_profile ( + await conn.execute(f""" + CREATE TABLE mx_user_profile ( room_id TEXT, user_id TEXT, membership membership NOT NULL {membership_check}, displayname TEXT, avatar_url TEXT, PRIMARY KEY (room_id, user_id) - )""" - ) + ) + """) @upgrade_table.register(description="Stop using size-limited string fields") @@ -60,16 +60,14 @@ async def upgrade_v2(conn: Connection, scheme: Scheme) -> None: @upgrade_table.register(description="Mark rooms that need crypto state event resynced") async def upgrade_v3(conn: Connection) -> None: if await conn.table_exists("portal"): - await conn.execute( - """ + await conn.execute(""" INSERT INTO mx_room_state (room_id, encryption) SELECT portal.mxid, '{"resync":true}' FROM portal WHERE portal.encrypted=true AND portal.mxid IS NOT NULL ON CONFLICT (room_id) DO UPDATE SET encryption=excluded.encryption WHERE mx_room_state.encryption IS NULL - """ - ) + """) @upgrade_table.register(description="Add create event to room state cache") diff --git a/mautrix/crypto/store/asyncpg/upgrade.py b/mautrix/crypto/store/asyncpg/upgrade.py index e097c5d9..8d413858 100644 --- a/mautrix/crypto/store/asyncpg/upgrade.py +++ b/mautrix/crypto/store/asyncpg/upgrade.py @@ -18,32 +18,32 @@ @upgrade_table.register(description="Latest revision", upgrades_to=10) async def upgrade_blank_to_latest(conn: Connection) -> None: - await conn.execute( - """CREATE TABLE IF NOT EXISTS crypto_account ( + await conn.execute(""" + CREATE TABLE IF NOT EXISTS crypto_account ( account_id TEXT PRIMARY KEY, device_id TEXT NOT NULL, shared BOOLEAN NOT NULL, sync_token TEXT NOT NULL, account bytea NOT NULL - )""" - ) - await conn.execute( - """CREATE TABLE IF NOT EXISTS crypto_message_index ( + ) + """) + await conn.execute(""" + CREATE TABLE IF NOT EXISTS crypto_message_index ( sender_key CHAR(43), session_id CHAR(43), "index" INTEGER, event_id TEXT NOT NULL, timestamp BIGINT NOT NULL, PRIMARY KEY (sender_key, session_id, "index") - )""" - ) - await conn.execute( - """CREATE TABLE IF NOT EXISTS crypto_tracked_user ( + ) + """) + await conn.execute(""" + CREATE TABLE IF NOT EXISTS crypto_tracked_user ( user_id TEXT PRIMARY KEY - )""" - ) - await conn.execute( - """CREATE TABLE IF NOT EXISTS crypto_device ( + ) + """) + await conn.execute(""" + CREATE TABLE IF NOT EXISTS crypto_device ( user_id TEXT, device_id TEXT, identity_key CHAR(43) NOT NULL, @@ -52,10 +52,10 @@ async def upgrade_blank_to_latest(conn: Connection) -> None: deleted BOOLEAN NOT NULL, name TEXT NOT NULL, PRIMARY KEY (user_id, device_id) - )""" - ) - await conn.execute( - """CREATE TABLE IF NOT EXISTS crypto_olm_session ( + ) + """) + await conn.execute(""" + CREATE TABLE IF NOT EXISTS crypto_olm_session ( account_id TEXT, session_id CHAR(43), sender_key CHAR(43) NOT NULL, @@ -64,10 +64,10 @@ async def upgrade_blank_to_latest(conn: Connection) -> None: last_decrypted timestamp NOT NULL, last_encrypted timestamp NOT NULL, PRIMARY KEY (account_id, session_id) - )""" - ) - await conn.execute( - """CREATE TABLE IF NOT EXISTS crypto_megolm_inbound_session ( + ) + """) + await conn.execute(""" + CREATE TABLE IF NOT EXISTS crypto_megolm_inbound_session ( account_id TEXT, session_id CHAR(43), sender_key CHAR(43) NOT NULL, @@ -83,10 +83,10 @@ async def upgrade_blank_to_latest(conn: Connection) -> None: max_messages INTEGER, is_scheduled BOOLEAN NOT NULL DEFAULT false, PRIMARY KEY (account_id, session_id) - )""" - ) - await conn.execute( - """CREATE TABLE IF NOT EXISTS crypto_megolm_outbound_session ( + ) + """) + await conn.execute(""" + CREATE TABLE IF NOT EXISTS crypto_megolm_outbound_session ( account_id TEXT, room_id TEXT, session_id CHAR(43) NOT NULL UNIQUE, @@ -98,10 +98,10 @@ async def upgrade_blank_to_latest(conn: Connection) -> None: created_at timestamp NOT NULL, last_used timestamp NOT NULL, PRIMARY KEY (account_id, room_id) - )""" - ) - await conn.execute( - """CREATE TABLE crypto_cross_signing_keys ( + ) + """) + await conn.execute(""" + CREATE TABLE crypto_cross_signing_keys ( user_id TEXT, usage TEXT, key CHAR(43) NOT NULL, @@ -109,18 +109,18 @@ async def upgrade_blank_to_latest(conn: Connection) -> None: first_seen_key CHAR(43) NOT NULL, PRIMARY KEY (user_id, usage) - )""" - ) - await conn.execute( - """CREATE TABLE crypto_cross_signing_signatures ( + ) + """) + await conn.execute(""" + CREATE TABLE crypto_cross_signing_signatures ( signed_user_id TEXT, signed_key TEXT, signer_user_id TEXT, signer_key TEXT, signature CHAR(88) NOT NULL, PRIMARY KEY (signed_user_id, signed_key, signer_user_id, signer_key) - )""" - ) + ) + """) @upgrade_table.register(description="Add account_id primary key column") @@ -130,17 +130,17 @@ async def upgrade_v2(conn: Connection, scheme: Scheme) -> None: await conn.execute("DROP TABLE crypto_olm_session") await conn.execute("DROP TABLE crypto_megolm_inbound_session") await conn.execute("DROP TABLE crypto_megolm_outbound_session") - await conn.execute( - """CREATE TABLE crypto_account ( + await conn.execute(""" + CREATE TABLE crypto_account ( account_id VARCHAR(255) PRIMARY KEY, device_id VARCHAR(255) NOT NULL, shared BOOLEAN NOT NULL, sync_token TEXT NOT NULL, account bytea NOT NULL - )""" - ) - await conn.execute( - """CREATE TABLE crypto_olm_session ( + ) + """) + await conn.execute(""" + CREATE TABLE crypto_olm_session ( account_id VARCHAR(255), session_id CHAR(43), sender_key CHAR(43) NOT NULL, @@ -148,10 +148,10 @@ async def upgrade_v2(conn: Connection, scheme: Scheme) -> None: created_at timestamp NOT NULL, last_used timestamp NOT NULL, PRIMARY KEY (account_id, session_id) - )""" - ) - await conn.execute( - """CREATE TABLE crypto_megolm_inbound_session ( + ) + """) + await conn.execute(""" + CREATE TABLE crypto_megolm_inbound_session ( account_id VARCHAR(255), session_id CHAR(43), sender_key CHAR(43) NOT NULL, @@ -160,10 +160,10 @@ async def upgrade_v2(conn: Connection, scheme: Scheme) -> None: session bytea NOT NULL, forwarding_chains TEXT NOT NULL, PRIMARY KEY (account_id, session_id) - )""" - ) - await conn.execute( - """CREATE TABLE crypto_megolm_outbound_session ( + ) + """) + await conn.execute(""" + CREATE TABLE crypto_megolm_outbound_session ( account_id VARCHAR(255), room_id VARCHAR(255), session_id CHAR(43) NOT NULL UNIQUE, @@ -175,8 +175,8 @@ async def upgrade_v2(conn: Connection, scheme: Scheme) -> None: created_at timestamp NOT NULL, last_used timestamp NOT NULL, PRIMARY KEY (account_id, room_id) - )""" - ) + ) + """) else: async def add_account_id_column(table: str, pkey_columns: list[str]) -> None: @@ -233,25 +233,25 @@ async def upgrade_v4(conn: Connection, scheme: Scheme) -> None: @upgrade_table.register(description="Add cross-signing key and signature caches") async def upgrade_v5(conn: Connection) -> None: - await conn.execute( - """CREATE TABLE crypto_cross_signing_keys ( + await conn.execute(""" + CREATE TABLE crypto_cross_signing_keys ( user_id TEXT, usage TEXT, key CHAR(43), first_seen_key CHAR(43), PRIMARY KEY (user_id, usage) - )""" - ) - await conn.execute( - """CREATE TABLE crypto_cross_signing_signatures ( + ) + """) + await conn.execute(""" + CREATE TABLE crypto_cross_signing_signatures ( signed_user_id TEXT, signed_key TEXT, signer_user_id TEXT, signer_key TEXT, signature TEXT, PRIMARY KEY (signed_user_id, signed_key, signer_user_id, signer_key) - )""" - ) + ) + """) @upgrade_table.register(description="Update trust state values") @@ -322,27 +322,25 @@ async def upgrade_v9_postgres(conn: Connection) -> None: async def upgrade_v9_sqlite(conn: Connection) -> None: await conn.execute("PRAGMA foreign_keys = OFF") async with conn.transaction(): - await conn.execute( - """CREATE TABLE new_crypto_account ( + await conn.execute(""" + CREATE TABLE new_crypto_account ( account_id TEXT PRIMARY KEY, device_id TEXT NOT NULL, shared BOOLEAN NOT NULL, sync_token TEXT NOT NULL, account bytea NOT NULL - )""" - ) - await conn.execute( - """ + ) + """) + await conn.execute(""" INSERT INTO new_crypto_account (account_id, device_id, shared, sync_token, account) SELECT account_id, COALESCE(device_id, ''), shared, sync_token, account FROM crypto_account - """ - ) + """) await conn.execute("DROP TABLE crypto_account") await conn.execute("ALTER TABLE new_crypto_account RENAME TO crypto_account") - await conn.execute( - """CREATE TABLE new_crypto_megolm_inbound_session ( + await conn.execute(""" + CREATE TABLE new_crypto_megolm_inbound_session ( account_id TEXT, session_id CHAR(43), sender_key CHAR(43) NOT NULL, @@ -353,10 +351,9 @@ async def upgrade_v9_sqlite(conn: Connection) -> None: withheld_code TEXT, withheld_reason TEXT, PRIMARY KEY (account_id, session_id) - )""" - ) - await conn.execute( - """ + ) + """) + await conn.execute(""" INSERT INTO new_crypto_megolm_inbound_session ( account_id, session_id, sender_key, signing_key, room_id, session, forwarding_chains @@ -364,8 +361,7 @@ async def upgrade_v9_sqlite(conn: Connection) -> None: SELECT account_id, session_id, sender_key, signing_key, room_id, session, forwarding_chains FROM crypto_megolm_inbound_session - """ - ) + """) await conn.execute("DROP TABLE crypto_megolm_inbound_session") await conn.execute( "ALTER TABLE new_crypto_megolm_inbound_session RENAME TO crypto_megolm_inbound_session" @@ -373,8 +369,8 @@ async def upgrade_v9_sqlite(conn: Connection) -> None: await conn.execute("UPDATE crypto_megolm_outbound_session SET max_age=max_age*1000") - await conn.execute( - """CREATE TABLE new_crypto_cross_signing_keys ( + await conn.execute(""" + CREATE TABLE new_crypto_cross_signing_keys ( user_id TEXT, usage TEXT, key CHAR(43) NOT NULL, @@ -382,41 +378,37 @@ async def upgrade_v9_sqlite(conn: Connection) -> None: first_seen_key CHAR(43) NOT NULL, PRIMARY KEY (user_id, usage) - )""" - ) - await conn.execute( - """ + ) + """) + await conn.execute(""" INSERT INTO new_crypto_cross_signing_keys (user_id, usage, key, first_seen_key) SELECT user_id, usage, key, COALESCE(first_seen_key, key) FROM crypto_cross_signing_keys WHERE key IS NOT NULL - """ - ) + """) await conn.execute("DROP TABLE crypto_cross_signing_keys") await conn.execute( "ALTER TABLE new_crypto_cross_signing_keys RENAME TO crypto_cross_signing_keys" ) - await conn.execute( - """CREATE TABLE new_crypto_cross_signing_signatures ( + await conn.execute(""" + CREATE TABLE new_crypto_cross_signing_signatures ( signed_user_id TEXT, signed_key TEXT, signer_user_id TEXT, signer_key TEXT, signature CHAR(88) NOT NULL, PRIMARY KEY (signed_user_id, signed_key, signer_user_id, signer_key) - )""" - ) - await conn.execute( - """ + ) + """) + await conn.execute(""" INSERT INTO new_crypto_cross_signing_signatures ( signed_user_id, signed_key, signer_user_id, signer_key, signature ) SELECT signed_user_id, signed_key, signer_user_id, signer_key, signature FROM crypto_cross_signing_signatures WHERE signature IS NOT NULL - """ - ) + """) await conn.execute("DROP TABLE crypto_cross_signing_signatures") await conn.execute( "ALTER TABLE new_crypto_cross_signing_signatures " diff --git a/mautrix/util/async_db/database.py b/mautrix/util/async_db/database.py index 6a17dc68..b5128b74 100644 --- a/mautrix/util/async_db/database.py +++ b/mautrix/util/async_db/database.py @@ -115,12 +115,12 @@ async def _check_foreign_tables(self) -> None: raise ForeignTablesFound("found roomserver_rooms likely belonging to Dendrite") async def _check_owner(self) -> None: - await self.execute( - """CREATE TABLE IF NOT EXISTS database_owner ( + await self.execute(""" + CREATE TABLE IF NOT EXISTS database_owner ( key INTEGER PRIMARY KEY DEFAULT 0, owner TEXT NOT NULL - )""" - ) + ) + """) owner = await self.fetchval("SELECT owner FROM database_owner WHERE key=0") if not owner: await self.execute( diff --git a/mautrix/util/async_db/upgrade.py b/mautrix/util/async_db/upgrade.py index 9e593ece..c084d28b 100644 --- a/mautrix/util/async_db/upgrade.py +++ b/mautrix/util/async_db/upgrade.py @@ -97,11 +97,11 @@ async def _save_version(self, conn: LoggingConnection, version: int) -> None: await conn.execute(f"INSERT INTO {self.version_table_name} (version) VALUES ($1)", version) async def upgrade(self, db: async_db.Database) -> None: - await db.execute( - f"""CREATE TABLE IF NOT EXISTS {self.version_table_name} ( + await db.execute(f""" + CREATE TABLE IF NOT EXISTS {self.version_table_name} ( version INTEGER PRIMARY KEY - )""" - ) + ) + """) row = await db.fetchrow(f"SELECT version FROM {self.version_table_name} LIMIT 1") version = row["version"] if row else 0