From c2438691e3b9a9d054667a94c8373badd777c215 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 17 Feb 2022 21:07:47 +0200 Subject: [PATCH 001/456] Fix type hint in SimpleLock --- mautrix/util/simple_lock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/util/simple_lock.py b/mautrix/util/simple_lock.py index 8cd71d34..ecfa1e1c 100644 --- a/mautrix/util/simple_lock.py +++ b/mautrix/util/simple_lock.py @@ -36,7 +36,7 @@ def __aexit__(self, exc_type, exc_val, exc_tb) -> None: def locked(self) -> bool: return not self._event.is_set() - async def wait(self, task: Optional[str] = None) -> None: + async def wait(self, task: str | None = None) -> None: if not self._event.is_set(): if self.log and self.message: self.log.debug(self.message, task) From 94040521c0b535189be567bc1a622986eb45b42b Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 20 Feb 2022 13:53:41 +0200 Subject: [PATCH 002/456] Remove legacy htmlparser file --- mautrix/util/formatter/html_reader_htmlparser.py | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 mautrix/util/formatter/html_reader_htmlparser.py diff --git a/mautrix/util/formatter/html_reader_htmlparser.py b/mautrix/util/formatter/html_reader_htmlparser.py deleted file mode 100644 index 6461a80d..00000000 --- a/mautrix/util/formatter/html_reader_htmlparser.py +++ /dev/null @@ -1,2 +0,0 @@ -# TODO: remove this file in v0.15 -from .html_reader import HTMLNode, read_html From 23a174755a724f4535338df7a5b089b4b4eb642f Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 20 Feb 2022 14:01:21 +0200 Subject: [PATCH 003/456] Remove typing_extensions usage --- mautrix/api.py | 8 +------- mautrix/client/api/modules/media_repository.py | 8 +------- mautrix/client/state_store/memory.py | 8 +------- mautrix/crypto/base.py | 8 +------- mautrix/util/file_store.py | 8 +------- requirements.txt | 1 - 6 files changed, 5 insertions(+), 36 deletions(-) diff --git a/mautrix/api.py b/mautrix/api.py index 56023c20..1428e2fb 100644 --- a/mautrix/api.py +++ b/mautrix/api.py @@ -5,7 +5,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import annotations -from typing import ClassVar, Mapping +from typing import ClassVar, Literal, Mapping from enum import Enum from json.decoder import JSONDecodeError from time import time @@ -14,7 +14,6 @@ import json import logging import platform -import sys from aiohttp import ClientSession, __version__ as aiohttp_version from aiohttp.client_exceptions import ClientError, ContentTypeError @@ -25,11 +24,6 @@ from mautrix.util.logging import TraceLogger from mautrix.util.opt_prometheus import Counter -if sys.version_info >= (3, 8): - from typing import Literal -else: - from typing_extensions import Literal - if __optional_imports__: # Safe to import, but it's not actually needed, so don't force-import the whole types module. from mautrix.types import JSON diff --git a/mautrix/client/api/modules/media_repository.py b/mautrix/client/api/modules/media_repository.py index 0cc6a2b3..6b0c6d3c 100644 --- a/mautrix/client/api/modules/media_repository.py +++ b/mautrix/client/api/modules/media_repository.py @@ -5,8 +5,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import annotations -from typing import AsyncIterable -import sys +from typing import AsyncIterable, Literal from mautrix import __optional_imports__ from mautrix.api import MediaPath, Method @@ -15,11 +14,6 @@ from ..base import BaseClientAPI -if sys.version_info >= (3, 8): - from typing import Literal -else: - from typing_extensions import Literal - try: from mautrix.util import magic except ImportError: diff --git a/mautrix/client/state_store/memory.py b/mautrix/client/state_store/memory.py index 6328c00d..29f8744b 100644 --- a/mautrix/client/state_store/memory.py +++ b/mautrix/client/state_store/memory.py @@ -5,8 +5,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import annotations -from typing import Any -import sys +from typing import Any, TypedDict from mautrix.types import ( Member, @@ -20,11 +19,6 @@ from .abstract import StateStore -if sys.version_info >= (3, 8): - from typing import TypedDict -else: - from typing_extensions import TypedDict - class SerializedStateStore(TypedDict): members: dict[RoomID, dict[UserID, Any]] diff --git a/mautrix/crypto/base.py b/mautrix/crypto/base.py index 4b27b4c9..29fb554a 100644 --- a/mautrix/crypto/base.py +++ b/mautrix/crypto/base.py @@ -5,11 +5,10 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import annotations -from typing import Any, Awaitable, Callable, Dict +from typing import Any, Awaitable, Callable, Dict, TypedDict import asyncio import functools import json -import sys import olm @@ -27,11 +26,6 @@ from .. import client as cli, crypto -if sys.version_info >= (3, 8): - from typing import TypedDict -else: - from typing_extensions import TypedDict - class SignedObject(TypedDict): signatures: Dict[UserID, Dict[str, str]] diff --git a/mautrix/util/file_store.py b/mautrix/util/file_store.py index 1ba88c5b..c99298fe 100644 --- a/mautrix/util/file_store.py +++ b/mautrix/util/file_store.py @@ -5,19 +5,13 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import annotations -from typing import IO, Any +from typing import IO, Any, Protocol from abc import ABC, abstractmethod from pathlib import Path import json import pickle -import sys import time -if sys.version_info >= (3, 8): - from typing import Protocol -else: - from typing_extensions import Protocol - class Filer(Protocol): def dump(self, obj: Any, file: IO) -> None: diff --git a/requirements.txt b/requirements.txt index 77e93ba7..e88b12f0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,3 @@ aiohttp attrs yarl -typing_extensions; python_version<"3.8" From b1879d76ca70b87d4d2981b88eda5913065cdc12 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 20 Feb 2022 14:06:52 +0200 Subject: [PATCH 004/456] Switch to v3 paths everywhere --- mautrix/api.py | 28 ++++++--------- mautrix/appservice/api/intent.py | 2 +- mautrix/bridge/custom_puppet.py | 2 +- mautrix/bridge/notification_disabler.py | 2 +- mautrix/bridge/user.py | 4 +-- mautrix/client/api/authentication.py | 15 ++++---- mautrix/client/api/events.py | 20 +++++------ mautrix/client/api/filtering.py | 4 +-- mautrix/client/api/modules/account_data.py | 4 +-- mautrix/client/api/modules/crypto.py | 8 ++--- .../client/api/modules/media_repository.py | 6 ++-- mautrix/client/api/modules/misc.py | 10 +++--- mautrix/client/api/modules/push_rules.py | 9 +++-- mautrix/client/api/modules/room_tag.py | 6 ++-- mautrix/client/api/rooms.py | 34 +++++++++---------- mautrix/client/api/user_data.py | 12 +++---- 16 files changed, 82 insertions(+), 84 deletions(-) diff --git a/mautrix/api.py b/mautrix/api.py index 1428e2fb..94503f8a 100644 --- a/mautrix/api.py +++ b/mautrix/api.py @@ -46,9 +46,8 @@ class APIPath(Enum): These don't start with a slash so they can be used nicely with yarl. """ - CLIENT = "_matrix/client/r0" - CLIENT_UNSTABLE = "_matrix/client/unstable" - MEDIA = "_matrix/media/r0" + CLIENT = "_matrix/client" + MEDIA = "_matrix/media" SYNAPSE_ADMIN = "_synapse/admin" def __repr__(self): @@ -82,8 +81,8 @@ class PathBuilder: >>> from mautrix.api import Path >>> room_id = "!foo:example.com" >>> event_id = "$bar:example.com" - >>> str(Path.rooms[room_id].event[event_id]) - "_matrix/client/r0/rooms/%21foo%3Aexample.com/event/%24bar%3Aexample.com" + >>> str(Path.v3.rooms[room_id].event[event_id]) + "_matrix/client/v3/rooms/%21foo%3Aexample.com/event/%24bar%3Aexample.com" """ def __init__(self, path: str | APIPath = "") -> None: @@ -126,33 +125,28 @@ def __getitem__(self, append: str | int) -> PathBuilder: ClientPath = PathBuilder(APIPath.CLIENT) ClientPath.__doc__ = """ -A path builder with the standard client r0 prefix ( ``/_matrix/client/r0``, :attr:`APIPath.CLIENT`) +A path builder with the standard client prefix ( ``/_matrix/client``, :attr:`APIPath.CLIENT`). """ Path = PathBuilder(APIPath.CLIENT) Path.__doc__ = """A shorter alias for :attr:`ClientPath`""" -UnstableClientPath = PathBuilder(APIPath.CLIENT_UNSTABLE) -UnstableClientPath.__doc__ = """ -A path builder for client endpoints that haven't reached the spec yet -(``/_matrix/client/unstable``, :attr:`APIPath.CLIENT_UNSTABLE`) -""" MediaPath = PathBuilder(APIPath.MEDIA) MediaPath.__doc__ = """ -A path builder for standard media r0 paths (``/_matrix/media/r0``, :attr:`APIPath.MEDIA`) +A path builder with the standard media prefix (``/_matrix/media``, :attr:`APIPath.MEDIA`) Examples: >>> from mautrix.api import MediaPath - >>> str(MediaPath.config) - "_matrix/media/r0/config" + >>> str(MediaPath.v3.config) + "_matrix/media/v3/config" """ SynapseAdminPath = PathBuilder(APIPath.SYNAPSE_ADMIN) SynapseAdminPath.__doc__ = """ A path builder for synapse-specific admin API paths -(``/_synapse/admin/v1``, :attr:`APIPath.SYNAPSE_ADMIN`) +(``/_synapse/admin``, :attr:`APIPath.SYNAPSE_ADMIN`) Examples: >>> from mautrix.api import SynapseAdminPath >>> user_id = "@user:example.com" - >>> str(SynapseAdminPath.users[user_id]/login) + >>> str(SynapseAdminPath.v1.users[user_id]/login) "_synapse/admin/v1/users/%40user%3Aexample.com/login" """ @@ -268,7 +262,7 @@ def _log_request( return log_content = content if not isinstance(content, bytes) else f"<{len(content)} bytes>" as_user = query_params.get("user_id", None) - level = 1 if path == Path.sync else 5 + level = 1 if path == Path.v3.sync else 5 self.log.log( level, f"{method}#{req_id} /{path} {log_content}".strip(" "), diff --git a/mautrix/appservice/api/intent.py b/mautrix/appservice/api/intent.py index 29d41cd4..394c1963 100644 --- a/mautrix/appservice/api/intent.py +++ b/mautrix/appservice/api/intent.py @@ -538,7 +538,7 @@ def _register(self) -> Awaitable[dict]: "inhibit_login": True, } query_params = {"kind": "user"} - return self.api.request(Method.POST, Path.register, content, query_params=query_params) + return self.api.request(Method.POST, Path.v3.register, content, query_params=query_params) async def ensure_registered(self) -> None: """ diff --git a/mautrix/bridge/custom_puppet.py b/mautrix/bridge/custom_puppet.py index b3c15c19..5bdb6441 100644 --- a/mautrix/bridge/custom_puppet.py +++ b/mautrix/bridge/custom_puppet.py @@ -170,7 +170,7 @@ async def _login_with_shared_secret(cls, mxid: UserID) -> str: else: raise AutologinError(f"No homeserver URL configured for {server}") password = hmac.new(secret, mxid.encode("utf-8"), hashlib.sha512).hexdigest() - url = base_url / str(Path.login) + url = base_url / str(Path.v3.login) resp = await cls.az.http_session.post( url, data=json.dumps( diff --git a/mautrix/bridge/notification_disabler.py b/mautrix/bridge/notification_disabler.py index 51f52ddf..df37543e 100644 --- a/mautrix/bridge/notification_disabler.py +++ b/mautrix/bridge/notification_disabler.py @@ -34,7 +34,7 @@ def __init__(self, room_id: RoomID, user: BaseUser) -> None: @property def _path(self) -> PathBuilder: - return Path.pushrules["global"].override[ + return Path.v3.pushrules["global"].override[ f"net.maunium.silence_while_backfilling:{self.room_id}" ] diff --git a/mautrix/bridge/user.py b/mautrix/bridge/user.py index 318ee82a..cc5ba495 100644 --- a/mautrix/bridge/user.py +++ b/mautrix/bridge/user.py @@ -12,7 +12,7 @@ import logging import time -from mautrix.api import Method, UnstableClientPath +from mautrix.api import Method, Path from mautrix.appservice import AppService from mautrix.errors import MNotFound from mautrix.types import EventID, EventType, Membership, MessageType, RoomID, UserID @@ -28,7 +28,7 @@ from .. import bridge as br -AsmuxPath = UnstableClientPath["com.beeper.asmux"] +AsmuxPath = Path.unstable["com.beeper.asmux"] class WrappedTask(NamedTuple): diff --git a/mautrix/client/api/authentication.py b/mautrix/client/api/authentication.py index e485a0b9..36e134fc 100644 --- a/mautrix/client/api/authentication.py +++ b/mautrix/client/api/authentication.py @@ -8,6 +8,7 @@ from mautrix.api import Method, Path from mautrix.errors import MatrixResponseError from mautrix.types import ( + DeviceID, LoginFlowList, LoginResponse, LoginType, @@ -40,7 +41,7 @@ async def get_login_flows(self) -> LoginFlowList: Returns: The list of login flows that the homeserver supports. """ - resp = await self.api.request(Method.GET, Path.login) + resp = await self.api.request(Method.GET, Path.v3.login) try: return LoginFlowList.deserialize(resp) except KeyError: @@ -93,7 +94,7 @@ async def login( kwargs["device_id"] = self.device_id resp = await self.api.request( Method.POST, - Path.login, + Path.v3.login, { "type": str(login_type), "identifier": identifier.serialize(), @@ -127,10 +128,10 @@ async def logout(self, clear_access_token: bool = True) -> None: Args: clear_access_token: Whether or not mautrix-python should forget the stored access token. """ - await self.api.request(Method.POST, Path.logout) + await self.api.request(Method.POST, Path.v3.logout) if clear_access_token: self.api.token = "" - self.device_id = "" + self.device_id = DeviceID("") async def logout_all(self, clear_access_token: bool = True) -> None: """ @@ -151,10 +152,10 @@ async def logout_all(self, clear_access_token: bool = True) -> None: Args: clear_access_token: Whether or not mautrix-python should forget the stored access token. """ - await self.api.request(Method.POST, Path.logout.all) + await self.api.request(Method.POST, Path.v3.logout.all) if clear_access_token: self.api.token = "" - self.device_id = "" + self.device_id = DeviceID("") # endregion @@ -170,7 +171,7 @@ async def whoami(self) -> WhoamiResponse: Returns: The user ID and device ID of the current user. """ - resp = await self.api.request(Method.GET, Path.account.whoami) + resp = await self.api.request(Method.GET, Path.v3.account.whoami) return WhoamiResponse.deserialize(resp) # endregion diff --git a/mautrix/client/api/events.py b/mautrix/client/api/events.py index 14c06079..676ccddd 100644 --- a/mautrix/client/api/events.py +++ b/mautrix/client/api/events.py @@ -97,7 +97,7 @@ def sync( if set_presence: request["set_presence"] = str(set_presence) return self.api.request( - Method.GET, Path.sync, query_params=request, retry_count=0, metrics_method="sync" + Method.GET, Path.v3.sync, query_params=request, retry_count=0, metrics_method="sync" ) # endregion @@ -119,7 +119,7 @@ async def get_event(self, room_id: RoomID, event_id: EventID) -> Event: The event. """ content = await self.api.request( - Method.GET, Path.rooms[room_id].event[event_id], metrics_method="getEvent" + Method.GET, Path.v3.rooms[room_id].event[event_id], metrics_method="getEvent" ) try: return Event.deserialize(content) @@ -149,7 +149,7 @@ async def get_state_event( """ content = await self.api.request( Method.GET, - Path.rooms[room_id].state[event_type][state_key], + Path.v3.rooms[room_id].state[event_type][state_key], metrics_method="getStateEvent", ) try: @@ -172,7 +172,7 @@ async def get_state(self, room_id: RoomID) -> list[StateEvent]: A list of state events with the most recent of each event_type/state_key pair. """ content = await self.api.request( - Method.GET, Path.rooms[room_id].state, metrics_method="getState" + Method.GET, Path.v3.rooms[room_id].state, metrics_method="getState" ) try: return [StateEvent.deserialize(event) for event in content] @@ -215,7 +215,7 @@ async def get_members( query["not_membership"] = not_membership.value content = await self.api.request( Method.GET, - Path.rooms[room_id].members, + Path.v3.rooms[room_id].members, query_params=query, metrics_method="getMembers", ) @@ -245,7 +245,7 @@ async def get_joined_members(self, room_id: RoomID) -> dict[UserID, Member]: https://spec.matrix.org/v1.1/client-server-api/#get_matrixclientv3roomsroomidmembers """ content = await self.api.request( - Method.GET, Path.rooms[room_id].joined_members, metrics_method="getJoinedMembers" + Method.GET, Path.v3.rooms[room_id].joined_members, metrics_method="getJoinedMembers" ) try: return { @@ -306,7 +306,7 @@ async def get_messages( } content = await self.api.request( Method.GET, - Path.rooms[room_id].messages, + Path.v3.rooms[room_id].messages, query_params=query_params, metrics_method="getMessages", ) @@ -358,7 +358,7 @@ async def send_state_event( content = content.serialize() if isinstance(content, Serializable) else content resp = await self.api.request( Method.PUT, - Path.rooms[room_id].state[event_type][state_key], + Path.v3.rooms[room_id].state[event_type][state_key], content, **kwargs, metrics_method="sendStateEvent", @@ -398,7 +398,7 @@ async def send_message_event( raise ValueError("Room ID not given") elif not event_type: raise ValueError("Event type not given") - url = Path.rooms[room_id].send[event_type][txn_id or self.api.get_txn_id()] + url = Path.v3.rooms[room_id].send[event_type][txn_id or self.api.get_txn_id()] content = content.serialize() if isinstance(content, Serializable) else content resp = await self.api.request( Method.PUT, url, content, **kwargs, metrics_method="sendMessageEvent" @@ -669,7 +669,7 @@ async def redact( Returns: The ID of the event that was sent to redact the other event. """ - url = Path.rooms[room_id].redact[event_id][self.api.get_txn_id()] + url = Path.v3.rooms[room_id].redact[event_id][self.api.get_txn_id()] content = extra_content or {} if reason: content["reason"] = reason diff --git a/mautrix/client/api/filtering.py b/mautrix/client/api/filtering.py index bcaa3361..5881ad07 100644 --- a/mautrix/client/api/filtering.py +++ b/mautrix/client/api/filtering.py @@ -32,7 +32,7 @@ async def get_filter(self, filter_id: FilterID) -> Filter: Returns: The filter data. """ - content = await self.api.request(Method.GET, Path.user[self.mxid].filter[filter_id]) + content = await self.api.request(Method.GET, Path.v3.user[self.mxid].filter[filter_id]) return Filter.deserialize(content) async def create_filter(self, filter_params: Filter) -> FilterID: @@ -49,7 +49,7 @@ async def create_filter(self, filter_params: Filter) -> FilterID: """ resp = await self.api.request( Method.POST, - Path.user[self.mxid].filter, + Path.v3.user[self.mxid].filter, filter_params.serialize() if isinstance(filter_params, Serializable) else filter_params, diff --git a/mautrix/client/api/modules/account_data.py b/mautrix/client/api/modules/account_data.py index ecf7764d..11c09e0e 100644 --- a/mautrix/client/api/modules/account_data.py +++ b/mautrix/client/api/modules/account_data.py @@ -33,7 +33,7 @@ async def get_account_data(self, type: EventType | str, room_id: RoomID | None = """ if isinstance(type, EventType) and not type.is_account_data: raise ValueError("Event type is not an account data event type") - base_path = Path.user[self.mxid] + base_path = Path.v3.user[self.mxid] if room_id: base_path = base_path.rooms[room_id] return await self.api.request(Method.GET, base_path.account_data[type]) @@ -56,7 +56,7 @@ async def set_account_data( """ if isinstance(type, EventType) and not type.is_account_data: raise ValueError("Event type is not an account data event type") - base_path = Path.user[self.mxid] + base_path = Path.v3.user[self.mxid] if room_id: base_path = base_path.rooms[room_id] await self.api.request( diff --git a/mautrix/client/api/modules/crypto.py b/mautrix/client/api/modules/crypto.py index bcec60d3..9549efd9 100644 --- a/mautrix/client/api/modules/crypto.py +++ b/mautrix/client/api/modules/crypto.py @@ -47,7 +47,7 @@ async def send_to_device( raise ValueError("Event type must be a to-device event type") await self.api.request( Method.PUT, - Path.sendToDevice[event_type][self.api.get_txn_id()], + Path.v3.sendToDevice[event_type][self.api.get_txn_id()], { "messages": { user_id: { @@ -105,7 +105,7 @@ async def upload_keys( data["device_keys"] = device_keys if one_time_keys: data["one_time_keys"] = one_time_keys - resp = await self.api.request(Method.POST, Path.keys.upload, data) + resp = await self.api.request(Method.POST, Path.v3.keys.upload, data) try: return { EncryptionKeyAlgorithm.deserialize(alg): count @@ -147,7 +147,7 @@ async def query_keys( } if token: data["token"] = token - resp = await self.api.request(Method.POST, Path.keys.query, data) + resp = await self.api.request(Method.POST, Path.v3.keys.query, data) return QueryKeysResponse.deserialize(resp) async def claim_keys( @@ -171,7 +171,7 @@ async def claim_keys( """ resp = await self.api.request( Method.POST, - Path.keys.claim, + Path.v3.keys.claim, { "timeout": timeout, "one_time_keys": { diff --git a/mautrix/client/api/modules/media_repository.py b/mautrix/client/api/modules/media_repository.py index 6b0c6d3c..00800925 100644 --- a/mautrix/client/api/modules/media_repository.py +++ b/mautrix/client/api/modules/media_repository.py @@ -65,7 +65,7 @@ async def upload_media( if filename: query["filename"] = filename resp = await self.api.request( - Method.POST, MediaPath.upload, content=data, headers=headers, query_params=query + Method.POST, MediaPath.v3.upload, content=data, headers=headers, query_params=query ) try: return resp["content_uri"] @@ -143,7 +143,7 @@ async def get_url_preview(self, url: str, timestamp: int | None = None) -> MXOpe if timestamp is not None: query_params["ts"] = timestamp content = await self.api.request( - Method.GET, MediaPath.preview_url, query_params=query_params + Method.GET, MediaPath.v3.preview_url, query_params=query_params ) try: return MXOpenGraph.deserialize(content) @@ -167,7 +167,7 @@ async def get_media_repo_config(self) -> MediaRepoConfig: Returns: The media repository config. """ - content = await self.api.request(Method.GET, MediaPath.config) + content = await self.api.request(Method.GET, MediaPath.v3.config) try: return MediaRepoConfig.deserialize(content) except SerializerError as e: diff --git a/mautrix/client/api/modules/misc.py b/mautrix/client/api/modules/misc.py index 485f345b..8abc4745 100644 --- a/mautrix/client/api/modules/misc.py +++ b/mautrix/client/api/modules/misc.py @@ -56,7 +56,7 @@ async def set_typing(self, room_id: RoomID, timeout: int = 0) -> None: content = {"typing": True, "timeout": timeout} else: content = {"typing": False} - await self.api.request(Method.PUT, Path.rooms[room_id].typing[self.mxid], content) + await self.api.request(Method.PUT, Path.v3.rooms[room_id].typing[self.mxid], content) # endregion # region 13.5 Receipts @@ -77,7 +77,7 @@ async def send_receipt( event_id: The last event ID to acknowledge. receipt_type: The type of receipt to send. Currently only ``m.read`` is supported. """ - await self.api.request(Method.POST, Path.rooms[room_id].receipt[receipt_type][event_id]) + await self.api.request(Method.POST, Path.v3.rooms[room_id].receipt[receipt_type][event_id]) # endregion # region 13.6 Fully read markers @@ -110,7 +110,7 @@ async def set_fully_read_marker( content["m.read"] = read_receipt if extra_content: content.update(extra_content) - await self.api.request(Method.POST, Path.rooms[room_id].read_markers, content) + await self.api.request(Method.POST, Path.v3.rooms[room_id].read_markers, content) # endregion # region 13.7 Presence @@ -134,7 +134,7 @@ async def set_presence( } if status: content["status_msg"] = status - await self.api.request(Method.PUT, Path.presence[self.mxid].status, content) + await self.api.request(Method.PUT, Path.v3.presence[self.mxid].status, content) async def get_presence(self, user_id: UserID) -> PresenceEventContent: """ @@ -148,7 +148,7 @@ async def get_presence(self, user_id: UserID) -> PresenceEventContent: Returns: The presence info of the given user. """ - content = await self.api.request(Method.GET, Path.presence[user_id].status) + content = await self.api.request(Method.GET, Path.v3.presence[user_id].status) try: return PresenceEventContent.deserialize(content) except SerializerError: diff --git a/mautrix/client/api/modules/push_rules.py b/mautrix/client/api/modules/push_rules.py index 31392b0c..c94dff87 100644 --- a/mautrix/client/api/modules/push_rules.py +++ b/mautrix/client/api/modules/push_rules.py @@ -41,7 +41,7 @@ async def get_push_rule( Returns: The push rule information. """ - resp = await self.api.request(Method.GET, Path.pushrules[scope][kind][rule_id]) + resp = await self.api.request(Method.GET, Path.v3.pushrules[scope][kind][rule_id]) return PushRule.deserialize(resp) async def set_push_rule( @@ -81,7 +81,10 @@ async def set_push_rule( if pattern: content["pattern"] = pattern await self.api.request( - Method.PUT, Path.pushrules[scope][kind][rule_id], query_params=query, content=content + Method.PUT, + Path.v3.pushrules[scope][kind][rule_id], + query_params=query, + content=content, ) async def remove_push_rule( @@ -97,4 +100,4 @@ async def remove_push_rule( kind: The kind of rule. rule_id: The identifier of the rule. """ - await self.api.request(Method.DELETE, Path.pushrules[scope][kind][rule_id]) + await self.api.request(Method.DELETE, Path.v3.pushrules[scope][kind][rule_id]) diff --git a/mautrix/client/api/modules/room_tag.py b/mautrix/client/api/modules/room_tag.py index fe0c6271..2c1a29ab 100644 --- a/mautrix/client/api/modules/room_tag.py +++ b/mautrix/client/api/modules/room_tag.py @@ -31,7 +31,7 @@ async def get_room_tags(self, room_id: RoomID) -> RoomTagAccountDataEventContent Returns: The m.tag account data event. """ - resp = await self.api.request(Method.GET, Path.user[self.mxid].rooms[room_id].tags) + resp = await self.api.request(Method.GET, Path.v3.user[self.mxid].rooms[room_id].tags) return RoomTagAccountDataEventContent.deserialize(resp) async def get_room_tag(self, room_id: RoomID, tag: str) -> RoomTagInfo | None: @@ -66,7 +66,7 @@ async def set_room_tag( """ await self.api.request( Method.PUT, - Path.user[self.mxid].rooms[room_id].tags[tag], + Path.v3.user[self.mxid].rooms[room_id].tags[tag], content=(info.serialize() if isinstance(info, Serializable) else (info or {})), ) @@ -80,4 +80,4 @@ async def remove_room_tag(self, room_id: RoomID, tag: str) -> None: room_id: The room ID to remove the tag from. tag: The tag to remove. """ - await self.api.request(Method.DELETE, Path.user[self.mxid].rooms[room_id].tags[tag]) + await self.api.request(Method.DELETE, Path.v3.user[self.mxid].rooms[room_id].tags[tag]) diff --git a/mautrix/client/api/rooms.py b/mautrix/client/api/rooms.py index 32d86960..b3b12e07 100644 --- a/mautrix/client/api/rooms.py +++ b/mautrix/client/api/rooms.py @@ -152,7 +152,7 @@ async def create_room( else power_level_override ) - resp = await self.api.request(Method.POST, Path.createRoom, content) + resp = await self.api.request(Method.POST, Path.v3.createRoom, content) try: return resp["room_id"] except KeyError: @@ -179,12 +179,12 @@ async def add_room_alias( room_alias = f"#{alias_localpart}:{self.domain}" content = {"room_id": room_id} try: - await self.api.request(Method.PUT, Path.directory.room[room_alias], content) + await self.api.request(Method.PUT, Path.v3.directory.room[room_alias], content) except MatrixRequestError as e: if e.http_status == 409: if override: await self.remove_room_alias(alias_localpart) - await self.api.request(Method.PUT, Path.directory.room[room_alias], content) + await self.api.request(Method.PUT, Path.v3.directory.room[room_alias], content) else: raise MRoomInUse(e.http_status, e.message) from e else: @@ -205,7 +205,7 @@ async def remove_room_alias(self, alias_localpart: str, raise_404: bool = False) """ room_alias = f"#{alias_localpart}:{self.domain}" try: - await self.api.request(Method.DELETE, Path.directory.room[room_alias]) + await self.api.request(Method.DELETE, Path.v3.directory.room[room_alias]) except MNotFound: if raise_404: raise @@ -226,7 +226,7 @@ async def resolve_room_alias(self, room_alias: RoomAlias) -> RoomAliasInfo: Returns: The room ID and a list of servers that are aware of the room. """ - content = await self.api.request(Method.GET, Path.directory.room[room_alias]) + content = await self.api.request(Method.GET, Path.v3.directory.room[room_alias]) return RoomAliasInfo.deserialize(content) # endregion @@ -235,7 +235,7 @@ async def resolve_room_alias(self, room_alias: RoomAlias) -> RoomAliasInfo: async def get_joined_rooms(self) -> list[RoomID]: """Get the list of rooms the user is in.""" - content = await self.api.request(Method.GET, Path.joined_rooms) + content = await self.api.request(Method.GET, Path.v3.joined_rooms) try: return content["joined_rooms"] except KeyError: @@ -273,7 +273,7 @@ async def join_room_by_id( return room_id content = await self.api.request( Method.POST, - Path.rooms[room_id].join, + Path.v3.rooms[room_id].join, {"third_party_signed": third_party_signed} if third_party_signed is not None else None, ) try: @@ -319,7 +319,7 @@ async def join_room( try: content = await self.api.request( Method.POST, - Path.join[room_id_or_alias], + Path.v3.join[room_id_or_alias], content=content, query_params=query_params, ) @@ -431,7 +431,7 @@ async def invite_user( data = {"user_id": user_id} if reason: data["reason"] = reason - await self.api.request(Method.POST, Path.rooms[room_id].invite, content=data) + await self.api.request(Method.POST, Path.v3.rooms[room_id].invite, content=data) # endregion # region 8.4.2 Leaving rooms @@ -477,7 +477,7 @@ async def leave_room( data = {} if reason: data["reason"] = reason - await self.api.request(Method.POST, Path.rooms[room_id].leave, content=data) + await self.api.request(Method.POST, Path.v3.rooms[room_id].leave, content=data) except MatrixRequestError as e: if "not in room" not in e.message or raise_not_in_room: raise @@ -498,7 +498,7 @@ async def forget_room(self, room_id: RoomID) -> None: Args: room_id: The ID of the room to forget. """ - await self.api.request(Method.POST, Path.rooms[room_id].forget) + await self.api.request(Method.POST, Path.v3.rooms[room_id].forget) async def kick_user( self, @@ -538,7 +538,7 @@ async def kick_user( ) return await self.api.request( - Method.POST, Path.rooms[room_id].kick, {"user_id": user_id, "reason": reason} + Method.POST, Path.v3.rooms[room_id].kick, {"user_id": user_id, "reason": reason} ) # endregion @@ -578,7 +578,7 @@ async def ban_user( ) return await self.api.request( - Method.POST, Path.rooms[room_id].ban, {"user_id": user_id, "reason": reason} + Method.POST, Path.v3.rooms[room_id].ban, {"user_id": user_id, "reason": reason} ) async def unban_user(self, room_id: RoomID, user_id: UserID) -> None: @@ -593,7 +593,7 @@ async def unban_user(self, room_id: RoomID, user_id: UserID) -> None: room_id: The ID of the room from which the user should be unbanned. user_id: The fully qualified user ID of the user being banned. """ - await self.api.request(Method.POST, Path.rooms[room_id].unban, {"user_id": user_id}) + await self.api.request(Method.POST, Path.v3.rooms[room_id].unban, {"user_id": user_id}) # endregion @@ -613,7 +613,7 @@ async def get_room_directory_visibility(self, room_id: RoomID) -> RoomDirectoryV Returns: The visibility of the room in the directory. """ - resp = await self.api.request(Method.GET, Path.directory.list.room[room_id]) + resp = await self.api.request(Method.GET, Path.v3.directory.list.room[room_id]) try: return RoomDirectoryVisibility(resp["visibility"]) except KeyError: @@ -640,7 +640,7 @@ async def set_room_directory_visibility( """ await self.api.request( Method.PUT, - Path.directory.list.room[room_id], + Path.v3.directory.list.room[room_id], { "visibility": visibility.value, }, @@ -700,7 +700,7 @@ async def get_room_directory( query_params = {"server": server} if server is not None else None content = await self.api.request( - method, Path.publicRooms, content, query_params=query_params + method, Path.v3.publicRooms, content, query_params=query_params ) return RoomDirectoryResponse.deserialize(content) diff --git a/mautrix/client/api/user_data.py b/mautrix/client/api/user_data.py index 68807be6..3d3a4575 100644 --- a/mautrix/client/api/user_data.py +++ b/mautrix/client/api/user_data.py @@ -46,7 +46,7 @@ async def search_users(self, search_query: str, limit: int | None = 10) -> UserS """ content = await self.api.request( Method.POST, - Path.user_directory.search, + Path.v3.user_directory.search, { "search_term": search_query, "limit": limit, @@ -83,7 +83,7 @@ async def set_displayname(self, displayname: str, check_current: bool = True) -> return await self.api.request( Method.PUT, - Path.profile[self.mxid].displayname, + Path.v3.profile[self.mxid].displayname, { "displayname": displayname, }, @@ -102,7 +102,7 @@ async def get_displayname(self, user_id: UserID) -> str | None: The display name of the given user. """ try: - content = await self.api.request(Method.GET, Path.profile[user_id].displayname) + content = await self.api.request(Method.GET, Path.v3.profile[user_id].displayname) except MNotFound: return None try: @@ -124,7 +124,7 @@ async def set_avatar_url(self, avatar_url: ContentURI, check_current: bool = Tru return await self.api.request( Method.PUT, - Path.profile[self.mxid].avatar_url, + Path.v3.profile[self.mxid].avatar_url, { "avatar_url": avatar_url, }, @@ -143,7 +143,7 @@ async def get_avatar_url(self, user_id: UserID) -> ContentURI | None: The ``mxc://`` URI to the user's avatar. """ try: - content = await self.api.request(Method.GET, Path.profile[user_id].avatar_url) + content = await self.api.request(Method.GET, Path.v3.profile[user_id].avatar_url) except MNotFound: return None try: @@ -163,7 +163,7 @@ async def get_profile(self, user_id: UserID) -> Member: Returns: The profile information of the given user. """ - content = await self.api.request(Method.GET, Path.profile[user_id]) + content = await self.api.request(Method.GET, Path.v3.profile[user_id]) try: return Member.deserialize(content) except SerializerError as e: From 103ee47379c475dfd3f6c5e0fdc76d0fe0ae05dd Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 20 Feb 2022 14:09:05 +0200 Subject: [PATCH 005/456] Update copyright year --- mautrix/api.py | 2 +- mautrix/appservice/api/appservice.py | 2 +- mautrix/appservice/api/intent.py | 2 +- mautrix/appservice/appservice.py | 2 +- mautrix/appservice/as_handler.py | 2 +- mautrix/appservice/state_store/asyncpg.py | 2 +- mautrix/appservice/state_store/file.py | 2 +- mautrix/appservice/state_store/memory.py | 2 +- mautrix/appservice/state_store/sqlalchemy.py | 2 +- mautrix/bridge/async_getter_lock.py | 2 +- mautrix/bridge/bridge.py | 2 +- mautrix/bridge/commands/admin.py | 2 +- mautrix/bridge/commands/clean_rooms.py | 2 +- mautrix/bridge/commands/crypto.py | 2 +- mautrix/bridge/commands/delete_portal.py | 2 +- mautrix/bridge/commands/handler.py | 2 +- mautrix/bridge/commands/login_matrix.py | 2 +- mautrix/bridge/commands/manhole.py | 2 +- mautrix/bridge/commands/meta.py | 2 +- mautrix/bridge/commands/relay.py | 2 +- mautrix/bridge/config.py | 2 +- mautrix/bridge/crypto_state_store.py | 2 +- mautrix/bridge/custom_puppet.py | 2 +- mautrix/bridge/e2ee.py | 2 +- mautrix/bridge/matrix.py | 2 +- mautrix/bridge/notification_disabler.py | 2 +- mautrix/bridge/portal.py | 2 +- mautrix/bridge/puppet.py | 2 +- mautrix/bridge/state_store/asyncpg.py | 2 +- mautrix/bridge/state_store/sqlalchemy.py | 2 +- mautrix/bridge/user.py | 2 +- mautrix/client/api/__init__.py | 2 +- mautrix/client/api/authentication.py | 2 +- mautrix/client/api/base.py | 2 +- mautrix/client/api/client.py | 2 +- mautrix/client/api/events.py | 2 +- mautrix/client/api/filtering.py | 2 +- mautrix/client/api/modules/__init__.py | 2 +- mautrix/client/api/modules/account_data.py | 2 +- mautrix/client/api/modules/crypto.py | 2 +- mautrix/client/api/modules/media_repository.py | 2 +- mautrix/client/api/modules/misc.py | 2 +- mautrix/client/api/modules/push_rules.py | 2 +- mautrix/client/api/modules/room_tag.py | 2 +- mautrix/client/api/rooms.py | 2 +- mautrix/client/api/user_data.py | 2 +- mautrix/client/client.py | 2 +- mautrix/client/dispatcher.py | 2 +- mautrix/client/encryption_manager.py | 2 +- mautrix/client/state_store/abstract.py | 2 +- mautrix/client/state_store/asyncpg/store.py | 2 +- mautrix/client/state_store/asyncpg/upgrade.py | 2 +- mautrix/client/state_store/file.py | 2 +- mautrix/client/state_store/memory.py | 2 +- mautrix/client/state_store/sqlalchemy/mx_room_state.py | 2 +- mautrix/client/state_store/sqlalchemy/mx_user_profile.py | 2 +- mautrix/client/state_store/sqlalchemy/sqlstatestore.py | 2 +- mautrix/client/state_store/sync.py | 2 +- mautrix/client/state_store/tests/store_test.py | 2 +- mautrix/client/store_updater.py | 2 +- mautrix/client/syncer.py | 2 +- mautrix/crypto/account.py | 2 +- mautrix/crypto/attachments/async_attachments.py | 2 +- mautrix/crypto/attachments/attachments.py | 2 +- mautrix/crypto/base.py | 2 +- mautrix/crypto/decrypt_megolm.py | 2 +- mautrix/crypto/decrypt_olm.py | 2 +- mautrix/crypto/device_lists.py | 2 +- mautrix/crypto/encrypt_megolm.py | 2 +- mautrix/crypto/encrypt_olm.py | 2 +- mautrix/crypto/key_request.py | 2 +- mautrix/crypto/key_share.py | 2 +- mautrix/crypto/machine.py | 2 +- mautrix/crypto/sessions.py | 2 +- mautrix/crypto/store/abstract.py | 2 +- mautrix/crypto/store/asyncpg/store.py | 2 +- mautrix/crypto/store/memory.py | 2 +- mautrix/crypto/types.py | 2 +- mautrix/crypto/unwedge.py | 2 +- mautrix/errors/base.py | 2 +- mautrix/errors/crypto.py | 2 +- mautrix/errors/request.py | 2 +- mautrix/errors/well_known.py | 2 +- mautrix/types/auth.py | 2 +- mautrix/types/crypto.py | 2 +- mautrix/types/event/__init__.py | 2 +- mautrix/types/event/account_data.py | 2 +- mautrix/types/event/base.py | 2 +- mautrix/types/event/encrypted.py | 2 +- mautrix/types/event/ephemeral.py | 2 +- mautrix/types/event/generic.py | 2 +- mautrix/types/event/message.py | 2 +- mautrix/types/event/reaction.py | 2 +- mautrix/types/event/redaction.py | 2 +- mautrix/types/event/state.py | 2 +- mautrix/types/event/to_device.py | 2 +- mautrix/types/event/type.py | 2 +- mautrix/types/event/type.pyi | 2 +- mautrix/types/event/voip.py | 2 +- mautrix/types/filter.py | 2 +- mautrix/types/matrixuri.py | 2 +- mautrix/types/matrixuri_test.py | 2 +- mautrix/types/media.py | 2 +- mautrix/types/misc.py | 2 +- mautrix/types/primitive.py | 2 +- mautrix/types/push_rules.py | 2 +- mautrix/types/users.py | 2 +- mautrix/types/util/enum.py | 2 +- mautrix/types/util/enum_test.py | 2 +- mautrix/types/util/serializable.py | 2 +- mautrix/types/util/serializable_attrs.py | 2 +- mautrix/types/util/serializable_attrs_test.py | 2 +- mautrix/util/async_db/aiosqlite.py | 2 +- mautrix/util/bridge_state.py | 2 +- mautrix/util/config/base.py | 2 +- mautrix/util/config/file.py | 2 +- mautrix/util/config/proxy.py | 2 +- mautrix/util/config/recursive_dict.py | 2 +- mautrix/util/config/string.py | 2 +- mautrix/util/config/validation.py | 2 +- mautrix/util/db/base.py | 2 +- mautrix/util/ffmpeg.py | 2 +- mautrix/util/file_store.py | 2 +- mautrix/util/format_duration.py | 2 +- mautrix/util/format_duration_test.py | 2 +- mautrix/util/formatter/__init__.py | 2 +- mautrix/util/formatter/entity_string.py | 2 +- mautrix/util/formatter/formatted_string.py | 2 +- mautrix/util/formatter/html_reader.py | 2 +- mautrix/util/formatter/html_reader.pyi | 2 +- mautrix/util/formatter/html_reader_lxml.py | 2 +- mautrix/util/formatter/markdown_string.py | 2 +- mautrix/util/logging/color.py | 2 +- mautrix/util/logging/trace.py | 2 +- mautrix/util/magic.py | 2 +- mautrix/util/manhole.py | 2 +- mautrix/util/markdown.py | 2 +- mautrix/util/opt_prometheus.py | 2 +- mautrix/util/opt_prometheus.pyi | 2 +- 139 files changed, 139 insertions(+), 139 deletions(-) diff --git a/mautrix/api.py b/mautrix/api.py index 94503f8a..c53446ae 100644 --- a/mautrix/api.py +++ b/mautrix/api.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/appservice/api/appservice.py b/mautrix/appservice/api/appservice.py index afbcc689..77e679fd 100644 --- a/mautrix/appservice/api/appservice.py +++ b/mautrix/appservice/api/appservice.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/appservice/api/intent.py b/mautrix/appservice/api/intent.py index 394c1963..62d224ed 100644 --- a/mautrix/appservice/api/intent.py +++ b/mautrix/appservice/api/intent.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/appservice/appservice.py b/mautrix/appservice/appservice.py index 396e4610..053ddb1b 100644 --- a/mautrix/appservice/appservice.py +++ b/mautrix/appservice/appservice.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/appservice/as_handler.py b/mautrix/appservice/as_handler.py index c06de019..42a29b3b 100644 --- a/mautrix/appservice/as_handler.py +++ b/mautrix/appservice/as_handler.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/appservice/state_store/asyncpg.py b/mautrix/appservice/state_store/asyncpg.py index 33927279..f144269e 100644 --- a/mautrix/appservice/state_store/asyncpg.py +++ b/mautrix/appservice/state_store/asyncpg.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/appservice/state_store/file.py b/mautrix/appservice/state_store/file.py index 9d4faded..fff99bbc 100644 --- a/mautrix/appservice/state_store/file.py +++ b/mautrix/appservice/state_store/file.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/appservice/state_store/memory.py b/mautrix/appservice/state_store/memory.py index d5fadf81..f7af8a36 100644 --- a/mautrix/appservice/state_store/memory.py +++ b/mautrix/appservice/state_store/memory.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/appservice/state_store/sqlalchemy.py b/mautrix/appservice/state_store/sqlalchemy.py index 6e876579..30c0b1fc 100644 --- a/mautrix/appservice/state_store/sqlalchemy.py +++ b/mautrix/appservice/state_store/sqlalchemy.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/bridge/async_getter_lock.py b/mautrix/bridge/async_getter_lock.py index 78497578..4a246c8f 100644 --- a/mautrix/bridge/async_getter_lock.py +++ b/mautrix/bridge/async_getter_lock.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/bridge/bridge.py b/mautrix/bridge/bridge.py index 1545f383..90cb827b 100644 --- a/mautrix/bridge/bridge.py +++ b/mautrix/bridge/bridge.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/bridge/commands/admin.py b/mautrix/bridge/commands/admin.py index b4508673..a7f0aed2 100644 --- a/mautrix/bridge/commands/admin.py +++ b/mautrix/bridge/commands/admin.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/bridge/commands/clean_rooms.py b/mautrix/bridge/commands/clean_rooms.py index ac508028..cdf72b35 100644 --- a/mautrix/bridge/commands/clean_rooms.py +++ b/mautrix/bridge/commands/clean_rooms.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/bridge/commands/crypto.py b/mautrix/bridge/commands/crypto.py index b06d9cf7..b8cb280a 100644 --- a/mautrix/bridge/commands/crypto.py +++ b/mautrix/bridge/commands/crypto.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/bridge/commands/delete_portal.py b/mautrix/bridge/commands/delete_portal.py index b40aef12..a821ce77 100644 --- a/mautrix/bridge/commands/delete_portal.py +++ b/mautrix/bridge/commands/delete_portal.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/bridge/commands/handler.py b/mautrix/bridge/commands/handler.py index 3dda0c7a..efe556ee 100644 --- a/mautrix/bridge/commands/handler.py +++ b/mautrix/bridge/commands/handler.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/bridge/commands/login_matrix.py b/mautrix/bridge/commands/login_matrix.py index e3c4a4e7..7ad91c72 100644 --- a/mautrix/bridge/commands/login_matrix.py +++ b/mautrix/bridge/commands/login_matrix.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/bridge/commands/manhole.py b/mautrix/bridge/commands/manhole.py index 75ace8d2..0b5edd2a 100644 --- a/mautrix/bridge/commands/manhole.py +++ b/mautrix/bridge/commands/manhole.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/bridge/commands/meta.py b/mautrix/bridge/commands/meta.py index b7f513a8..d0f066b0 100644 --- a/mautrix/bridge/commands/meta.py +++ b/mautrix/bridge/commands/meta.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/bridge/commands/relay.py b/mautrix/bridge/commands/relay.py index ff131580..9b7ec622 100644 --- a/mautrix/bridge/commands/relay.py +++ b/mautrix/bridge/commands/relay.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/bridge/config.py b/mautrix/bridge/config.py index b669c8cf..fbe64c5e 100644 --- a/mautrix/bridge/config.py +++ b/mautrix/bridge/config.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/bridge/crypto_state_store.py b/mautrix/bridge/crypto_state_store.py index 84c05c4b..84169e2e 100644 --- a/mautrix/bridge/crypto_state_store.py +++ b/mautrix/bridge/crypto_state_store.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/bridge/custom_puppet.py b/mautrix/bridge/custom_puppet.py index 5bdb6441..c5c77f25 100644 --- a/mautrix/bridge/custom_puppet.py +++ b/mautrix/bridge/custom_puppet.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/bridge/e2ee.py b/mautrix/bridge/e2ee.py index 633f98de..9428c9d4 100644 --- a/mautrix/bridge/e2ee.py +++ b/mautrix/bridge/e2ee.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/bridge/matrix.py b/mautrix/bridge/matrix.py index 8a9c2228..cdf7dc52 100644 --- a/mautrix/bridge/matrix.py +++ b/mautrix/bridge/matrix.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/bridge/notification_disabler.py b/mautrix/bridge/notification_disabler.py index df37543e..bd53a6ea 100644 --- a/mautrix/bridge/notification_disabler.py +++ b/mautrix/bridge/notification_disabler.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/bridge/portal.py b/mautrix/bridge/portal.py index 3a1e0f8b..8e6db600 100644 --- a/mautrix/bridge/portal.py +++ b/mautrix/bridge/portal.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/bridge/puppet.py b/mautrix/bridge/puppet.py index 4f4e63d3..a09f3548 100644 --- a/mautrix/bridge/puppet.py +++ b/mautrix/bridge/puppet.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/bridge/state_store/asyncpg.py b/mautrix/bridge/state_store/asyncpg.py index 3456bd2a..d9c476ce 100644 --- a/mautrix/bridge/state_store/asyncpg.py +++ b/mautrix/bridge/state_store/asyncpg.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/bridge/state_store/sqlalchemy.py b/mautrix/bridge/state_store/sqlalchemy.py index 26340021..2fa7353e 100644 --- a/mautrix/bridge/state_store/sqlalchemy.py +++ b/mautrix/bridge/state_store/sqlalchemy.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/bridge/user.py b/mautrix/bridge/user.py index cc5ba495..63fd96f4 100644 --- a/mautrix/bridge/user.py +++ b/mautrix/bridge/user.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/client/api/__init__.py b/mautrix/client/api/__init__.py index 783eb248..74081a4e 100644 --- a/mautrix/client/api/__init__.py +++ b/mautrix/client/api/__init__.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/client/api/authentication.py b/mautrix/client/api/authentication.py index 36e134fc..ec97ea5e 100644 --- a/mautrix/client/api/authentication.py +++ b/mautrix/client/api/authentication.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/client/api/base.py b/mautrix/client/api/base.py index 743fd879..da157ae9 100644 --- a/mautrix/client/api/base.py +++ b/mautrix/client/api/base.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/client/api/client.py b/mautrix/client/api/client.py index c66ec7af..51070a04 100644 --- a/mautrix/client/api/client.py +++ b/mautrix/client/api/client.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/client/api/events.py b/mautrix/client/api/events.py index 676ccddd..eb911fb4 100644 --- a/mautrix/client/api/events.py +++ b/mautrix/client/api/events.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/client/api/filtering.py b/mautrix/client/api/filtering.py index 5881ad07..09889ec5 100644 --- a/mautrix/client/api/filtering.py +++ b/mautrix/client/api/filtering.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/client/api/modules/__init__.py b/mautrix/client/api/modules/__init__.py index d134a32b..e6c0449d 100644 --- a/mautrix/client/api/modules/__init__.py +++ b/mautrix/client/api/modules/__init__.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/client/api/modules/account_data.py b/mautrix/client/api/modules/account_data.py index 11c09e0e..18531ad4 100644 --- a/mautrix/client/api/modules/account_data.py +++ b/mautrix/client/api/modules/account_data.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/client/api/modules/crypto.py b/mautrix/client/api/modules/crypto.py index 9549efd9..af656916 100644 --- a/mautrix/client/api/modules/crypto.py +++ b/mautrix/client/api/modules/crypto.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/client/api/modules/media_repository.py b/mautrix/client/api/modules/media_repository.py index 00800925..868909c6 100644 --- a/mautrix/client/api/modules/media_repository.py +++ b/mautrix/client/api/modules/media_repository.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/client/api/modules/misc.py b/mautrix/client/api/modules/misc.py index 8abc4745..443877c1 100644 --- a/mautrix/client/api/modules/misc.py +++ b/mautrix/client/api/modules/misc.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/client/api/modules/push_rules.py b/mautrix/client/api/modules/push_rules.py index c94dff87..38554ac3 100644 --- a/mautrix/client/api/modules/push_rules.py +++ b/mautrix/client/api/modules/push_rules.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/client/api/modules/room_tag.py b/mautrix/client/api/modules/room_tag.py index 2c1a29ab..0dd9a1a8 100644 --- a/mautrix/client/api/modules/room_tag.py +++ b/mautrix/client/api/modules/room_tag.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/client/api/rooms.py b/mautrix/client/api/rooms.py index b3b12e07..b8a9e6fe 100644 --- a/mautrix/client/api/rooms.py +++ b/mautrix/client/api/rooms.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/client/api/user_data.py b/mautrix/client/api/user_data.py index 3d3a4575..f37cb769 100644 --- a/mautrix/client/api/user_data.py +++ b/mautrix/client/api/user_data.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/client/client.py b/mautrix/client/client.py index 830bf70f..5d4e5682 100644 --- a/mautrix/client/client.py +++ b/mautrix/client/client.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/client/dispatcher.py b/mautrix/client/dispatcher.py index e5565a44..7eb51619 100644 --- a/mautrix/client/dispatcher.py +++ b/mautrix/client/dispatcher.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/client/encryption_manager.py b/mautrix/client/encryption_manager.py index 5a97a8ca..3749da37 100644 --- a/mautrix/client/encryption_manager.py +++ b/mautrix/client/encryption_manager.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/client/state_store/abstract.py b/mautrix/client/state_store/abstract.py index 98f198de..dc3e7071 100644 --- a/mautrix/client/state_store/abstract.py +++ b/mautrix/client/state_store/abstract.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/client/state_store/asyncpg/store.py b/mautrix/client/state_store/asyncpg/store.py index 240d6e2c..265e607b 100644 --- a/mautrix/client/state_store/asyncpg/store.py +++ b/mautrix/client/state_store/asyncpg/store.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/client/state_store/asyncpg/upgrade.py b/mautrix/client/state_store/asyncpg/upgrade.py index bb611b21..fd03e2b2 100644 --- a/mautrix/client/state_store/asyncpg/upgrade.py +++ b/mautrix/client/state_store/asyncpg/upgrade.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/client/state_store/file.py b/mautrix/client/state_store/file.py index 57c7a5ec..18644b08 100644 --- a/mautrix/client/state_store/file.py +++ b/mautrix/client/state_store/file.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/client/state_store/memory.py b/mautrix/client/state_store/memory.py index 29f8744b..5db7fa8d 100644 --- a/mautrix/client/state_store/memory.py +++ b/mautrix/client/state_store/memory.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/client/state_store/sqlalchemy/mx_room_state.py b/mautrix/client/state_store/sqlalchemy/mx_room_state.py index 18adcbe0..6124cc91 100644 --- a/mautrix/client/state_store/sqlalchemy/mx_room_state.py +++ b/mautrix/client/state_store/sqlalchemy/mx_room_state.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/client/state_store/sqlalchemy/mx_user_profile.py b/mautrix/client/state_store/sqlalchemy/mx_user_profile.py index 731b6885..0bb6b766 100644 --- a/mautrix/client/state_store/sqlalchemy/mx_user_profile.py +++ b/mautrix/client/state_store/sqlalchemy/mx_user_profile.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/client/state_store/sqlalchemy/sqlstatestore.py b/mautrix/client/state_store/sqlalchemy/sqlstatestore.py index 159b4747..201e6d76 100644 --- a/mautrix/client/state_store/sqlalchemy/sqlstatestore.py +++ b/mautrix/client/state_store/sqlalchemy/sqlstatestore.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/client/state_store/sync.py b/mautrix/client/state_store/sync.py index 0f5e087f..cc054495 100644 --- a/mautrix/client/state_store/sync.py +++ b/mautrix/client/state_store/sync.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/client/state_store/tests/store_test.py b/mautrix/client/state_store/tests/store_test.py index 3e9b6884..6bceb673 100644 --- a/mautrix/client/state_store/tests/store_test.py +++ b/mautrix/client/state_store/tests/store_test.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/client/store_updater.py b/mautrix/client/store_updater.py index 27905044..8b197e64 100644 --- a/mautrix/client/store_updater.py +++ b/mautrix/client/store_updater.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/client/syncer.py b/mautrix/client/syncer.py index a7c7bf23..373a4597 100644 --- a/mautrix/client/syncer.py +++ b/mautrix/client/syncer.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/crypto/account.py b/mautrix/crypto/account.py index 48a433f3..db508262 100644 --- a/mautrix/crypto/account.py +++ b/mautrix/crypto/account.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/crypto/attachments/async_attachments.py b/mautrix/crypto/attachments/async_attachments.py index 7d1d78a7..23037837 100644 --- a/mautrix/crypto/attachments/async_attachments.py +++ b/mautrix/crypto/attachments/async_attachments.py @@ -5,7 +5,7 @@ # any purpose with or without fee is hereby granted, provided that the # above copyright notice and this permission notice appear in all copies. # -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/crypto/attachments/attachments.py b/mautrix/crypto/attachments/attachments.py index c0054aaf..6dfaa3f3 100644 --- a/mautrix/crypto/attachments/attachments.py +++ b/mautrix/crypto/attachments/attachments.py @@ -1,6 +1,6 @@ # Copyright 2018 Zil0 (under the Apache 2.0 license) # Copyright © 2019 Damir Jelić (under the Apache 2.0 license) -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/crypto/base.py b/mautrix/crypto/base.py index 29fb554a..e1f65a6d 100644 --- a/mautrix/crypto/base.py +++ b/mautrix/crypto/base.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/crypto/decrypt_megolm.py b/mautrix/crypto/decrypt_megolm.py index fd654808..3017096b 100644 --- a/mautrix/crypto/decrypt_megolm.py +++ b/mautrix/crypto/decrypt_megolm.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/crypto/decrypt_olm.py b/mautrix/crypto/decrypt_olm.py index 4cc777c9..bfe233f6 100644 --- a/mautrix/crypto/decrypt_olm.py +++ b/mautrix/crypto/decrypt_olm.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/crypto/device_lists.py b/mautrix/crypto/device_lists.py index 9fe01ea7..61fb34b2 100644 --- a/mautrix/crypto/device_lists.py +++ b/mautrix/crypto/device_lists.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/crypto/encrypt_megolm.py b/mautrix/crypto/encrypt_megolm.py index 353e4f7e..88c16041 100644 --- a/mautrix/crypto/encrypt_megolm.py +++ b/mautrix/crypto/encrypt_megolm.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/crypto/encrypt_olm.py b/mautrix/crypto/encrypt_olm.py index de41c486..54ead8a9 100644 --- a/mautrix/crypto/encrypt_olm.py +++ b/mautrix/crypto/encrypt_olm.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/crypto/key_request.py b/mautrix/crypto/key_request.py index d27263d5..a638fe28 100644 --- a/mautrix/crypto/key_request.py +++ b/mautrix/crypto/key_request.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/crypto/key_share.py b/mautrix/crypto/key_share.py index c8c134d3..aba8d344 100644 --- a/mautrix/crypto/key_share.py +++ b/mautrix/crypto/key_share.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/crypto/machine.py b/mautrix/crypto/machine.py index 700aa0e9..e08accb7 100644 --- a/mautrix/crypto/machine.py +++ b/mautrix/crypto/machine.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/crypto/sessions.py b/mautrix/crypto/sessions.py index cae175c5..55292639 100644 --- a/mautrix/crypto/sessions.py +++ b/mautrix/crypto/sessions.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/crypto/store/abstract.py b/mautrix/crypto/store/abstract.py index 8adc9387..a6d0a7ec 100644 --- a/mautrix/crypto/store/abstract.py +++ b/mautrix/crypto/store/abstract.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/crypto/store/asyncpg/store.py b/mautrix/crypto/store/asyncpg/store.py index 204afa74..708b7433 100644 --- a/mautrix/crypto/store/asyncpg/store.py +++ b/mautrix/crypto/store/asyncpg/store.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/crypto/store/memory.py b/mautrix/crypto/store/memory.py index 507680a9..d3a73c9a 100644 --- a/mautrix/crypto/store/memory.py +++ b/mautrix/crypto/store/memory.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/crypto/types.py b/mautrix/crypto/types.py index e210afa5..d6fc6c8b 100644 --- a/mautrix/crypto/types.py +++ b/mautrix/crypto/types.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/crypto/unwedge.py b/mautrix/crypto/unwedge.py index 3f07d528..c3e952f6 100644 --- a/mautrix/crypto/unwedge.py +++ b/mautrix/crypto/unwedge.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/errors/base.py b/mautrix/errors/base.py index 086cc32f..8eb4cc1e 100644 --- a/mautrix/errors/base.py +++ b/mautrix/errors/base.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/errors/crypto.py b/mautrix/errors/crypto.py index 70e07c52..1c67e320 100644 --- a/mautrix/errors/crypto.py +++ b/mautrix/errors/crypto.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/errors/request.py b/mautrix/errors/request.py index d0facbe0..6bef7311 100644 --- a/mautrix/errors/request.py +++ b/mautrix/errors/request.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/errors/well_known.py b/mautrix/errors/well_known.py index 8ff98d72..d1c3f035 100644 --- a/mautrix/errors/well_known.py +++ b/mautrix/errors/well_known.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/types/auth.py b/mautrix/types/auth.py index 847615ab..c0f59af1 100644 --- a/mautrix/types/auth.py +++ b/mautrix/types/auth.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/types/crypto.py b/mautrix/types/crypto.py index f5d41db9..a58737ec 100644 --- a/mautrix/types/crypto.py +++ b/mautrix/types/crypto.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/types/event/__init__.py b/mautrix/types/event/__init__.py index f79eeb0f..930e05cc 100644 --- a/mautrix/types/event/__init__.py +++ b/mautrix/types/event/__init__.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/types/event/account_data.py b/mautrix/types/event/account_data.py index c4d3c8d8..ea7aacca 100644 --- a/mautrix/types/event/account_data.py +++ b/mautrix/types/event/account_data.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/types/event/base.py b/mautrix/types/event/base.py index 84e2b166..5e71430b 100644 --- a/mautrix/types/event/base.py +++ b/mautrix/types/event/base.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/types/event/encrypted.py b/mautrix/types/event/encrypted.py index ad1580ab..397d8cfa 100644 --- a/mautrix/types/event/encrypted.py +++ b/mautrix/types/event/encrypted.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/types/event/ephemeral.py b/mautrix/types/event/ephemeral.py index 4ab43988..930a7de3 100644 --- a/mautrix/types/event/ephemeral.py +++ b/mautrix/types/event/ephemeral.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/types/event/generic.py b/mautrix/types/event/generic.py index d9a19df6..67f5bbad 100644 --- a/mautrix/types/event/generic.py +++ b/mautrix/types/event/generic.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/types/event/message.py b/mautrix/types/event/message.py index b0deaead..2fcbcdf9 100644 --- a/mautrix/types/event/message.py +++ b/mautrix/types/event/message.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/types/event/reaction.py b/mautrix/types/event/reaction.py index abebd579..437d4b31 100644 --- a/mautrix/types/event/reaction.py +++ b/mautrix/types/event/reaction.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/types/event/redaction.py b/mautrix/types/event/redaction.py index 5076e9e3..7714d8ba 100644 --- a/mautrix/types/event/redaction.py +++ b/mautrix/types/event/redaction.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/types/event/state.py b/mautrix/types/event/state.py index 5a66741f..8132d418 100644 --- a/mautrix/types/event/state.py +++ b/mautrix/types/event/state.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/types/event/to_device.py b/mautrix/types/event/to_device.py index f2f189d1..32ffe6e3 100644 --- a/mautrix/types/event/to_device.py +++ b/mautrix/types/event/to_device.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/types/event/type.py b/mautrix/types/event/type.py index 7ca8fd57..1de04812 100644 --- a/mautrix/types/event/type.py +++ b/mautrix/types/event/type.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/types/event/type.pyi b/mautrix/types/event/type.pyi index 567ec5f1..7c573da5 100644 --- a/mautrix/types/event/type.pyi +++ b/mautrix/types/event/type.pyi @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/types/event/voip.py b/mautrix/types/event/voip.py index ce034d06..579f93a6 100644 --- a/mautrix/types/event/voip.py +++ b/mautrix/types/event/voip.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/types/filter.py b/mautrix/types/filter.py index c76f67de..7dda8a78 100644 --- a/mautrix/types/filter.py +++ b/mautrix/types/filter.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/types/matrixuri.py b/mautrix/types/matrixuri.py index 3fb35eb7..a8c62da7 100644 --- a/mautrix/types/matrixuri.py +++ b/mautrix/types/matrixuri.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/types/matrixuri_test.py b/mautrix/types/matrixuri_test.py index 5762c48a..611ead0e 100644 --- a/mautrix/types/matrixuri_test.py +++ b/mautrix/types/matrixuri_test.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/types/media.py b/mautrix/types/media.py index a7661064..f21a7de4 100644 --- a/mautrix/types/media.py +++ b/mautrix/types/media.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/types/misc.py b/mautrix/types/misc.py index 61dc3a54..d8cbe9b2 100644 --- a/mautrix/types/misc.py +++ b/mautrix/types/misc.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/types/primitive.py b/mautrix/types/primitive.py index 7c92b0ee..7e93a25b 100644 --- a/mautrix/types/primitive.py +++ b/mautrix/types/primitive.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/types/push_rules.py b/mautrix/types/push_rules.py index a36c9663..12875770 100644 --- a/mautrix/types/push_rules.py +++ b/mautrix/types/push_rules.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/types/users.py b/mautrix/types/users.py index 69176101..aacd612e 100644 --- a/mautrix/types/users.py +++ b/mautrix/types/users.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/types/util/enum.py b/mautrix/types/util/enum.py index 072dd927..2ee7cab4 100644 --- a/mautrix/types/util/enum.py +++ b/mautrix/types/util/enum.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/types/util/enum_test.py b/mautrix/types/util/enum_test.py index 68495970..38917cf9 100644 --- a/mautrix/types/util/enum_test.py +++ b/mautrix/types/util/enum_test.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/types/util/serializable.py b/mautrix/types/util/serializable.py index 760df3cb..9bdd7d6b 100644 --- a/mautrix/types/util/serializable.py +++ b/mautrix/types/util/serializable.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/types/util/serializable_attrs.py b/mautrix/types/util/serializable_attrs.py index 49f6f619..1172133c 100644 --- a/mautrix/types/util/serializable_attrs.py +++ b/mautrix/types/util/serializable_attrs.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/types/util/serializable_attrs_test.py b/mautrix/types/util/serializable_attrs_test.py index 25fef77d..d311cd5d 100644 --- a/mautrix/types/util/serializable_attrs_test.py +++ b/mautrix/types/util/serializable_attrs_test.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/util/async_db/aiosqlite.py b/mautrix/util/async_db/aiosqlite.py index c0e5a5ad..71320cee 100644 --- a/mautrix/util/async_db/aiosqlite.py +++ b/mautrix/util/async_db/aiosqlite.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/util/bridge_state.py b/mautrix/util/bridge_state.py index 3d25b305..9ff3cadd 100644 --- a/mautrix/util/bridge_state.py +++ b/mautrix/util/bridge_state.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/util/config/base.py b/mautrix/util/config/base.py index 7d14ae0e..46bb3678 100644 --- a/mautrix/util/config/base.py +++ b/mautrix/util/config/base.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/util/config/file.py b/mautrix/util/config/file.py index ce8c5c66..5911af3c 100644 --- a/mautrix/util/config/file.py +++ b/mautrix/util/config/file.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/util/config/proxy.py b/mautrix/util/config/proxy.py index 06b6acdc..5a1828ab 100644 --- a/mautrix/util/config/proxy.py +++ b/mautrix/util/config/proxy.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/util/config/recursive_dict.py b/mautrix/util/config/recursive_dict.py index cf6d0330..2d3c2b69 100644 --- a/mautrix/util/config/recursive_dict.py +++ b/mautrix/util/config/recursive_dict.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/util/config/string.py b/mautrix/util/config/string.py index 1967700f..45a2ce77 100644 --- a/mautrix/util/config/string.py +++ b/mautrix/util/config/string.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/util/config/validation.py b/mautrix/util/config/validation.py index 6ab36432..6b863b27 100644 --- a/mautrix/util/config/validation.py +++ b/mautrix/util/config/validation.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/util/db/base.py b/mautrix/util/db/base.py index 4acf7906..25b32a3e 100644 --- a/mautrix/util/db/base.py +++ b/mautrix/util/db/base.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/util/ffmpeg.py b/mautrix/util/ffmpeg.py index 2a415390..f158ae52 100644 --- a/mautrix/util/ffmpeg.py +++ b/mautrix/util/ffmpeg.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/util/file_store.py b/mautrix/util/file_store.py index c99298fe..a0e43c56 100644 --- a/mautrix/util/file_store.py +++ b/mautrix/util/file_store.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/util/format_duration.py b/mautrix/util/format_duration.py index af2977ce..7b1ca1a1 100644 --- a/mautrix/util/format_duration.py +++ b/mautrix/util/format_duration.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/util/format_duration_test.py b/mautrix/util/format_duration_test.py index 7405845c..9693651c 100644 --- a/mautrix/util/format_duration_test.py +++ b/mautrix/util/format_duration_test.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/util/formatter/__init__.py b/mautrix/util/formatter/__init__.py index aa99777a..2a8c10ad 100644 --- a/mautrix/util/formatter/__init__.py +++ b/mautrix/util/formatter/__init__.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/util/formatter/entity_string.py b/mautrix/util/formatter/entity_string.py index 53977e2f..af1a3c81 100644 --- a/mautrix/util/formatter/entity_string.py +++ b/mautrix/util/formatter/entity_string.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/util/formatter/formatted_string.py b/mautrix/util/formatter/formatted_string.py index 3414f4e3..8fbcdb89 100644 --- a/mautrix/util/formatter/formatted_string.py +++ b/mautrix/util/formatter/formatted_string.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/util/formatter/html_reader.py b/mautrix/util/formatter/html_reader.py index c7b582f3..29697180 100644 --- a/mautrix/util/formatter/html_reader.py +++ b/mautrix/util/formatter/html_reader.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/util/formatter/html_reader.pyi b/mautrix/util/formatter/html_reader.pyi index d70e1bae..63b5b5c3 100644 --- a/mautrix/util/formatter/html_reader.pyi +++ b/mautrix/util/formatter/html_reader.pyi @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/util/formatter/html_reader_lxml.py b/mautrix/util/formatter/html_reader_lxml.py index 4ca1a8a0..4e25d78d 100644 --- a/mautrix/util/formatter/html_reader_lxml.py +++ b/mautrix/util/formatter/html_reader_lxml.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/util/formatter/markdown_string.py b/mautrix/util/formatter/markdown_string.py index 50fd8087..7eaca9c9 100644 --- a/mautrix/util/formatter/markdown_string.py +++ b/mautrix/util/formatter/markdown_string.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/util/logging/color.py b/mautrix/util/logging/color.py index 86572d35..62701119 100644 --- a/mautrix/util/logging/color.py +++ b/mautrix/util/logging/color.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/util/logging/trace.py b/mautrix/util/logging/trace.py index 6b7ac943..f2c893aa 100644 --- a/mautrix/util/logging/trace.py +++ b/mautrix/util/logging/trace.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/util/magic.py b/mautrix/util/magic.py index a3b9ffc7..4f66d02d 100644 --- a/mautrix/util/magic.py +++ b/mautrix/util/magic.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/util/manhole.py b/mautrix/util/manhole.py index 02b122da..f6967336 100644 --- a/mautrix/util/manhole.py +++ b/mautrix/util/manhole.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/util/markdown.py b/mautrix/util/markdown.py index 6d4b275f..1f24ab4e 100644 --- a/mautrix/util/markdown.py +++ b/mautrix/util/markdown.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/util/opt_prometheus.py b/mautrix/util/opt_prometheus.py index 91a1c90a..1afc4a54 100644 --- a/mautrix/util/opt_prometheus.py +++ b/mautrix/util/opt_prometheus.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this diff --git a/mautrix/util/opt_prometheus.pyi b/mautrix/util/opt_prometheus.pyi index e3d943ed..10c763a5 100644 --- a/mautrix/util/opt_prometheus.pyi +++ b/mautrix/util/opt_prometheus.pyi @@ -1,4 +1,4 @@ -# Copyright (c) 2021 Tulir Asokan +# Copyright (c) 2022 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this From 20fd675858f617772fb9e4db566b7bfc593d3ea2 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 20 Feb 2022 14:46:41 +0200 Subject: [PATCH 006/456] Add hacky r0 endpoint fallback for old servers --- mautrix/api.py | 12 +++++++++++- mautrix/appservice/api/appservice.py | 2 ++ mautrix/bridge/e2ee.py | 1 + mautrix/bridge/matrix.py | 4 ++++ mautrix/client/api/base.py | 28 +++++++++++++++++++++++----- mautrix/types/misc.py | 8 ++++++++ 6 files changed, 49 insertions(+), 6 deletions(-) diff --git a/mautrix/api.py b/mautrix/api.py index c53446ae..c6b6b4a3 100644 --- a/mautrix/api.py +++ b/mautrix/api.py @@ -122,6 +122,9 @@ def __getitem__(self, append: str | int) -> PathBuilder: return self return PathBuilder(f"{self.path}/{self._quote(str(append))}") + def replace(self, find: str, replace: str) -> PathBuilder: + return PathBuilder(self.path.replace(find, replace)) + ClientPath = PathBuilder(APIPath.CLIENT) ClientPath.__doc__ = """ @@ -188,6 +191,9 @@ class HTTPAPI: default_retry_count: int """The default retry count to use if a custom value is not passed to :meth:`request`""" + hacky_replace_v3_with_r0: bool = False + """A hacky flag to replace /v3 with /r0 in all API paths.""" + def __init__( self, base_url: URL | str, @@ -262,7 +268,7 @@ def _log_request( return log_content = content if not isinstance(content, bytes) else f"<{len(content)} bytes>" as_user = query_params.get("user_id", None) - level = 1 if path == Path.v3.sync else 5 + level = 1 if path == Path.v3.sync or path == Path.r0.sync else 5 self.log.log( level, f"{method}#{req_id} /{path} {log_content}".strip(" "), @@ -317,6 +323,10 @@ async def request( Returns: The parsed response JSON. """ + if self.hacky_replace_v3_with_r0: + path = path.replace("_matrix/client/v3", "_matrix/client/r0") + path = path.replace("_matrix/media/v3", "_matrix/media/r0") + headers = headers or {} if self.token: headers["Authorization"] = f"Bearer {self.token}" diff --git a/mautrix/appservice/api/appservice.py b/mautrix/appservice/api/appservice.py index 77e679fd..62253626 100644 --- a/mautrix/appservice/api/appservice.py +++ b/mautrix/appservice/api/appservice.py @@ -148,6 +148,7 @@ def real_user(self, mxid: UserID, token: str, base_url: URL | None = None) -> Ap bridge_name=self.bridge_name, default_retry_count=self.default_retry_count, ) + child.hacky_replace_v3_with_r0 = self.hacky_replace_v3_with_r0 self.real_users[mxid] = child return child @@ -255,6 +256,7 @@ def __init__(self, user: UserID, parent: AppServiceAPI) -> None: bridge_name=parent.bridge_name, default_retry_count=parent.default_retry_count, ) + self.hacky_replace_v3_with_r0 = parent.hacky_replace_v3_with_r0 self.parent = parent @property diff --git a/mautrix/bridge/e2ee.py b/mautrix/bridge/e2ee.py index 9428c9d4..1c0e0304 100644 --- a/mautrix/bridge/e2ee.py +++ b/mautrix/bridge/e2ee.py @@ -226,6 +226,7 @@ async def decrypt(self, evt: EncryptedEvent, wait_session_timeout: int = 5) -> M return decrypted async def start(self) -> None: + self.client.api.hacky_replace_v3_with_r0 = self.az.intent.api.hacky_replace_v3_with_r0 flows = await self.client.get_login_flows() flow = flows.get_first_of_type(LoginType.APPSERVICE, LoginType.UNSTABLE_APPSERVICE) if flow is None: diff --git a/mautrix/bridge/matrix.py b/mautrix/bridge/matrix.py index cdf7dc52..fc853451 100644 --- a/mautrix/bridge/matrix.py +++ b/mautrix/bridge/matrix.py @@ -45,6 +45,7 @@ TextMessageEventContent, TypingEvent, UserID, + VersionsResponse, ) from mautrix.util import markdown from mautrix.util.logging import TraceLogger @@ -92,6 +93,7 @@ class BaseMatrixHandler: bridge: br.Bridge e2ee: EncryptionManager | None media_config: MediaRepoConfig + versions: VersionsResponse user_id_prefix: str user_id_suffix: str @@ -106,6 +108,7 @@ def __init__( self.bridge = bridge self.commands = command_processor or cmd.CommandProcessor(bridge=bridge) self.media_config = MediaRepoConfig(upload_size=50 * 1024 * 1024) + self.versions = VersionsResponse(versions=["v1.2"]) self.az.matrix_event_handler(self.int_handle_event) self.e2ee = None @@ -151,6 +154,7 @@ async def wait_for_connection(self) -> None: tried_to_register = False while True: try: + self.versions = await self.az.intent.versions() await self.az.intent.whoami() break except (MUnknownToken, MExclusive): diff --git a/mautrix/client/api/base.py b/mautrix/client/api/base.py index da157ae9..b913f08d 100644 --- a/mautrix/client/api/base.py +++ b/mautrix/client/api/base.py @@ -10,7 +10,7 @@ from aiohttp import ClientError, ClientSession, ContentTypeError from yarl import URL -from mautrix.api import HTTPAPI, Method +from mautrix.api import HTTPAPI, Method, Path from mautrix.errors import ( WellKnownInvalidVersionsResponse, WellKnownMissingHomeserver, @@ -37,6 +37,7 @@ class BaseClientAPI: device_id: DeviceID api: HTTPAPI log: TraceLogger + versions_cache: VersionsResponse | None def __init__( self, mxid: UserID = "", device_id: DeviceID = "", api: HTTPAPI | None = None, **kwargs @@ -62,6 +63,7 @@ def __init__( self.localpart = None self.domain = None self.fill_member_event_callback = None + self.versions_cache = None self.device_id = device_id self.api = api or HTTPAPI(**kwargs) self.log = self.api.log @@ -101,10 +103,26 @@ def mxid(self, mxid: UserID) -> None: self.localpart, self.domain = self.parse_user_id(mxid) self._mxid = mxid - async def versions(self) -> VersionsResponse: - """Get client-server spec versions supported by the server.""" - resp = await self.api.request(Method.GET, "_matrix/client/versions") - return VersionsResponse.deserialize(resp) + async def versions(self, no_cache: bool = False) -> VersionsResponse: + """ + Get client-server spec versions supported by the server. + + Args: + no_cache: If true, the versions will always be fetched from the server + rather than using cached results when availab.e. + + Returns: + The supported Matrix spec versions and unstable features. + """ + if no_cache or not self.versions_cache: + resp = await self.api.request(Method.GET, Path.versions) + vers = self.versions_cache = VersionsResponse.deserialize(resp) + if not vers.has_modern_versions and vers.has_legacy_versions: + self.log.warning( + "Server isn't advertising modern spec versions, falling back to /r0 endpoints" + ) + self.api.hacky_replace_v3_with_r0 = True + return self.versions_cache @classmethod async def discover(cls, domain: str, session: ClientSession | None = None) -> URL | None: diff --git a/mautrix/types/misc.py b/mautrix/types/misc.py index d8cbe9b2..f34bf18f 100644 --- a/mautrix/types/misc.py +++ b/mautrix/types/misc.py @@ -112,6 +112,14 @@ class VersionsResponse(SerializableAttrs): versions: List[str] unstable_features: Dict[str, bool] = attr.ib(factory=lambda: {}) + @property + def has_legacy_versions(self) -> bool: + return any(v for v in self.versions if v.startswith("r0.")) + + @property + def has_modern_versions(self) -> bool: + return any(v for v in self.versions if v.startswith("v")) + @dataclass class BatchSendResponse(SerializableAttrs): From 0e476ec385a3a85e10d95133f79493752c5b8fe5 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 20 Feb 2022 14:46:56 +0200 Subject: [PATCH 007/456] Update batch send API path --- mautrix/appservice/api/intent.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/mautrix/appservice/api/intent.py b/mautrix/appservice/api/intent.py index 62d224ed..62cc2509 100644 --- a/mautrix/appservice/api/intent.py +++ b/mautrix/appservice/api/intent.py @@ -8,8 +8,7 @@ from typing import Any, Awaitable, Iterable from urllib.parse import quote as urllib_quote -from mautrix import __optional_imports__ -from mautrix.api import Method, Path, UnstableClientPath +from mautrix.api import Method, Path from mautrix.client import ClientAPI, StoreUpdatingAPI from mautrix.errors import ( IntentError, @@ -458,7 +457,7 @@ async def batch_send( Returns: All the event IDs generated, plus a batch ID that can be passed back to this method. """ - path = UnstableClientPath["org.matrix.msc2716"].rooms[room_id].batch_send + path = Path.unstable["org.matrix.msc2716"].rooms[room_id].batch_send query = {"prev_event_id": prev_event_id} if batch_id: query["batch_id"] = batch_id From 2a592b7c9f8e0f55e9c15b66d0ab0a3149fe2b6f Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 20 Feb 2022 14:53:04 +0200 Subject: [PATCH 008/456] Update changelog --- CHANGELOG.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fe9f5eb1..d284b2cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,16 @@ -## v0.14.11 (unreleased) +## v0.15.0 (unreleased) * Removed Python 3.7 support. +* **Breaking change *(api)*** Removed `r0` from default path builders in order + to update to `v3` and per-endpoint versioning. + * The client API modules have been updated to specify v3 in the paths, other + direct usage of `Path`, `ClientPath` and `MediaPath` will have to be + updated manually. `UnstableClientPath` no longer exists and should be + replaced with `Path.unstable`. + * There's a temporary hacky backwards-compatibility layer which replaces /v3 + with /r0 if the server doesn't advertise support for Matrix v1.1 or higher. + It can be activated by calling the `.versions()` method in `ClientAPI`. + The bridge module calls that method automatically. * *(bridge)* Removed legacy community utilities. * *(util.async_db)* Fixed counting number of db upgrades. * *(util.async_db)* Added support for schema migrations that jump versions. From 5bffa7fee30853282352148a0d6faa85867cf115 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 20 Feb 2022 15:06:18 +0200 Subject: [PATCH 009/456] Update spec links and other things --- .../client/api/modules/media_repository.py | 2 +- mautrix/types/auth.py | 43 +++++++++++-------- mautrix/types/event/state.py | 6 +-- mautrix/types/filter.py | 12 +++--- mautrix/types/media.py | 43 +++++++++---------- mautrix/types/misc.py | 6 +-- mautrix/types/primitive.py | 2 +- 7 files changed, 59 insertions(+), 55 deletions(-) diff --git a/mautrix/client/api/modules/media_repository.py b/mautrix/client/api/modules/media_repository.py index 868909c6..7e293152 100644 --- a/mautrix/client/api/modules/media_repository.py +++ b/mautrix/client/api/modules/media_repository.py @@ -132,7 +132,7 @@ async def get_url_preview(self, url: str, timestamp: int | None = None) -> MXOpe """ Get information about a URL for a client. - See also: `API reference `__ + See also: `API reference `__ Args: url: The URL to get a preview of. diff --git a/mautrix/types/auth.py b/mautrix/types/auth.py index c0f59af1..3fd1b2d3 100644 --- a/mautrix/types/auth.py +++ b/mautrix/types/auth.py @@ -6,10 +6,9 @@ from typing import List, NewType, Optional, Union from attr import dataclass -import attr from .primitive import JSON, DeviceID, UserID -from .util import ExtensibleEnum, Obj, SerializableAttrs, deserializer +from .util import ExtensibleEnum, Obj, SerializableAttrs, deserializer, field class LoginType(ExtensibleEnum): @@ -17,7 +16,7 @@ class LoginType(ExtensibleEnum): A login type, as specified in the `POST /login endpoint`_ .. _POST /login endpoint: - https://matrix.org/docs/spec/client_server/r0.6.1#post-matrix-client-r0-login + https://spec.matrix.org/v1.2/client-server-api/#post_matrixclientv3login """ PASSWORD: "LoginType" = "m.login.password" @@ -35,7 +34,7 @@ class LoginFlow(SerializableAttrs): A login flow, as specified in the `GET /login endpoint`_ .. _GET /login endpoint: - https://matrix.org/docs/spec/client_server/r0.6.1#get-matrix-client-r0-login + https://spec.matrix.org/v1.2/client-server-api/#get_matrixclientv3login """ type: LoginType @@ -60,7 +59,7 @@ class UserIdentifierType(ExtensibleEnum): A user identifier type, as specified in the `Identifier types`_ section of the login spec. .. _Identifier types: - https://matrix.org/docs/spec/client_server/latest#identifier-types + https://spec.matrix.org/v1.2/client-server-api/#identifier-types """ MATRIX_USER: "UserIdentifierType" = "m.id.user" @@ -89,9 +88,9 @@ class ThirdPartyIdentifier(SerializableAttrs): Appendix for a list of Third-party ID media. .. _/account/3pid: - https://matrix.org/docs/spec/client_server/latest#post-matrix-client-r0-account-3pid + https://spec.matrix.org/v1.2/client-server-api/#post_matrixclientv3account3pid .. _3PID Types: - https://matrix.org/docs/spec/appendices.html#pid-types + https://spec.matrix.org/v1.2/appendices/#3pid-types """ medium: str @@ -109,7 +108,7 @@ class PhoneIdentifier(SerializableAttrs): identifier type with a ``medium`` of ``msisdn`` instead. .. _/account/3pid: - https://matrix.org/docs/spec/client_server/latest#post-matrix-client-r0-account-3pid + https://spec.matrix.org/v1.2/client-server-api/#post_matrixclientv3account3pid """ country: str @@ -154,19 +153,24 @@ class DiscoveryIntegrationServer(SerializableAttrs): @dataclass class DiscoveryIntegrations(SerializableAttrs): - managers: List[DiscoveryIntegrationServer] = attr.ib(factory=lambda: []) + managers: List[DiscoveryIntegrationServer] = field(factory=lambda: []) @dataclass class DiscoveryInformation(SerializableAttrs): - homeserver: Optional[DiscoveryServer] = attr.ib( - metadata={"json": "m.homeserver"}, factory=DiscoveryServer - ) - identity_server: Optional[DiscoveryServer] = attr.ib( - metadata={"json": "m.identity_server"}, factory=DiscoveryServer + """ + .well-known discovery information, as specified in the `GET /.well-known/matrix/client endpoint`_ + + .. _GET /.well-known/matrix/client endpoint: + https://spec.matrix.org/v1.2/client-server-api/#getwell-knownmatrixclient + """ + + homeserver: Optional[DiscoveryServer] = field(json="m.homeserver", factory=DiscoveryServer) + identity_server: Optional[DiscoveryServer] = field( + json="m.identity_server", factory=DiscoveryServer ) - integrations: Optional[DiscoveryServer] = attr.ib( - metadata={"json": "m.integrations"}, factory=DiscoveryIntegrations + integrations: Optional[DiscoveryServer] = field( + json="m.integrations", factory=DiscoveryIntegrations ) @@ -176,13 +180,13 @@ class LoginResponse(SerializableAttrs): The response for a login request, as specified in the `POST /login endpoint`_ .. _POST /login endpoint: - https://matrix.org/docs/spec/client_server/r0.6.1#post-matrix-client-r0-login + https://spec.matrix.org/v1.2/client-server-api/#post_matrixclientv3login """ user_id: UserID device_id: DeviceID access_token: str - well_known: DiscoveryInformation = attr.ib(factory=DiscoveryInformation) + well_known: DiscoveryInformation = field(factory=DiscoveryInformation) @dataclass @@ -191,8 +195,9 @@ class WhoamiResponse(SerializableAttrs): The response for a whoami request, as specified in the `GET /account/whoami endpoint`_ .. _GET /account/whoami endpoint: - https://spec.matrix.org/v1.1/client-server-api/#get_matrixclientv3accountwhoami + https://spec.matrix.org/v1.2/client-server-api/#get_matrixclientv3accountwhoami """ user_id: UserID device_id: Optional[DeviceID] = None + is_guest: bool = False diff --git a/mautrix/types/event/state.py b/mautrix/types/event/state.py index 8132d418..f1f1053c 100644 --- a/mautrix/types/event/state.py +++ b/mautrix/types/event/state.py @@ -74,7 +74,7 @@ class Membership(SerializableEnum): The membership state of a user in a room as specified in section `8.4 Room membership`_ of the spec. - .. _8.4 Room membership: https://spec.matrix.org/v1.1/client-server-api/#room-membership + .. _8.4 Room membership: https://spec.matrix.org/v1.2/client-server-api/#room-membership """ JOIN = "join" @@ -88,7 +88,7 @@ class Membership(SerializableEnum): class MemberStateEventContent(SerializableAttrs): """The content of a membership event. `Spec link`_ - .. _Spec link: https://spec.matrix.org/v1.1/client-server-api/#mroommember""" + .. _Spec link: https://spec.matrix.org/v1.2/client-server-api/#mroommember""" membership: Membership = Membership.LEAVE avatar_url: ContentURI = None @@ -109,7 +109,7 @@ class CanonicalAliasStateEventContent(SerializableAttrs): See also: `m.room.canonical_alias in the spec`_ - .. _m.room.canonical_alias in the spec: https://spec.matrix.org/v1.1/client-server-api/#mroomcanonical_alias + .. _m.room.canonical_alias in the spec: https://spec.matrix.org/v1.2/client-server-api/#mroomcanonical_alias """ canonical_alias: RoomAlias = attr.ib(default=None, metadata={"json": "alias"}) diff --git a/mautrix/types/filter.py b/mautrix/types/filter.py index 7dda8a78..6aa7e78d 100644 --- a/mautrix/types/filter.py +++ b/mautrix/types/filter.py @@ -17,7 +17,7 @@ class EventFormat(SerializableEnum): Federation event format enum, as specified in the `create filter endpoint`_. .. _create filter endpoint: - https://matrix.org/docs/spec/client_server/r0.5.0#post-matrix-client-r0-user-userid-filter + https://spec.matrix.org/v1.2/client-server-api/#post_matrixclientv3useruseridfilter """ CLIENT = "client" @@ -30,7 +30,7 @@ class EventFilter(SerializableAttrs): Event filter object, as specified in the `create filter endpoint`_. .. _create filter endpoint: - https://matrix.org/docs/spec/client_server/r0.5.0#post-matrix-client-r0-user-userid-filter + https://spec.matrix.org/v1.2/client-server-api/#post_matrixclientv3useruseridfilter """ limit: int = None @@ -59,7 +59,7 @@ class RoomEventFilter(EventFilter, SerializableAttrs): Room event filter object, as specified in the `create filter endpoint`_. .. _create filter endpoint: - https://matrix.org/docs/spec/client_server/r0.5.0#post-matrix-client-r0-user-userid-filter + https://spec.matrix.org/v1.2/client-server-api/#post_matrixclientv3useruseridfilter """ lazy_load_members: bool = False @@ -95,7 +95,7 @@ class StateFilter(RoomEventFilter, SerializableAttrs): same as :class:`RoomEventFilter`. .. _create filter endpoint: - https://matrix.org/docs/spec/client_server/r0.5.0#post-matrix-client-r0-user-userid-filter + https://spec.matrix.org/v1.2/client-server-api/#post_matrixclientv3useruseridfilter """ pass @@ -107,7 +107,7 @@ class RoomFilter(SerializableAttrs): Room filter object, as specified in the `create filter endpoint`_. .. _create filter endpoint: - https://matrix.org/docs/spec/client_server/r0.5.0#post-matrix-client-r0-user-userid-filter + https://spec.matrix.org/v1.2/client-server-api/#post_matrixclientv3useruseridfilter """ not_rooms: List[RoomID] = None @@ -144,7 +144,7 @@ class Filter(SerializableAttrs): Base filter object, as specified in the `create filter endpoint`_. .. _create filter endpoint: - https://matrix.org/docs/spec/client_server/r0.5.0#post-matrix-client-r0-user-userid-filter + https://spec.matrix.org/v1.2/client-server-api/#post_matrixclientv3useruseridfilter """ event_fields: List[str] = None diff --git a/mautrix/types/media.py b/mautrix/types/media.py index f21a7de4..ec005be5 100644 --- a/mautrix/types/media.py +++ b/mautrix/types/media.py @@ -4,10 +4,9 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. from attr import dataclass -import attr from .primitive import ContentURI -from .util import SerializableAttrs +from .util import SerializableAttrs, field @dataclass @@ -16,34 +15,34 @@ class MediaRepoConfig(SerializableAttrs): Matrix media repo config. See `GET /_matrix/media/r0/config`_. .. _GET /_matrix/media/r0/config: - https://matrix.org/docs/spec/client_server/r0.5.0#get-matrix-media-r0-config + https://spec.matrix.org/v1.2/client-server-api/#get_matrixmediav3config """ - upload_size: int = attr.ib(metadata={"json": "m.upload.size"}) + upload_size: int = field(json="m.upload.size") @dataclass class OpenGraphImage(SerializableAttrs): - url: ContentURI = attr.ib(default=None, metadata={"json": "og:image"}) - mimetype: str = attr.ib(default=None, metadata={"json": "og:image:type"}) - height: int = attr.ib(default=None, metadata={"json": "og:image:width"}) - width: int = attr.ib(default=None, metadata={"json": "og:image:height"}) - size: int = attr.ib(default=None, metadata={"json": "matrix:image:size"}) + url: ContentURI = field(default=None, json="og:image") + mimetype: str = field(default=None, json="og:image:type") + height: int = field(default=None, json="og:image:width") + width: int = field(default=None, json="og:image:height") + size: int = field(default=None, json="matrix:image:size") @dataclass class OpenGraphVideo(SerializableAttrs): - url: ContentURI = attr.ib(default=None, metadata={"json": "og:video"}) - mimetype: str = attr.ib(default=None, metadata={"json": "og:video:type"}) - height: int = attr.ib(default=None, metadata={"json": "og:video:width"}) - width: int = attr.ib(default=None, metadata={"json": "og:video:height"}) - size: int = attr.ib(default=None, metadata={"json": "matrix:video:size"}) + url: ContentURI = field(default=None, json="og:video") + mimetype: str = field(default=None, json="og:video:type") + height: int = field(default=None, json="og:video:width") + width: int = field(default=None, json="og:video:height") + size: int = field(default=None, json="matrix:video:size") @dataclass class OpenGraphAudio(SerializableAttrs): - url: ContentURI = attr.ib(default=None, metadata={"json": "og:audio"}) - mimetype: str = attr.ib(default=None, metadata={"json": "og:audio:type"}) + url: ContentURI = field(default=None, json="og:audio") + mimetype: str = field(default=None, json="og:audio:type") @dataclass @@ -52,11 +51,11 @@ class MXOpenGraph(SerializableAttrs): Matrix URL preview response. See `GET /_matrix/media/r0/preview_url`_. .. _GET /_matrix/media/r0/preview_url: - https://matrix.org/docs/spec/client_server/r0.5.0#get-matrix-media-r0-preview-url + https://spec.matrix.org/v1.2/client-server-api/#get_matrixmediav3preview_url """ - title: str = attr.ib(default=None, metadata={"json": "og:title"}) - description: str = attr.ib(default=None, metadata={"json": "og:description"}) - image: OpenGraphImage = attr.ib(default=None, metadata={"flatten": True}) - video: OpenGraphVideo = attr.ib(default=None, metadata={"flatten": True}) - audio: OpenGraphAudio = attr.ib(default=None, metadata={"flatten": True}) + title: str = field(default=None, json="og:title") + description: str = field(default=None, json="og:description") + image: OpenGraphImage = field(default=None, flatten=True) + video: OpenGraphVideo = field(default=None, flatten=True) + audio: OpenGraphAudio = field(default=None, flatten=True) diff --git a/mautrix/types/misc.py b/mautrix/types/misc.py index f34bf18f..5340534e 100644 --- a/mautrix/types/misc.py +++ b/mautrix/types/misc.py @@ -31,7 +31,7 @@ class RoomCreatePreset(Enum): Room creation preset, as specified in the `createRoom endpoint`_ .. _createRoom endpoint: - https://spec.matrix.org/v1.1/client-server-api/#post_matrixclientv3createroom + https://spec.matrix.org/v1.2/client-server-api/#post_matrixclientv3createroom """ PRIVATE = "private_chat" @@ -44,7 +44,7 @@ class RoomDirectoryVisibility(Enum): Room directory visibility, as specified in the `createRoom endpoint`_ .. _createRoom endpoint: - https://spec.matrix.org/v1.1/client-server-api/#post_matrixclientv3createroom + https://spec.matrix.org/v1.2/client-server-api/#post_matrixclientv3createroom """ PRIVATE = "private" @@ -64,7 +64,7 @@ class RoomAliasInfo(SerializableAttrs): Room alias query result, as specified in the `alias resolve endpoint`_ .. _alias resolve endpoint: - https://spec.matrix.org/v1.1/client-server-api/#get_matrixclientv3directoryroomroomalias + https://spec.matrix.org/v1.2/client-server-api/#get_matrixclientv3directoryroomroomalias """ room_id: RoomID = None diff --git a/mautrix/types/primitive.py b/mautrix/types/primitive.py index 7e93a25b..f318f966 100644 --- a/mautrix/types/primitive.py +++ b/mautrix/types/primitive.py @@ -32,7 +32,7 @@ A Matrix `content URI`_, used by the content repository. .. _content URI: - https://spec.matrix.org/v1.1/client-server-api/#matrix-content-mxc-uris + https://spec.matrix.org/v1.2/client-server-api/#matrix-content-mxc-uris """ SyncToken = NewType("SyncToken", str) From 04a151898416556a40970d99585437a89e2b4392 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 20 Feb 2022 15:11:48 +0200 Subject: [PATCH 010/456] Remove lxml HTML parser --- CHANGELOG.md | 4 ++++ mautrix/util/formatter/html_reader_lxml.py | 12 ------------ 2 files changed, 4 insertions(+), 12 deletions(-) delete mode 100644 mautrix/util/formatter/html_reader_lxml.py diff --git a/CHANGELOG.md b/CHANGELOG.md index d284b2cd..da3f57a5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,10 @@ with /r0 if the server doesn't advertise support for Matrix v1.1 or higher. It can be activated by calling the `.versions()` method in `ClientAPI`. The bridge module calls that method automatically. +* **Breaking change *(util.formatter)*** Removed lxml-based HTML parser. + * The parsed data format is still compatible with lxml, so it is possible to + use lxml with `MatrixParser` by setting `lxml.html.fromstring` as the + `read_html` method. * *(bridge)* Removed legacy community utilities. * *(util.async_db)* Fixed counting number of db upgrades. * *(util.async_db)* Added support for schema migrations that jump versions. diff --git a/mautrix/util/formatter/html_reader_lxml.py b/mautrix/util/formatter/html_reader_lxml.py deleted file mode 100644 index 4e25d78d..00000000 --- a/mautrix/util/formatter/html_reader_lxml.py +++ /dev/null @@ -1,12 +0,0 @@ -# Copyright (c) 2022 Tulir Asokan -# -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. -from lxml import html - -HTMLNode = html.HtmlElement - - -def read_html(data: str) -> HTMLNode: - return html.fromstring(data) From 790e38e21226859d624c17bd5ec7db2335740bb9 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 20 Feb 2022 15:17:41 +0200 Subject: [PATCH 011/456] Update random things --- CHANGELOG.md | 4 +++- mautrix/util/db/base.py | 8 ++++++++ mautrix/util/message_send_checkpoint.py | 5 +++++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index da3f57a5..1b8f1d66 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ ## v0.15.0 (unreleased) -* Removed Python 3.7 support. +* **Breaking change (all)** Removed Python 3.7 support. * **Breaking change *(api)*** Removed `r0` from default path builders in order to update to `v3` and per-endpoint versioning. * The client API modules have been updated to specify v3 in the paths, other @@ -18,6 +18,8 @@ * *(bridge)* Removed legacy community utilities. * *(util.async_db)* Fixed counting number of db upgrades. * *(util.async_db)* Added support for schema migrations that jump versions. +* *(util.db)* Module deprecated. The async_db module is recommended. However, + the SQLAlchemy helpers will remain until maubot has switched to asyncpg. ## v0.14.10 (2022-02-01) diff --git a/mautrix/util/db/base.py b/mautrix/util/db/base.py index 25b32a3e..b9baeb21 100644 --- a/mautrix/util/db/base.py +++ b/mautrix/util/db/base.py @@ -23,6 +23,9 @@ class BaseClass: """ Base class for SQLAlchemy models. Provides SQLAlchemy declarative base features and some additional utilities. + + .. deprecated:: 0.15.0 + The :mod:`mautrix.util.async_db` utility is now recommended over SQLAlchemy. """ __tablename__: str @@ -237,4 +240,9 @@ def __iter__(self): @as_declarative() class Base(BaseClass): + """ + .. deprecated:: 0.15.0 + The :mod:`mautrix.util.async_db` utility is now recommended over SQLAlchemy. + """ + pass diff --git a/mautrix/util/message_send_checkpoint.py b/mautrix/util/message_send_checkpoint.py index 276f7b7a..5c0e052e 100644 --- a/mautrix/util/message_send_checkpoint.py +++ b/mautrix/util/message_send_checkpoint.py @@ -1,3 +1,8 @@ +# Copyright (c) 2022 Sumner Evans +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. from typing import Optional import logging From 339b8c39f5d4f6434d292ab2b20ad8ebfc7b0efd Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 20 Feb 2022 15:20:48 +0200 Subject: [PATCH 012/456] Add module-level deprecation notice to db module --- docs/api/mautrix.util/db.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/api/mautrix.util/db.rst b/docs/api/mautrix.util/db.rst index 4607ad48..4aec7409 100644 --- a/docs/api/mautrix.util/db.rst +++ b/docs/api/mautrix.util/db.rst @@ -3,3 +3,6 @@ db .. automodule:: mautrix.util.db :imported-members: + + .. deprecated:: 0.15.0 + The :mod:`mautrix.util.async_db` utility is now recommended over SQLAlchemy. From aec5a6c9f6b70a9ae550b7c8bd19ef6d7363c0ba Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 20 Feb 2022 15:52:52 +0200 Subject: [PATCH 013/456] Move crypto types to types module --- CHANGELOG.md | 5 ++- mautrix/bridge/e2ee.py | 12 ++----- mautrix/crypto/__init__.py | 1 - mautrix/crypto/decrypt_megolm.py | 9 +++-- mautrix/crypto/decrypt_olm.py | 2 +- mautrix/crypto/device_lists.py | 11 ++++-- mautrix/crypto/encrypt_megolm.py | 3 +- mautrix/crypto/encrypt_olm.py | 4 ++- mautrix/crypto/key_request.py | 2 +- mautrix/crypto/key_share.py | 3 +- mautrix/crypto/machine.py | 2 +- mautrix/crypto/store/abstract.py | 4 ++- mautrix/crypto/store/asyncpg/store.py | 21 ++++++----- mautrix/crypto/store/memory.py | 16 +++++++-- mautrix/crypto/types.py | 52 --------------------------- mautrix/types/__init__.py | 11 +++++- mautrix/types/crypto.py | 38 ++++++++++++++++++-- 17 files changed, 107 insertions(+), 89 deletions(-) delete mode 100644 mautrix/crypto/types.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b8f1d66..4d022c74 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ ## v0.15.0 (unreleased) -* **Breaking change (all)** Removed Python 3.7 support. +* **Breaking change *(\*)*** Removed Python 3.7 support. * **Breaking change *(api)*** Removed `r0` from default path builders in order to update to `v3` and per-endpoint versioning. * The client API modules have been updated to specify v3 in the paths, other @@ -15,6 +15,9 @@ * The parsed data format is still compatible with lxml, so it is possible to use lxml with `MatrixParser` by setting `lxml.html.fromstring` as the `read_html` method. +* **Breaking change *(crypto)*** Moved `TrustState`, `DeviceIdentity`, + `OlmEventKeys` and `DecryptedOlmEvent` dataclasses from `crypto.types` + into `types.crypto`. * *(bridge)* Removed legacy community utilities. * *(util.async_db)* Fixed counting number of db upgrades. * *(util.async_db)* Added support for schema migrations that jump versions. diff --git a/mautrix/bridge/e2ee.py b/mautrix/bridge/e2ee.py index 1c0e0304..958d46a5 100644 --- a/mautrix/bridge/e2ee.py +++ b/mautrix/bridge/e2ee.py @@ -12,18 +12,11 @@ from mautrix import __optional_imports__ from mautrix.appservice import AppService from mautrix.client import Client, SyncStore -from mautrix.crypto import ( - CryptoStore, - DeviceIdentity, - OlmMachine, - PgCryptoStore, - RejectKeyShare, - StateStore, - TrustState, -) +from mautrix.crypto import CryptoStore, OlmMachine, PgCryptoStore, RejectKeyShare, StateStore from mautrix.errors import EncryptionError, SessionNotFound from mautrix.types import ( JSON, + DeviceIdentity, EncryptedEvent, EncryptedMegolmEventContent, EventFilter, @@ -39,6 +32,7 @@ Serializable, StateEvent, StateFilter, + TrustState, ) from mautrix.util.logging import TraceLogger diff --git a/mautrix/crypto/__init__.py b/mautrix/crypto/__init__.py index d7c934b9..15adb3ea 100644 --- a/mautrix/crypto/__init__.py +++ b/mautrix/crypto/__init__.py @@ -1,7 +1,6 @@ from .account import OlmAccount from .key_share import RejectKeyShare from .sessions import InboundGroupSession, OutboundGroupSession, Session -from .types import DecryptedOlmEvent, DeviceIdentity, TrustState # These have to be last from .store import ( # isort: skip diff --git a/mautrix/crypto/decrypt_megolm.py b/mautrix/crypto/decrypt_megolm.py index 3017096b..c1c0100e 100644 --- a/mautrix/crypto/decrypt_megolm.py +++ b/mautrix/crypto/decrypt_megolm.py @@ -15,10 +15,15 @@ SessionNotFound, VerificationError, ) -from mautrix.types import EncryptedEvent, EncryptedMegolmEventContent, EncryptionAlgorithm, Event +from mautrix.types import ( + EncryptedEvent, + EncryptedMegolmEventContent, + EncryptionAlgorithm, + Event, + TrustState, +) from .base import BaseOlmMachine -from .types import TrustState class MegolmDecryptionMachine(BaseOlmMachine): diff --git a/mautrix/crypto/decrypt_olm.py b/mautrix/crypto/decrypt_olm.py index bfe233f6..8182b160 100644 --- a/mautrix/crypto/decrypt_olm.py +++ b/mautrix/crypto/decrypt_olm.py @@ -10,6 +10,7 @@ from mautrix.errors import DecryptionError, MatchingSessionDecryptionError from mautrix.types import ( + DecryptedOlmEvent, EncryptedOlmEventContent, EncryptionAlgorithm, IdentityKey, @@ -21,7 +22,6 @@ from .base import BaseOlmMachine from .sessions import Session -from .types import DecryptedOlmEvent class OlmDecryptionMachine(BaseOlmMachine): diff --git a/mautrix/crypto/device_lists.py b/mautrix/crypto/device_lists.py index 61fb34b2..b5e4cc03 100644 --- a/mautrix/crypto/device_lists.py +++ b/mautrix/crypto/device_lists.py @@ -6,10 +6,17 @@ from typing import Dict, List, Optional from mautrix.errors import DeviceValidationError -from mautrix.types import DeviceID, DeviceKeys, IdentityKey, SyncToken, UserID +from mautrix.types import ( + DeviceID, + DeviceIdentity, + DeviceKeys, + IdentityKey, + SyncToken, + TrustState, + UserID, +) from .base import BaseOlmMachine, verify_signature_json -from .types import DeviceIdentity, TrustState class DeviceListMachine(BaseOlmMachine): diff --git a/mautrix/crypto/encrypt_megolm.py b/mautrix/crypto/encrypt_megolm.py index 88c16041..e5830611 100644 --- a/mautrix/crypto/encrypt_megolm.py +++ b/mautrix/crypto/encrypt_megolm.py @@ -13,6 +13,7 @@ from mautrix.errors import EncryptionError, SessionShareError from mautrix.types import ( DeviceID, + DeviceIdentity, EncryptedMegolmEventContent, EncryptionAlgorithm, EventType, @@ -24,13 +25,13 @@ Serializable, SessionID, SigningKey, + TrustState, UserID, ) from .device_lists import DeviceListMachine from .encrypt_olm import OlmEncryptionMachine from .sessions import InboundGroupSession, OutboundGroupSession, Session -from .types import DeviceIdentity, TrustState class Sentinel: diff --git a/mautrix/crypto/encrypt_olm.py b/mautrix/crypto/encrypt_olm.py index 54ead8a9..e76fc9ef 100644 --- a/mautrix/crypto/encrypt_olm.py +++ b/mautrix/crypto/encrypt_olm.py @@ -7,17 +7,19 @@ import asyncio from mautrix.types import ( + DecryptedOlmEvent, DeviceID, + DeviceIdentity, EncryptedOlmEventContent, EncryptionKeyAlgorithm, EventType, + OlmEventKeys, ToDeviceEventContent, UserID, ) from .base import BaseOlmMachine, verify_signature_json from .sessions import Session -from .types import DecryptedOlmEvent, DeviceIdentity, OlmEventKeys ClaimKeysList = Dict[UserID, Dict[DeviceID, DeviceIdentity]] diff --git a/mautrix/crypto/key_request.py b/mautrix/crypto/key_request.py index a638fe28..b70cc947 100644 --- a/mautrix/crypto/key_request.py +++ b/mautrix/crypto/key_request.py @@ -8,6 +8,7 @@ import uuid from mautrix.types import ( + DecryptedOlmEvent, DeviceID, EncryptionAlgorithm, EventType, @@ -23,7 +24,6 @@ from .base import BaseOlmMachine from .sessions import InboundGroupSession -from .types import DecryptedOlmEvent class KeyRequestingMachine(BaseOlmMachine): diff --git a/mautrix/crypto/key_share.py b/mautrix/crypto/key_share.py index aba8d344..ca2d984f 100644 --- a/mautrix/crypto/key_share.py +++ b/mautrix/crypto/key_share.py @@ -7,6 +7,7 @@ from mautrix.errors import MatrixConnectionError, MatrixError, MatrixRequestError from mautrix.types import ( + DeviceIdentity, EncryptionAlgorithm, EventType, ForwardedRoomKeyEventContent, @@ -16,11 +17,11 @@ RoomKeyWithheldCode, RoomKeyWithheldEventContent, ToDeviceEvent, + TrustState, ) from .device_lists import DeviceListMachine from .encrypt_olm import OlmEncryptionMachine -from .types import DeviceIdentity, TrustState class RejectKeyShare(MatrixError): diff --git a/mautrix/crypto/machine.py b/mautrix/crypto/machine.py index e08accb7..8268ecc3 100644 --- a/mautrix/crypto/machine.py +++ b/mautrix/crypto/machine.py @@ -11,6 +11,7 @@ from mautrix import client as cli from mautrix.types import ( + DecryptedOlmEvent, DeviceLists, DeviceOTKCount, EncryptionAlgorithm, @@ -27,7 +28,6 @@ from .key_request import KeyRequestingMachine from .key_share import KeySharingMachine from .store import CryptoStore, StateStore -from .types import DecryptedOlmEvent from .unwedge import OlmUnwedgingMachine diff --git a/mautrix/crypto/store/abstract.py b/mautrix/crypto/store/abstract.py index a6d0a7ec..85c99f2f 100644 --- a/mautrix/crypto/store/abstract.py +++ b/mautrix/crypto/store/abstract.py @@ -9,6 +9,7 @@ from mautrix.types import ( DeviceID, + DeviceIdentity, EventID, IdentityKey, RoomEncryptionStateEventContent, @@ -17,7 +18,8 @@ UserID, ) -from .. import DeviceIdentity, InboundGroupSession, OlmAccount, OutboundGroupSession, Session +from ..account import OlmAccount +from ..sessions import InboundGroupSession, OutboundGroupSession, Session class StateStore(ABC): diff --git a/mautrix/crypto/store/asyncpg/store.py b/mautrix/crypto/store/asyncpg/store.py index 708b7433..cb9b232c 100644 --- a/mautrix/crypto/store/asyncpg/store.py +++ b/mautrix/crypto/store/asyncpg/store.py @@ -10,18 +10,21 @@ from mautrix.client.state_store import SyncStore from mautrix.client.state_store.asyncpg import PgStateStore -from mautrix.types import DeviceID, EventID, IdentityKey, RoomID, SessionID, SyncToken, UserID -from mautrix.util.async_db import Database, Scheme -from mautrix.util.logging import TraceLogger - -from ... import ( +from mautrix.types import ( + DeviceID, DeviceIdentity, - InboundGroupSession, - OlmAccount, - OutboundGroupSession, - Session, + EventID, + IdentityKey, + RoomID, + SessionID, + SyncToken, TrustState, + UserID, ) +from mautrix.util.async_db import Database, Scheme +from mautrix.util.logging import TraceLogger + +from ... import InboundGroupSession, OlmAccount, OutboundGroupSession, Session from ..abstract import CryptoStore, StateStore from .upgrade import upgrade_table diff --git a/mautrix/crypto/store/memory.py b/mautrix/crypto/store/memory.py index d3a73c9a..4aef397a 100644 --- a/mautrix/crypto/store/memory.py +++ b/mautrix/crypto/store/memory.py @@ -6,9 +6,19 @@ from __future__ import annotations from mautrix.client.state_store import SyncStore -from mautrix.types import DeviceID, EventID, IdentityKey, RoomID, SessionID, SyncToken, UserID - -from .. import DeviceIdentity, InboundGroupSession, OlmAccount, OutboundGroupSession, Session +from mautrix.types import ( + DeviceID, + DeviceIdentity, + EventID, + IdentityKey, + RoomID, + SessionID, + SyncToken, + UserID, +) + +from ..account import OlmAccount +from ..sessions import InboundGroupSession, OutboundGroupSession, Session from .abstract import CryptoStore diff --git a/mautrix/crypto/types.py b/mautrix/crypto/types.py deleted file mode 100644 index d6fc6c8b..00000000 --- a/mautrix/crypto/types.py +++ /dev/null @@ -1,52 +0,0 @@ -# Copyright (c) 2022 Tulir Asokan -# -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. -from typing import Optional -from enum import IntEnum - -from attr import dataclass -import attr - -from mautrix.types import ( - DeviceID, - IdentityKey, - SerializableAttrs, - SigningKey, - ToDeviceEvent, - UserID, -) - - -class TrustState(IntEnum): - UNSET = 0 - VERIFIED = 1 - BLACKLISTED = 2 - IGNORED = 3 - - -@dataclass -class DeviceIdentity: - user_id: UserID - device_id: DeviceID - identity_key: IdentityKey - signing_key: SigningKey - - trust: TrustState - deleted: bool - name: str - - -@dataclass -class OlmEventKeys(SerializableAttrs): - ed25519: SigningKey - - -@dataclass -class DecryptedOlmEvent(ToDeviceEvent, SerializableAttrs): - keys: OlmEventKeys - recipient: UserID - recipient_keys: OlmEventKeys - sender_device: Optional[DeviceID] = None - sender_key: IdentityKey = attr.ib(metadata={"hidden": True}, default=None) diff --git a/mautrix/types/__init__.py b/mautrix/types/__init__.py index 6e8da575..5837cebc 100644 --- a/mautrix/types/__init__.py +++ b/mautrix/types/__init__.py @@ -14,7 +14,16 @@ UserIdentifierType, WhoamiResponse, ) -from .crypto import ClaimKeysResponse, DeviceKeys, QueryKeysResponse, UnsignedDeviceInfo +from .crypto import ( + ClaimKeysResponse, + DecryptedOlmEvent, + DeviceIdentity, + DeviceKeys, + OlmEventKeys, + QueryKeysResponse, + TrustState, + UnsignedDeviceInfo, +) from .event import ( AccountDataEvent, AccountDataEventContent, diff --git a/mautrix/types/crypto.py b/mautrix/types/crypto.py index a58737ec..90c3f830 100644 --- a/mautrix/types/crypto.py +++ b/mautrix/types/crypto.py @@ -4,12 +4,13 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. from typing import Any, Dict, List, Optional +from enum import IntEnum from attr import dataclass -from .event.encrypted import EncryptionAlgorithm, EncryptionKeyAlgorithm +from .event import EncryptionAlgorithm, EncryptionKeyAlgorithm, ToDeviceEvent from .primitive import DeviceID, IdentityKey, SigningKey, UserID -from .util import SerializableAttrs +from .util import SerializableAttrs, field @dataclass @@ -55,3 +56,36 @@ class QueryKeysResponse(SerializableAttrs): class ClaimKeysResponse(SerializableAttrs): failures: Dict[str, Any] one_time_keys: Dict[UserID, Dict[DeviceID, Dict[str, Any]]] + + +class TrustState(IntEnum): + UNSET = 0 + VERIFIED = 1 + BLACKLISTED = 2 + IGNORED = 3 + + +@dataclass +class DeviceIdentity: + user_id: UserID + device_id: DeviceID + identity_key: IdentityKey + signing_key: SigningKey + + trust: TrustState + deleted: bool + name: str + + +@dataclass +class OlmEventKeys(SerializableAttrs): + ed25519: SigningKey + + +@dataclass +class DecryptedOlmEvent(ToDeviceEvent, SerializableAttrs): + keys: OlmEventKeys + recipient: UserID + recipient_keys: OlmEventKeys + sender_device: Optional[DeviceID] = None + sender_key: IdentityKey = field(hidden=True, default=None) From e1453f94fd8fbaf32284e9f149f318fff379d5ef Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 20 Feb 2022 16:24:42 +0200 Subject: [PATCH 014/456] Add __all__ to a bunch of places --- CHANGELOG.md | 1 + mautrix/appservice/__init__.py | 1 + mautrix/appservice/state_store/__init__.py | 2 + mautrix/bridge/__init__.py | 21 ++ mautrix/bridge/state_store/__init__.py | 1 + mautrix/client/__init__.py | 21 ++ mautrix/client/api/events.py | 6 +- mautrix/client/state_store/__init__.py | 10 + .../client/state_store/asyncpg/__init__.py | 2 + mautrix/client/syncer.py | 2 +- mautrix/crypto/__init__.py | 15 ++ mautrix/crypto/attachments/__init__.py | 8 + mautrix/crypto/store/asyncpg/__init__.py | 2 + mautrix/errors/__init__.py | 58 ++++++ mautrix/genall.py | 44 +++++ mautrix/types/__init__.py | 184 ++++++++++++++++++ mautrix/types/event/__init__.py | 1 + mautrix/util/config/__init__.py | 15 ++ mautrix/util/db/__init__.py | 2 + mautrix/util/formatter/__init__.py | 16 ++ mautrix/util/logging/__init__.py | 2 + 21 files changed, 409 insertions(+), 5 deletions(-) create mode 100644 mautrix/genall.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d022c74..1b712759 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ * **Breaking change *(crypto)*** Moved `TrustState`, `DeviceIdentity`, `OlmEventKeys` and `DecryptedOlmEvent` dataclasses from `crypto.types` into `types.crypto`. +* Added a redundant `__all__` to various `__init__.py` files to appease pyright. * *(bridge)* Removed legacy community utilities. * *(util.async_db)* Fixed counting number of db upgrades. * *(util.async_db)* Added support for schema migrations that jump versions. diff --git a/mautrix/appservice/__init__.py b/mautrix/appservice/__init__.py index a6028208..c4b2ae74 100644 --- a/mautrix/appservice/__init__.py +++ b/mautrix/appservice/__init__.py @@ -11,4 +11,5 @@ "ASStateStore", "AppServiceServerMixin", "DOUBLE_PUPPET_SOURCE_KEY", + "state_store", ] diff --git a/mautrix/appservice/state_store/__init__.py b/mautrix/appservice/state_store/__init__.py index 40e4bead..bc23b5f5 100644 --- a/mautrix/appservice/state_store/__init__.py +++ b/mautrix/appservice/state_store/__init__.py @@ -1,2 +1,4 @@ from .file import FileASStateStore from .memory import ASStateStore + +__all__ = ["FileASStateStore", "ASStateStore", "sqlalchemy", "asyncpg"] diff --git a/mautrix/bridge/__init__.py b/mautrix/bridge/__init__.py index 83ad6473..0da3cb9b 100644 --- a/mautrix/bridge/__init__.py +++ b/mautrix/bridge/__init__.py @@ -16,3 +16,24 @@ from .portal import BasePortal from .puppet import BasePuppet from .user import BaseUser + +__all__ = [ + "async_getter_lock", + "Bridge", + "BaseBridgeConfig", + "AutologinError", + "CustomPuppetError", + "CustomPuppetMixin", + "HomeserverURLNotFound", + "InvalidAccessToken", + "OnlyLoginSelf", + "OnlyLoginTrustedDomain", + "AbstractDisappearingMessage", + "BaseMatrixHandler", + "NotificationDisabler", + "BasePortal", + "BasePuppet", + "BaseUser", + "state_store", + "commands", +] diff --git a/mautrix/bridge/state_store/__init__.py b/mautrix/bridge/state_store/__init__.py index e69de29b..0e8137a5 100644 --- a/mautrix/bridge/state_store/__init__.py +++ b/mautrix/bridge/state_store/__init__.py @@ -0,0 +1 @@ +__all__ = ["asyncpg", "sqlalchemy"] diff --git a/mautrix/client/__init__.py b/mautrix/client/__init__.py index 2fe700ba..30c65988 100644 --- a/mautrix/client/__init__.py +++ b/mautrix/client/__init__.py @@ -5,3 +5,24 @@ from .state_store import FileStateStore, MemoryStateStore, MemorySyncStore, StateStore, SyncStore from .store_updater import StoreUpdatingAPI from .syncer import EventHandler, InternalEventType, Syncer, SyncStream + +__all__ = [ + "ClientAPI", + "Client", + "Dispatcher", + "MembershipEventDispatcher", + "SimpleDispatcher", + "DecryptionDispatcher", + "EncryptingAPI", + "FileStateStore", + "MemoryStateStore", + "MemorySyncStore", + "StateStore", + "SyncStore", + "StoreUpdatingAPI", + "EventHandler", + "InternalEventType", + "Syncer", + "SyncStream", + "state_store", +] diff --git a/mautrix/client/api/events.py b/mautrix/client/api/events.py index eb911fb4..8839edfe 100644 --- a/mautrix/client/api/events.py +++ b/mautrix/client/api/events.py @@ -43,7 +43,6 @@ TextMessageEventContent, UserID, ) -from mautrix.types.event.state import state_event_content_map from mautrix.util.formatter import parse_html from .base import BaseClientAPI @@ -152,10 +151,9 @@ async def get_state_event( Path.v3.rooms[room_id].state[event_type][state_key], metrics_method="getStateEvent", ) + content["__mautrix_event_type"] = event_type try: - return state_event_content_map[event_type].deserialize(content) - except KeyError: - return Obj(**content) + return StateEvent.deserialize_content(content) except SerializerError as e: raise MatrixResponseError("Invalid state event in response") from e diff --git a/mautrix/client/state_store/__init__.py b/mautrix/client/state_store/__init__.py index 0c8deb58..5f76c7dc 100644 --- a/mautrix/client/state_store/__init__.py +++ b/mautrix/client/state_store/__init__.py @@ -2,3 +2,13 @@ from .file import FileStateStore from .memory import MemoryStateStore from .sync import MemorySyncStore, SyncStore + +__all__ = [ + "StateStore", + "FileStateStore", + "MemoryStateStore", + "MemorySyncStore", + "SyncStore", + "asyncpg", + "sqlalchemy", +] diff --git a/mautrix/client/state_store/asyncpg/__init__.py b/mautrix/client/state_store/asyncpg/__init__.py index 7b97ca3f..a7fd8c63 100644 --- a/mautrix/client/state_store/asyncpg/__init__.py +++ b/mautrix/client/state_store/asyncpg/__init__.py @@ -1 +1,3 @@ from .store import PgStateStore + +__all__ = ["PgStateStore"] diff --git a/mautrix/client/syncer.py b/mautrix/client/syncer.py index 373a4597..4a052691 100644 --- a/mautrix/client/syncer.py +++ b/mautrix/client/syncer.py @@ -16,6 +16,7 @@ from mautrix.types import ( JSON, AccountDataEvent, + BaseMessageEventContentFuncs, DeviceLists, DeviceOTKCount, EphemeralEvent, @@ -31,7 +32,6 @@ ToDeviceEvent, UserID, ) -from mautrix.types.event.message import BaseMessageEventContentFuncs from mautrix.util.logging import TraceLogger from . import dispatcher diff --git a/mautrix/crypto/__init__.py b/mautrix/crypto/__init__.py index 15adb3ea..39867225 100644 --- a/mautrix/crypto/__init__.py +++ b/mautrix/crypto/__init__.py @@ -12,3 +12,18 @@ ) from .machine import OlmMachine # isort: skip + +__all__ = [ + "OlmAccount", + "RejectKeyShare", + "InboundGroupSession", + "OutboundGroupSession", + "Session", + "CryptoStore", + "MemoryCryptoStore", + "PgCryptoStateStore", + "PgCryptoStore", + "StateStore", + "OlmMachine", + "attachments", +] diff --git a/mautrix/crypto/attachments/__init__.py b/mautrix/crypto/attachments/__init__.py index ecf9d839..016bf808 100644 --- a/mautrix/crypto/attachments/__init__.py +++ b/mautrix/crypto/attachments/__init__.py @@ -1,2 +1,10 @@ from .async_attachments import async_encrypt_attachment, async_generator_from_data from .attachments import decrypt_attachment, encrypt_attachment, encrypted_attachment_generator + +__all__ = [ + "async_encrypt_attachment", + "async_generator_from_data", + "decrypt_attachment", + "encrypt_attachment", + "encrypted_attachment_generator", +] diff --git a/mautrix/crypto/store/asyncpg/__init__.py b/mautrix/crypto/store/asyncpg/__init__.py index 253ccb85..fe2645da 100644 --- a/mautrix/crypto/store/asyncpg/__init__.py +++ b/mautrix/crypto/store/asyncpg/__init__.py @@ -1 +1,3 @@ from .store import PgCryptoStateStore, PgCryptoStore + +__all__ = ["PgCryptoStore", "PgCryptoStateStore"] diff --git a/mautrix/errors/__init__.py b/mautrix/errors/__init__.py index 0adb3e2c..1bb90885 100644 --- a/mautrix/errors/__init__.py +++ b/mautrix/errors/__init__.py @@ -56,3 +56,61 @@ WellKnownUnexpectedStatus, WellKnownUnsupportedScheme, ) + +__all__ = [ + "IntentError", + "MatrixConnectionError", + "MatrixError", + "MatrixResponseError", + "CryptoError", + "DecryptedPayloadError", + "DecryptionError", + "DeviceValidationError", + "DuplicateMessageIndex", + "EncryptionError", + "MatchingSessionDecryptionError", + "MismatchingRoomError", + "SessionNotFound", + "SessionShareError", + "VerificationError", + "MatrixBadContent", + "MatrixBadRequest", + "MatrixInvalidToken", + "MatrixRequestError", + "MatrixStandardRequestError", + "MatrixUnknownRequestError", + "MBadJSON", + "MBadState", + "MCaptchaInvalid", + "MCaptchaNeeded", + "MExclusive", + "MForbidden", + "MGuestAccessForbidden", + "MIncompatibleRoomVersion", + "MInvalidParam", + "MInvalidRoomState", + "MInvalidUsername", + "MLimitExceeded", + "MMissingParam", + "MMissingToken", + "MNotFound", + "MNotJSON", + "MRoomInUse", + "MTooLarge", + "MUnauthorized", + "MUnknown", + "MUnknownToken", + "MUnrecognized", + "MUnsupportedRoomVersion", + "MUserDeactivated", + "MUserInUse", + "make_request_error", + "standard_error", + "WellKnownError", + "WellKnownInvalidVersionsResponse", + "WellKnownMissingHomeserver", + "WellKnownNotJSON", + "WellKnownNotURL", + "WellKnownUnexpectedStatus", + "WellKnownUnsupportedScheme", +] diff --git a/mautrix/genall.py b/mautrix/genall.py new file mode 100644 index 00000000..60bc01a8 --- /dev/null +++ b/mautrix/genall.py @@ -0,0 +1,44 @@ +# This script generates the __all__ arrays for types/__init__.py and errors/__init__.py +# to avoid having to manually add both the import and the __all__ entry. +# See https://github.com/mautrix/python/issues/90 for why __all__ is needed at all. +from pathlib import Path +import ast + +import black + +root_module = Path(__file__).parent + +black_cfg = black.parse_pyproject_toml(str(root_module.parent / "pyproject.toml")) +assert ( + black.__version__ == black_cfg["required_version"] +), f"Incorrect Black version {black.__version__}" +black_mode = black.Mode( + target_versions={black.TargetVersion[ver.upper()] for ver in black_cfg["target_version"]}, + line_length=black_cfg["line_length"], +) + + +def add_imports_to_all(dir: str) -> None: + init_file = root_module / dir / "__init__.py" + with open(init_file) as f: + init_ast = ast.parse(f.read(), filename=f"mautrix/{dir}/__init__.py") + + imports: list[str] = [] + all_node: ast.List | None = None + + for node in ast.iter_child_nodes(init_ast): + if isinstance(node, (ast.Import, ast.ImportFrom)): + imports += (name.name for name in node.names) + elif isinstance(node, ast.Assign) and isinstance(node.value, ast.List): + target = node.targets[0] + if len(node.targets) == 1 and isinstance(target, ast.Name) and target.id == "__all__": + all_node = node.value + + all_node.elts = [ast.Constant(name) for name in imports] + + with open(init_file, "w") as f: + f.write(black.format_str(ast.unparse(init_ast), mode=black_mode)) + + +add_imports_to_all("types") +add_imports_to_all("errors") diff --git a/mautrix/types/__init__.py b/mautrix/types/__init__.py index 5837cebc..2e99fd24 100644 --- a/mautrix/types/__init__.py +++ b/mautrix/types/__init__.py @@ -31,6 +31,7 @@ BaseEvent, BaseFileInfo, BaseMessageEventContent, + BaseMessageEventContentFuncs, BaseRoomEvent, BaseUnsigned, CallAnswerEventContent, @@ -178,3 +179,186 @@ field, serializer, ) + +__all__ = [ + "DiscoveryInformation", + "DiscoveryIntegrations", + "DiscoveryIntegrationServer", + "DiscoveryServer", + "LoginFlow", + "LoginFlowList", + "LoginResponse", + "LoginType", + "MatrixUserIdentifier", + "PhoneIdentifier", + "ThirdPartyIdentifier", + "UserIdentifier", + "UserIdentifierType", + "WhoamiResponse", + "ClaimKeysResponse", + "DecryptedOlmEvent", + "DeviceIdentity", + "DeviceKeys", + "OlmEventKeys", + "QueryKeysResponse", + "TrustState", + "UnsignedDeviceInfo", + "AccountDataEvent", + "AccountDataEventContent", + "AudioInfo", + "BaseEvent", + "BaseFileInfo", + "BaseMessageEventContent", + "BaseMessageEventContentFuncs", + "BaseRoomEvent", + "BaseUnsigned", + "CallAnswerEventContent", + "CallCandidate", + "CallCandidatesEventContent", + "CallData", + "CallDataType", + "CallEvent", + "CallEventContent", + "CallHangupEventContent", + "CallHangupReason", + "CallInviteEventContent", + "CallNegotiateEventContent", + "CallRejectEventContent", + "CallSelectAnswerEventContent", + "CanonicalAliasStateEventContent", + "EncryptedEvent", + "EncryptedEventContent", + "EncryptedFile", + "EncryptedMegolmEventContent", + "EncryptedOlmEventContent", + "EncryptionAlgorithm", + "EncryptionKeyAlgorithm", + "EphemeralEvent", + "Event", + "EventContent", + "EventType", + "FileInfo", + "Format", + "ForwardedRoomKeyEventContent", + "GenericEvent", + "ImageInfo", + "JoinRule", + "JoinRulesStateEventContent", + "JSONWebKey", + "KeyRequestAction", + "LocationInfo", + "LocationMessageEventContent", + "MediaInfo", + "MediaMessageEventContent", + "Membership", + "MemberStateEventContent", + "MessageEvent", + "MessageEventContent", + "MessageType", + "MessageUnsigned", + "OlmCiphertext", + "OlmMsgType", + "PowerLevelStateEventContent", + "PresenceEvent", + "PresenceEventContent", + "PresenceState", + "ReactionEvent", + "ReactionEventContent", + "ReceiptEvent", + "ReceiptEventContent", + "ReceiptType", + "RedactionEvent", + "RedactionEventContent", + "RelatesTo", + "RelationType", + "RequestedKeyInfo", + "RoomAvatarStateEventContent", + "RoomCreateStateEventContent", + "RoomEncryptionStateEventContent", + "RoomKeyEventContent", + "RoomKeyRequestEventContent", + "RoomKeyWithheldCode", + "RoomKeyWithheldEventContent", + "RoomNameStateEventContent", + "RoomPinnedEventsStateEventContent", + "RoomPredecessor", + "RoomTagAccountDataEventContent", + "RoomTagInfo", + "RoomTombstoneStateEventContent", + "RoomTopicStateEventContent", + "SingleReceiptEventContent", + "SpaceChildStateEventContent", + "SpaceParentStateEventContent", + "StateEvent", + "StateEventContent", + "StateUnsigned", + "StrippedStateEvent", + "TextMessageEventContent", + "ThumbnailInfo", + "ToDeviceEvent", + "ToDeviceEventContent", + "TypingEvent", + "TypingEventContent", + "VideoInfo", + "EventFilter", + "Filter", + "RoomEventFilter", + "RoomFilter", + "StateFilter", + "IdentifierType", + "MatrixURI", + "MatrixURIError", + "URIAction", + "MediaRepoConfig", + "MXOpenGraph", + "OpenGraphAudio", + "OpenGraphImage", + "OpenGraphVideo", + "BatchSendResponse", + "DeviceLists", + "DeviceOTKCount", + "DirectoryPaginationToken", + "PaginatedMessages", + "PaginationDirection", + "RoomAliasInfo", + "RoomCreatePreset", + "RoomDirectoryResponse", + "RoomDirectoryVisibility", + "VersionsResponse", + "JSON", + "BatchID", + "ContentURI", + "DeviceID", + "EventID", + "FilterID", + "IdentityKey", + "RoomAlias", + "RoomID", + "SessionID", + "SigningKey", + "SyncToken", + "UserID", + "PushAction", + "PushActionDict", + "PushActionType", + "PushCondition", + "PushConditionKind", + "PushOperator", + "PushRule", + "PushRuleID", + "PushRuleKind", + "PushRuleScope", + "Member", + "User", + "UserSearchResults", + "ExtensibleEnum", + "Lst", + "Obj", + "Serializable", + "SerializableAttrs", + "SerializableEnum", + "SerializerError", + "deserializer", + "field", + "serializer", +] diff --git a/mautrix/types/event/__init__.py b/mautrix/types/event/__init__.py index 930e05cc..c36b92c4 100644 --- a/mautrix/types/event/__init__.py +++ b/mautrix/types/event/__init__.py @@ -37,6 +37,7 @@ AudioInfo, BaseFileInfo, BaseMessageEventContent, + BaseMessageEventContentFuncs, EncryptedFile, FileInfo, Format, diff --git a/mautrix/util/config/__init__.py b/mautrix/util/config/__init__.py index 87ee3262..89322556 100644 --- a/mautrix/util/config/__init__.py +++ b/mautrix/util/config/__init__.py @@ -4,3 +4,18 @@ from .recursive_dict import RecursiveDict from .string import BaseStringConfig from .validation import BaseValidatableConfig, ConfigValueError, ForbiddenDefault, ForbiddenKey + +__all__ = [ + "BaseConfig", + "BaseMissingError", + "ConfigUpdateHelper", + "BaseFileConfig", + "yaml", + "BaseProxyConfig", + "RecursiveDict", + "BaseStringConfig", + "BaseValidatableConfig", + "ConfigValueError", + "ForbiddenDefault", + "ForbiddenKey", +] diff --git a/mautrix/util/db/__init__.py b/mautrix/util/db/__init__.py index 15ade9f4..b13bcfd6 100644 --- a/mautrix/util/db/__init__.py +++ b/mautrix/util/db/__init__.py @@ -1 +1,3 @@ from .base import Base, BaseClass + +__all__ = ["Base", "BaseClass"] diff --git a/mautrix/util/formatter/__init__.py b/mautrix/util/formatter/__init__.py index 2a8c10ad..f922ff57 100644 --- a/mautrix/util/formatter/__init__.py +++ b/mautrix/util/formatter/__init__.py @@ -12,3 +12,19 @@ async def parse_html(input_html: str) -> str: return (await MatrixParser().parse(input_html)).text + + +__all__ = [ + "AbstractEntity", + "EntityString", + "SemiAbstractEntity", + "SimpleEntity", + "EntityType", + "FormattedString", + "HTMLNode", + "read_html", + "MarkdownString", + "MatrixParser", + "RecursionContext", + "parse_html", +] diff --git a/mautrix/util/logging/__init__.py b/mautrix/util/logging/__init__.py index fedf9dc6..e0792a17 100644 --- a/mautrix/util/logging/__init__.py +++ b/mautrix/util/logging/__init__.py @@ -1,2 +1,4 @@ from .color import ColorFormatter from .trace import SILLY, TRACE, TraceLogger + +__all__ = ["ColorFormatter", "TraceLogger", "SILLY", "TRACE"] From eb4bb22086d7c49f74d10e69b692548193b9e03d Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 20 Feb 2022 16:25:23 +0200 Subject: [PATCH 015/456] Update changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b712759..c733483c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ ## v0.15.0 (unreleased) -* **Breaking change *(\*)*** Removed Python 3.7 support. +* **Breaking change** Removed Python 3.7 support. * **Breaking change *(api)*** Removed `r0` from default path builders in order to update to `v3` and per-endpoint versioning. * The client API modules have been updated to specify v3 in the paths, other From 678561a85f86778ec00808b181acfc978c4a377d Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 22 Feb 2022 11:55:41 +0200 Subject: [PATCH 016/456] Bump version to 0.15.0rc1 --- mautrix/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/__init__.py b/mautrix/__init__.py index f5e75340..2a2693f0 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.14.10" +__version__ = "0.15.0rc1" __author__ = "Tulir Asokan " __all__ = [ "api", From e549149b745cf08a119a4800b2bd1852544a67b4 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 22 Feb 2022 12:39:27 +0200 Subject: [PATCH 017/456] Fix downloading attachments --- mautrix/__init__.py | 2 +- mautrix/api.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 2a2693f0..379c4c00 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.15.0rc1" +__version__ = "0.15.0rc2" __author__ = "Tulir Asokan " __all__ = [ "api", diff --git a/mautrix/api.py b/mautrix/api.py index c6b6b4a3..2b33ee5e 100644 --- a/mautrix/api.py +++ b/mautrix/api.py @@ -406,6 +406,7 @@ def get_download_url( "https://matrix.org/_matrix/media/r0/download/matrix.org/pqjkOuKZ1ZKRULWXgz2IVZV6" """ if mxc_uri.startswith("mxc://"): - return self.base_url / str(APIPath.MEDIA) / download_type / mxc_uri[6:] + version = "r0" if self.hacky_replace_v3_with_r0 else "v3" + return self.base_url / str(APIPath.MEDIA) / version / download_type / mxc_uri[6:] else: raise ValueError("MXC URI did not begin with `mxc://`") From 96e839dfcbf4dddffc16096962105e5e6f999df5 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 22 Feb 2022 13:27:43 +0200 Subject: [PATCH 018/456] Replace path in shared secret login --- mautrix/__init__.py | 2 +- mautrix/bridge/custom_puppet.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 379c4c00..5273a7ff 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.15.0rc2" +__version__ = "0.15.0rc3" __author__ = "Tulir Asokan " __all__ = [ "api", diff --git a/mautrix/bridge/custom_puppet.py b/mautrix/bridge/custom_puppet.py index c5c77f25..db0027d6 100644 --- a/mautrix/bridge/custom_puppet.py +++ b/mautrix/bridge/custom_puppet.py @@ -171,6 +171,8 @@ async def _login_with_shared_secret(cls, mxid: UserID) -> str: raise AutologinError(f"No homeserver URL configured for {server}") password = hmac.new(secret, mxid.encode("utf-8"), hashlib.sha512).hexdigest() url = base_url / str(Path.v3.login) + if cls.az.intent.api.hacky_replace_v3_with_r0: + url = base_url / str(Path.r0.login) resp = await cls.az.http_session.post( url, data=json.dumps( From 49b0beedc92313679ea2fe951afe8032a784a53b Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 22 Feb 2022 13:41:21 +0200 Subject: [PATCH 019/456] Fix upgrading database without jumping --- mautrix/__init__.py | 2 +- mautrix/util/async_db/upgrade.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 5273a7ff..f98581f9 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.15.0rc3" +__version__ = "0.15.0rc4" __author__ = "Tulir Asokan " __all__ = [ "api", diff --git a/mautrix/util/async_db/upgrade.py b/mautrix/util/async_db/upgrade.py index 0b1dd438..6ee2807a 100644 --- a/mautrix/util/async_db/upgrade.py +++ b/mautrix/util/async_db/upgrade.py @@ -125,7 +125,9 @@ async def upgrade(self, db: async_db.Database) -> None: while version < len(self.upgrades): old_version = version upgrade = self.upgrades[version] - new_version = getattr(upgrade, "__mau_db_upgrade_destination__", version + 1) + new_version = ( + getattr(upgrade, "__mau_db_upgrade_destination__", None) or version + 1 + ) if callable(new_version): new_version = await new_version(conn, db.scheme) desc = getattr(upgrade, "__mau_db_upgrade_description__", None) From 548dc3c90edbc03c5752904578b962cf7b426df7 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 28 Feb 2022 20:47:40 +0200 Subject: [PATCH 020/456] Update some comments --- mautrix/api.py | 2 +- mautrix/types/media.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/mautrix/api.py b/mautrix/api.py index 2b33ee5e..7a3b0260 100644 --- a/mautrix/api.py +++ b/mautrix/api.py @@ -403,7 +403,7 @@ def get_download_url( Examples: >>> api = HTTPAPI(...) >>> api.get_download_url("mxc://matrix.org/pqjkOuKZ1ZKRULWXgz2IVZV6") - "https://matrix.org/_matrix/media/r0/download/matrix.org/pqjkOuKZ1ZKRULWXgz2IVZV6" + "https://matrix.org/_matrix/media/v3/download/matrix.org/pqjkOuKZ1ZKRULWXgz2IVZV6" """ if mxc_uri.startswith("mxc://"): version = "r0" if self.hacky_replace_v3_with_r0 else "v3" diff --git a/mautrix/types/media.py b/mautrix/types/media.py index ec005be5..38746cbf 100644 --- a/mautrix/types/media.py +++ b/mautrix/types/media.py @@ -12,9 +12,9 @@ @dataclass class MediaRepoConfig(SerializableAttrs): """ - Matrix media repo config. See `GET /_matrix/media/r0/config`_. + Matrix media repo config. See `GET /_matrix/media/v3/config`_. - .. _GET /_matrix/media/r0/config: + .. _GET /_matrix/media/v3/config: https://spec.matrix.org/v1.2/client-server-api/#get_matrixmediav3config """ @@ -48,9 +48,9 @@ class OpenGraphAudio(SerializableAttrs): @dataclass class MXOpenGraph(SerializableAttrs): """ - Matrix URL preview response. See `GET /_matrix/media/r0/preview_url`_. + Matrix URL preview response. See `GET /_matrix/media/v3/preview_url`_. - .. _GET /_matrix/media/r0/preview_url: + .. _GET /_matrix/media/v3/preview_url: https://spec.matrix.org/v1.2/client-server-api/#get_matrixmediav3preview_url """ From 4a44f04580ab26c6e534a2ab39acafadb944f584 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 4 Mar 2022 01:48:35 +0200 Subject: [PATCH 021/456] Add support for creating DM portals in a generic way --- CHANGELOG.md | 3 + mautrix/bridge/__init__.py | 2 +- mautrix/bridge/matrix.py | 105 +++++++++++++++---- mautrix/bridge/portal.py | 177 +++++++++++++++++++++++++++++++- mautrix/bridge/user.py | 24 +++++ mautrix/types/__init__.py | 2 + mautrix/types/event/__init__.py | 1 + 7 files changed, 292 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c733483c..67fefa3e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,9 @@ * **Breaking change *(crypto)*** Moved `TrustState`, `DeviceIdentity`, `OlmEventKeys` and `DecryptedOlmEvent` dataclasses from `crypto.types` into `types.crypto`. +* **Breaking change *(bridge)*** Made `User.get_puppet` abstract and added new + abstract `User.get_portal_with` and `Portal.get_dm_puppet` methods. +* Added support for creating DM portals with minimal bridge-specific code. * Added a redundant `__all__` to various `__init__.py` files to appease pyright. * *(bridge)* Removed legacy community utilities. * *(util.async_db)* Fixed counting number of db upgrades. diff --git a/mautrix/bridge/__init__.py b/mautrix/bridge/__init__.py index 0da3cb9b..3ee081f4 100644 --- a/mautrix/bridge/__init__.py +++ b/mautrix/bridge/__init__.py @@ -13,7 +13,7 @@ from .disappearing_message import AbstractDisappearingMessage from .matrix import BaseMatrixHandler from .notification_disabler import NotificationDisabler -from .portal import BasePortal +from .portal import BasePortal, DMCreateError, IgnoreMatrixInvite, RejectMatrixInvite from .puppet import BasePuppet from .user import BaseUser diff --git a/mautrix/bridge/matrix.py b/mautrix/bridge/matrix.py index fc853451..bdfd922f 100644 --- a/mautrix/bridge/matrix.py +++ b/mautrix/bridge/matrix.py @@ -39,6 +39,7 @@ ReceiptType, RedactionEvent, RoomID, + RoomType, SingleReceiptEventContent, StateEvent, StateUnsigned, @@ -217,6 +218,10 @@ async def allow_command(user: br.BaseUser) -> bool: async def allow_bridging_message(user: br.BaseUser, portal: br.BasePortal) -> bool: return await user.is_logged_in() or (user.relay_whitelisted and portal.has_relay) + @staticmethod + async def allow_puppet_invite(user: br.BaseUser, puppet: br.BasePuppet) -> bool: + return await user.is_logged_in() + async def handle_leave(self, room_id: RoomID, user_id: UserID, event_id: EventID) -> None: pass @@ -248,13 +253,81 @@ async def handle_member_info_change( ) -> None: pass + async def handle_puppet_group_invite( + self, + room_id: RoomID, + puppet: br.BasePuppet, + invited_by: br.BaseUser, + evt: StateEvent, + members: list[UserID], + ) -> None: + if self.az.bot_mxid not in members: + await puppet.default_mxid_intent.leave_room( + room_id, reason="This ghost does not join multi-user rooms without the bridge bot." + ) + + async def handle_puppet_dm_invite( + self, room_id: RoomID, puppet: br.BasePuppet, invited_by: br.BaseUser, evt: StateEvent + ) -> None: + portal = await invited_by.get_portal_with(puppet) + if portal: + await portal.accept_matrix_dm(room_id, invited_by, puppet) + else: + await puppet.default_mxid_intent.leave_room( + room_id, reason="This bridge does not support creating DMs." + ) + + async def handle_puppet_space_invite( + self, room_id: RoomID, puppet: br.BasePuppet, invited_by: br.BaseUser, evt: StateEvent + ) -> None: + await puppet.default_mxid_intent.leave_room( + room_id, reason="This ghost does not join spaces." + ) + + async def handle_puppet_nonportal_invite( + self, room_id: RoomID, puppet: br.BasePuppet, invited_by: br.BaseUser, evt: StateEvent + ) -> None: + intent = puppet.default_mxid_intent + await intent.join_room(room_id) + try: + create_evt = await intent.get_state_event(room_id, EventType.ROOM_CREATE) + members = await intent.get_room_members(room_id) + except MatrixError: + self.log.exception(f"Failed to get state after joining {room_id} as {intent.mxid}") + asyncio.create_task(intent.leave_room(room_id, reason="Internal error")) + return + if create_evt.type == RoomType.SPACE: + await self.handle_puppet_space_invite(room_id, puppet, invited_by, evt) + elif len(members) > 2: + await self.handle_puppet_group_invite(room_id, puppet, invited_by, evt, members) + else: + await self.handle_puppet_dm_invite(room_id, puppet, invited_by, evt) + async def handle_puppet_invite( - self, room_id: RoomID, puppet: br.BasePuppet, invited_by: br.BaseUser, event_id: EventID + self, room_id: RoomID, puppet: br.BasePuppet, invited_by: br.BaseUser, evt: StateEvent ) -> None: - pass + intent = puppet.default_mxid_intent + if not await self.allow_puppet_invite(invited_by, puppet): + self.log.debug(f"Rejecting invite for {intent.mxid} to {room_id}: user can't invite") + await intent.leave_room(room_id, reason="You're not allowed to invite this ghost.") + return + + portal = await self.bridge.get_portal(room_id) + if portal: + try: + await portal.handle_matrix_invite(invited_by, puppet) + except br.RejectMatrixInvite as e: + await intent.leave_room(room_id, reason=e.message) + except br.IgnoreMatrixInvite: + pass + else: + await intent.join_room(room_id) + return + else: + await self.handle_puppet_nonportal_invite(room_id, puppet, invited_by, evt) async def handle_invite( - self, room_id: RoomID, user_id: UserID, inviter: br.BaseUser, event_id: EventID + self, room_id: RoomID, user_id: UserID, invited_by: br.BaseUser, evt: StateEvent ) -> None: pass @@ -363,26 +436,22 @@ async def send_welcome_message(self, room_id: RoomID, inviter: br.BaseUser) -> N combined_html = "".join(map(markdown.render, welcome_messages)) await self.az.intent.send_notice(room_id, text=combined, html=combined_html) - async def int_handle_invite( - self, room_id: RoomID, user_id: UserID, invited_by: UserID, event_id: EventID - ) -> None: - self.log.debug(f"{invited_by} invited {user_id} to {room_id}") - inviter = await self.bridge.get_user(invited_by) + async def int_handle_invite(self, evt: StateEvent) -> None: + self.log.debug(f"{evt.sender} invited {evt.state_key} to {evt.room_id}") + inviter = await self.bridge.get_user(evt.sender) if inviter is None: - self.log.exception(f"Failed to find user with Matrix ID {invited_by}") + self.log.exception(f"Failed to find user with Matrix ID {evt.sender}") return - elif user_id == self.az.bot_mxid: - await self.accept_bot_invite(room_id, inviter) - return - elif not await self.allow_command(inviter): + elif evt.state_key == self.az.bot_mxid: + await self.accept_bot_invite(evt.room_id, inviter) return - puppet = await self.bridge.get_puppet(user_id) + puppet = await self.bridge.get_puppet(UserID(evt.state_key)) if puppet: - await self.handle_puppet_invite(room_id, puppet, inviter, event_id) + await self.handle_puppet_invite(evt.room_id, puppet, inviter, evt) return - await self.handle_invite(room_id, user_id, inviter, event_id) + await self.handle_invite(evt.room_id, UserID(evt.state_key), inviter, evt) def is_command(self, message: MessageEventContent) -> tuple[bool, str]: text = message.body @@ -748,9 +817,7 @@ async def int_handle_event(self, evt: Event, send_bridge_checkpoint: bool = True prev_content = unsigned.prev_content or MemberStateEventContent() prev_membership = prev_content.membership if prev_content else Membership.JOIN if evt.content.membership == Membership.INVITE: - await self.int_handle_invite( - evt.room_id, UserID(evt.state_key), evt.sender, evt.event_id - ) + await self.int_handle_invite(evt) elif evt.content.membership == Membership.LEAVE: if prev_membership == Membership.BAN: await self.handle_unban( diff --git a/mautrix/bridge/portal.py b/mautrix/bridge/portal.py index 8e6db600..705a1ace 100644 --- a/mautrix/bridge/portal.py +++ b/mautrix/bridge/portal.py @@ -5,7 +5,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import annotations -from typing import Any, ClassVar, NamedTuple +from typing import Any, NamedTuple from abc import ABC, abstractmethod from collections import defaultdict from string import Template @@ -15,7 +15,7 @@ import time from mautrix.appservice import DOUBLE_PUPPET_SOURCE_KEY, AppService, IntentAPI -from mautrix.errors import MatrixError, MatrixRequestError, MNotFound +from mautrix.errors import MatrixError, MatrixRequestError, MForbidden, MNotFound from mautrix.types import ( EncryptionAlgorithm, EventID, @@ -25,6 +25,8 @@ MessageType, RoomEncryptionStateEventContent, RoomID, + RoomTombstoneStateEventContent, + TextMessageEventContent, UserID, ) from mautrix.util.logging import TraceLogger @@ -38,6 +40,24 @@ class RelaySender(NamedTuple): is_relay: bool +class RejectMatrixInvite(Exception): + def __init__(self, message: str) -> None: + super().__init__(message) + self.message = message + + +class IgnoreMatrixInvite(Exception): + pass + + +class DMCreateError(RejectMatrixInvite): + """ + An error raised by :meth:`BasePortal.prepare_dm` if the DM can't be set up. + + The message in the exception will be sent to the user as a message before the ghost leaves. + """ + + class BasePortal(ABC): log: TraceLogger = logging.getLogger("mau.portal") _async_get_locks: dict[Any, asyncio.Lock] = defaultdict(lambda: asyncio.Lock()) @@ -66,12 +86,151 @@ def __init__(self) -> None: async def save(self) -> None: pass + @abstractmethod + async def get_dm_puppet(self) -> br.BasePuppet | None: + """ + Get the ghost representing the other end of this direct chat. + + Returns: + A puppet entity, or ``None`` if this is not a 1:1 chat. + """ + @abstractmethod async def handle_matrix_message( self, sender: br.BaseUser, message: MessageEventContent, event_id: EventID ) -> None: pass + async def prepare_remote_dm( + self, room_id: RoomID, invited_by: br.BaseUser, puppet: br.BasePuppet + ) -> str: + """ + Do whatever is needed on the remote platform to set up a direct chat between the user + and the ghost. By default, this does nothing (and lets :meth:`setup_matrix_dm` handle + everything). + + Args: + room_id: The room ID that will be used. + invited_by: The Matrix user who invited the ghost. + puppet: The ghost who was invited. + + Returns: + A simple message indicating what was done (will be sent as a notice to the room). + If empty, the message won't be sent. + + Raises: + DMCreateError: if the DM could not be created and the ghost should leave the room. + """ + return "Portal to private chat created." + + async def postprocess_matrix_dm(self, user: br.BaseUser, puppet: br.BasePuppet) -> None: + await self.update_bridge_info() + + async def reject_duplicate_dm( + self, room_id: RoomID, invited_by: br.BaseUser, puppet: br.BasePuppet + ) -> None: + try: + await puppet.default_mxid_intent.send_notice( + room_id, + text=f"You already have a private chat with me: {self.mxid}", + html=( + "You already have a private chat with me: " + f"Link to room" + ), + ) + except Exception as e: + self.log.debug(f"Failed to send notice to duplicate private chat room: {e}") + + try: + await puppet.default_mxid_intent.send_state_event( + room_id, + event_type=EventType.ROOM_TOMBSTONE, + content=RoomTombstoneStateEventContent( + replacement_room=self.mxid, + body="You already have a private chat with me", + ), + ) + except Exception as e: + self.log.debug(f"Failed to send tombstone to duplicate private chat room: {e}") + + await puppet.default_mxid_intent.leave_room(room_id) + + async def accept_matrix_dm( + self, room_id: RoomID, invited_by: br.BaseUser, puppet: br.BasePuppet + ) -> None: + """ + Set up a room as a direct chat portal. + + The ghost has already accepted the invite at this point, so this method needs to make it + leave if the DM can't be created for some reason. + + By default, this checks if there's an existing portal and redirects the user there if it + does exist. If a portal doesn't exist, this will call :meth:`prepare_matrix_dm` and then + save the room ID, enable encryption and update bridge info. If the portal exists, but isn't + usable, the old room will be cleaned up and the function will continue. + + Args: + room_id: The room ID that will be used. + invited_by: The Matrix user who invited the ghost. + puppet: The ghost who was invited. + """ + if self.mxid: + try: + portal_members = await self.main_intent.get_room_members(self.mxid) + except (MForbidden, MNotFound): + portal_members = [] + if invited_by.mxid in portal_members: + await self.reject_duplicate_dm(room_id, invited_by, puppet) + return + self.log.debug( + f"{invited_by.mxid} isn't in old portal room {self.mxid}," + " cleaning up and accepting new room as the DM portal" + ) + await self.cleanup_portal( + message="User seems to have left DM portal", puppets_only=True + ) + try: + message = await self.prepare_remote_dm(room_id, invited_by, puppet) + except DMCreateError as e: + if e.message: + await puppet.default_mxid_intent.send_notice(room_id, text=e.message) + await puppet.default_mxid_intent.leave_room(room_id, reason="Failed to create DM") + return + self.mxid = room_id + e2be_ok = await self.check_dm_encryption() + await self.save() + if e2be_ok is False: + message += "\n\nWarning: Failed to enable end-to-bridge encryption." + if message: + await self._send_message( + puppet.default_mxid_intent, + TextMessageEventContent( + msgtype=MessageType.NOTICE, + body=message, + ), + ) + await self.postprocess_matrix_dm(invited_by, puppet) + + async def handle_matrix_invite(self, invited_by: br.BaseUser, puppet: br.BasePuppet) -> None: + """ + Called when a Matrix user invites a bridge ghost to a room to process the invite (and check + if it should be accepted). + + Args: + invited_by: The user who invited the ghost. + puppet: The ghost who was invited. + + Raises: + RejectMatrixInvite: if the invite should be rejected. + IgnoreMatrixInvite: if the invite should be ignored (e.g. if it was already accepted). + """ + if self.is_direct: + raise RejectMatrixInvite("You can't invite additional users to private chats.") + raise RejectMatrixInvite("This bridge does not implement inviting users to portals.") + + async def update_bridge_info(self) -> None: + """Resend the ``m.bridge`` event into the room.""" + @property def _relay_is_implemented(self) -> bool: return hasattr(self, "relay_user_id") and hasattr(self, "_relay_user") @@ -178,8 +337,22 @@ async def enable_dm_encryption(self) -> bool: return False self.encrypted = True + await self.update_info_from_puppet() return True + async def update_info_from_puppet(self, puppet: br.BasePuppet | None = None) -> None: + """ + Update the room metadata to match the ghost's name/avatar. + + This is called after enabling encryption, as the bridge bot needs to join for e2ee, + but that messes up the default name generation. If/when canonical DMs happen, + this might not be necessary anymore. + + Args: + puppet: The ghost that is the other participant in the room. + If ``None``, the entity should be fetched as necessary. + """ + @property def disappearing_enabled(self) -> bool: return bool(self.disappearing_msg_class) diff --git a/mautrix/bridge/user.py b/mautrix/bridge/user.py index 63fd96f4..36d7bdc6 100644 --- a/mautrix/bridge/user.py +++ b/mautrix/bridge/user.py @@ -68,9 +68,33 @@ def __init__(self) -> None: async def is_logged_in(self) -> bool: raise NotImplementedError() + @abstractmethod async def get_puppet(self) -> br.BasePuppet | None: + """ + Get the ghost that represents this Matrix user on the remote network. + + Returns: + The puppet entity, or ``None`` if the user is not logged in, + or it's otherwise not possible to find the remote ghost. + """ raise NotImplementedError() + @abstractmethod + async def get_portal_with( + self, puppet: br.BasePuppet, create: bool = True + ) -> br.BasePortal | None: + """ + Get a private chat portal between this user and the given ghost. + + Args: + puppet: The ghost who the portal should be with. + create: ``True`` if the portal entity should be created if it doesn't exist. + + Returns: + The portal entity, or ``None`` if it can't be found, + or doesn't exist and ``create`` is ``False``. + """ + async def needs_relay(self, portal: br.BasePortal) -> bool: return not await self.is_logged_in() diff --git a/mautrix/types/__init__.py b/mautrix/types/__init__.py index 2e99fd24..352ea4bd 100644 --- a/mautrix/types/__init__.py +++ b/mautrix/types/__init__.py @@ -108,6 +108,7 @@ RoomTagInfo, RoomTombstoneStateEventContent, RoomTopicStateEventContent, + RoomType, SingleReceiptEventContent, SpaceChildStateEventContent, SpaceParentStateEventContent, @@ -286,6 +287,7 @@ "RoomTagInfo", "RoomTombstoneStateEventContent", "RoomTopicStateEventContent", + "RoomType", "SingleReceiptEventContent", "SpaceChildStateEventContent", "SpaceParentStateEventContent", diff --git a/mautrix/types/event/__init__.py b/mautrix/types/event/__init__.py index c36b92c4..5cc6cf91 100644 --- a/mautrix/types/event/__init__.py +++ b/mautrix/types/event/__init__.py @@ -74,6 +74,7 @@ RoomPredecessor, RoomTombstoneStateEventContent, RoomTopicStateEventContent, + RoomType, SpaceChildStateEventContent, SpaceParentStateEventContent, StateEvent, From ebacede313816755348c90caf9306472f31a7a7c Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 4 Mar 2022 01:50:29 +0200 Subject: [PATCH 022/456] Add room meta to checkpoint types --- mautrix/util/message_send_checkpoint.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mautrix/util/message_send_checkpoint.py b/mautrix/util/message_send_checkpoint.py index 5c0e052e..eb83c6df 100644 --- a/mautrix/util/message_send_checkpoint.py +++ b/mautrix/util/message_send_checkpoint.py @@ -83,6 +83,9 @@ async def send(self, endpoint: str, as_token: str, log: logging.Logger) -> None: EventType.ROOM_MESSAGE, EventType.ROOM_ENCRYPTED, EventType.ROOM_MEMBER, + EventType.ROOM_NAME, + EventType.ROOM_AVATAR, + EventType.ROOM_TOPIC, EventType.STICKER, EventType.REACTION, EventType.CALL_INVITE, From 48e7c92881384b4a48ed374907669136e65af6e5 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 4 Mar 2022 01:51:15 +0200 Subject: [PATCH 023/456] Bump version to 0.15.0rc5 --- mautrix/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/__init__.py b/mautrix/__init__.py index f98581f9..61ff1b12 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.15.0rc4" +__version__ = "0.15.0rc5" __author__ = "Tulir Asokan " __all__ = [ "api", From 40bafed2c3eec27b5001e211ff1e426546e62143 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 8 Mar 2022 12:24:49 +0200 Subject: [PATCH 024/456] Add method to encrypt a bytearray in-place --- mautrix/crypto/attachments/__init__.py | 15 +++++++- .../crypto/attachments/async_attachments.py | 18 ++++----- mautrix/crypto/attachments/attachments.py | 37 ++++++++++++------- 3 files changed, 45 insertions(+), 25 deletions(-) diff --git a/mautrix/crypto/attachments/__init__.py b/mautrix/crypto/attachments/__init__.py index 016bf808..41ee2c34 100644 --- a/mautrix/crypto/attachments/__init__.py +++ b/mautrix/crypto/attachments/__init__.py @@ -1,10 +1,21 @@ -from .async_attachments import async_encrypt_attachment, async_generator_from_data -from .attachments import decrypt_attachment, encrypt_attachment, encrypted_attachment_generator +from .async_attachments import ( + async_encrypt_attachment, + async_generator_from_data, + async_inplace_encrypt_attachment, +) +from .attachments import ( + decrypt_attachment, + encrypt_attachment, + encrypted_attachment_generator, + inplace_encrypt_attachment, +) __all__ = [ "async_encrypt_attachment", "async_generator_from_data", + "async_inplace_encrypt_attachment", "decrypt_attachment", "encrypt_attachment", "encrypted_attachment_generator", + "inplace_encrypt_attachment", ] diff --git a/mautrix/crypto/attachments/async_attachments.py b/mautrix/crypto/attachments/async_attachments.py index 23037837..57b6fc6f 100644 --- a/mautrix/crypto/attachments/async_attachments.py +++ b/mautrix/crypto/attachments/async_attachments.py @@ -19,7 +19,7 @@ from mautrix.types import EncryptedFile -from .attachments import AES, SHA256, Counter, Random, _get_decryption_info +from .attachments import _get_decryption_info, _prepare_encryption, inplace_encrypt_attachment async def async_encrypt_attachment( @@ -48,16 +48,9 @@ async def async_encrypt_attachment( | hashes.sha256: Base64 encoded SHA-256 hash of the ciphertext. """ - key = Random.new().read(32) - # 8 bytes IV - iv = Random.new().read(8) - # 8 bytes counter, prefixed by the IV - ctr = Counter.new(64, prefix=iv, initial_value=0) + key, iv, cipher, sha256 = _prepare_encryption() - cipher = AES.new(key, AES.MODE_CTR, counter=ctr) - sha256 = SHA256.new() - - loop = asyncio.get_event_loop() + loop = asyncio.get_running_loop() async for chunk in async_generator_from_data(data): update_crypt = partial(cipher.encrypt, chunk) @@ -71,6 +64,11 @@ async def async_encrypt_attachment( yield _get_decryption_info(key, iv, sha256) +async def async_inplace_encrypt_attachment(data: bytearray) -> EncryptedFile: + loop = asyncio.get_running_loop() + return await loop.run_in_executor(None, partial(inplace_encrypt_attachment, data)) + + async def async_generator_from_data( data: bytes | Iterable[bytes] | AsyncIterable[bytes] | io.BufferedIOBase, chunk_size: int = 4 * 1024, diff --git a/mautrix/crypto/attachments/attachments.py b/mautrix/crypto/attachments/attachments.py index 6dfaa3f3..7d91c745 100644 --- a/mautrix/crypto/attachments/attachments.py +++ b/mautrix/crypto/attachments/attachments.py @@ -90,6 +90,28 @@ def encrypt_attachment(plaintext: bytes) -> tuple[bytes, EncryptedFile]: return b"".join(values[:-1]), values[-1] +def _prepare_encryption() -> tuple[bytes, bytes, AES, SHA256.SHA256Hash]: + key = Random.new().read(32) + # 8 bytes IV + iv = Random.new().read(8) + # 8 bytes counter, prefixed by the IV + ctr = Counter.new(64, prefix=iv, initial_value=0) + + cipher = AES.new(key, AES.MODE_CTR, counter=ctr) + sha256 = SHA256.new() + + return key, iv, cipher, sha256 + + +def inplace_encrypt_attachment(data: bytearray) -> EncryptedFile: + key, iv, cipher, sha256 = _prepare_encryption() + + cipher.encrypt(plaintext=data, output=data) + sha256.update(data) + + return _get_decryption_info(key, iv, sha256) + + def encrypted_attachment_generator( data: bytes | Iterable[bytes], ) -> Generator[bytes | EncryptedFile, None, None]: @@ -105,21 +127,10 @@ def encrypted_attachment_generator( Yields: The encrypted bytes for each chunk of data. - The last yielded value will be a dict containing the info needed to - decrypt data. The keys are: - | key: AES-CTR JWK key object. - | iv: Base64 encoded 16 byte AES-CTR IV. - | hashes.sha256: Base64 encoded SHA-256 hash of the ciphertext. + The last yielded value will be a dict containing the info needed to decrypt data. """ - key = Random.new().read(32) - # 8 bytes IV - iv = Random.new().read(8) - # 8 bytes counter, prefixed by the IV - ctr = Counter.new(64, prefix=iv, initial_value=0) - - cipher = AES.new(key, AES.MODE_CTR, counter=ctr) - sha256 = SHA256.new() + key, iv, cipher, sha256 = _prepare_encryption() if isinstance(data, bytes): data = [data] From 6fa8b4081d3e92b095cf8bf23b725ed70663538e Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 8 Mar 2022 12:30:59 +0200 Subject: [PATCH 025/456] Allow bytearray in magic.mimetype --- mautrix/util/magic.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mautrix/util/magic.py b/mautrix/util/magic.py index 4f66d02d..5c061993 100644 --- a/mautrix/util/magic.py +++ b/mautrix/util/magic.py @@ -17,7 +17,7 @@ _from_filename = lambda file: magic.detect_from_filename(file).mime_type -def mimetype(data: bytes | str) -> str: +def mimetype(data: bytes | bytearray | str) -> str: """ Uses magic to determine the mimetype of a file on disk or in memory. @@ -33,6 +33,9 @@ def mimetype(data: bytes | str) -> str: return _from_filename(data) elif isinstance(data, bytes): return _from_buffer(data) + elif isinstance(data, bytearray): + # Magic doesn't like bytearrays directly, so just copy the first 1024 bytes for it. + return _from_buffer(bytes(data[:1024])) else: raise TypeError( f"mimetype() argument must be a string or bytes, not {type(data).__name__!r}" From a13fae8998156a6a05369024bbb18b657a3f32af Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 8 Mar 2022 12:40:58 +0200 Subject: [PATCH 026/456] Don't log bytearrays in request content --- mautrix/api.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mautrix/api.py b/mautrix/api.py index 7a3b0260..7b318b20 100644 --- a/mautrix/api.py +++ b/mautrix/api.py @@ -266,7 +266,9 @@ def _log_request( ) -> None: if not self.log: return - log_content = content if not isinstance(content, bytes) else f"<{len(content)} bytes>" + log_content = ( + content if not isinstance(content, (bytes, bytearray)) else f"<{len(content)} bytes>" + ) as_user = query_params.get("user_id", None) level = 1 if path == Path.v3.sync or path == Path.r0.sync else 5 self.log.log( From 7d8138a5ab2a64caf79c98e9778c5db72c1e9a5a Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 8 Mar 2022 12:41:13 +0200 Subject: [PATCH 027/456] Bump version to 0.15.0rc6 --- mautrix/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 61ff1b12..de8724b6 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.15.0rc5" +__version__ = "0.15.0rc6" __author__ = "Tulir Asokan " __all__ = [ "api", From 8f2f2ad661dc42a00fa2ee68a5654bc4c93e6dfb Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 8 Mar 2022 13:56:33 +0200 Subject: [PATCH 028/456] Try to reduce aiohttp memory usage by creating async iterable --- CHANGELOG.md | 5 +++ mautrix/__init__.py | 2 +- mautrix/api.py | 43 +++++++++++++++---- .../client/api/modules/media_repository.py | 2 +- 4 files changed, 41 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 67fefa3e..a99aa7cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,11 +22,16 @@ abstract `User.get_portal_with` and `Portal.get_dm_puppet` methods. * Added support for creating DM portals with minimal bridge-specific code. * Added a redundant `__all__` to various `__init__.py` files to appease pyright. +* *(api)* Reduced aiohttp memory usage when uploading large files by making + an in-memory async iterable instead of passing the bytes directly. * *(bridge)* Removed legacy community utilities. * *(util.async_db)* Fixed counting number of db upgrades. * *(util.async_db)* Added support for schema migrations that jump versions. * *(util.db)* Module deprecated. The async_db module is recommended. However, the SQLAlchemy helpers will remain until maubot has switched to asyncpg. +* *(util.magic)* Allowed `bytearray` as an input type for the `mimetype` method. +* *(crypto.attachments)* Added method to encrypt a `bytearray` in-place to + avoid unnecessarily duplicating data in memory. ## v0.14.10 (2022-02-01) diff --git a/mautrix/__init__.py b/mautrix/__init__.py index de8724b6..41b8dc7b 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.15.0rc6" +__version__ = "0.15.0rc7" __author__ = "Tulir Asokan " __all__ = [ "api", diff --git a/mautrix/api.py b/mautrix/api.py index 7b318b20..097b44e4 100644 --- a/mautrix/api.py +++ b/mautrix/api.py @@ -5,12 +5,13 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import annotations -from typing import ClassVar, Literal, Mapping +from typing import AsyncGenerator, AsyncIterable, ClassVar, Literal, Mapping, Union from enum import Enum from json.decoder import JSONDecodeError from time import time from urllib.parse import quote as urllib_quote, urljoin as urllib_join import asyncio +import inspect import json import logging import platform @@ -154,6 +155,7 @@ def replace(self, find: str, replace: str) -> PathBuilder: """ _req_id = 0 +AsyncBody = AsyncGenerator[Union[bytes, bytearray, memoryview], None] def _next_global_req_id() -> int: @@ -162,6 +164,13 @@ def _next_global_req_id() -> int: return _req_id +async def _async_iter_bytes(data: bytearray | bytes, chunk_size: int = 1024**2) -> AsyncBody: + mv = memoryview(data) + for i in range(0, len(data), chunk_size): + yield mv[i : i + chunk_size] + mv.release() + + class HTTPAPI: """HTTPAPI is a simple asyncio Matrix API request sender.""" @@ -231,7 +240,7 @@ async def _send( self, method: Method, url: URL, - content: bytes | str, + content: bytes | bytearray | str | AsyncBody, query_params: dict[str, str], headers: dict[str, str], ) -> JSON: @@ -259,16 +268,21 @@ def _log_request( self, method: Method, path: PathBuilder, - content: str | bytes, + content: str | bytes | bytearray | AsyncBody, orig_content, query_params: dict[str, str], + headers: dict[str, str], req_id: int, ) -> None: if not self.log: return - log_content = ( - content if not isinstance(content, (bytes, bytearray)) else f"<{len(content)} bytes>" - ) + if isinstance(content, (bytes, bytearray)): + log_content = f"<{len(content)} bytes>" + elif inspect.isasyncgen(content): + size = headers.get("Content-Length", None) + log_content = f"<{size} async bytes>" if size else f"" + else: + log_content = content as_user = query_params.get("user_id", None) level = 1 if path == Path.v3.sync or path == Path.r0.sync else 5 self.log.log( @@ -300,11 +314,12 @@ async def request( self, method: Method, path: PathBuilder | str, - content: dict | list | bytes | str | None = None, + content: dict | list | bytes | bytearray | str | AsyncBody | None = None, headers: dict[str, str] | None = None, query_params: Mapping[str, str] | None = None, retry_count: int | None = None, metrics_method: str = "", + min_iter_size: int = 25 * 1024 * 1024, ) -> JSON: """ Make a raw Matrix API request. @@ -321,6 +336,9 @@ async def request( retry_count: Number of times to retry if the homeserver isn't reachable. Defaults to :attr:`default_retry_count`. metrics_method: Name of the method to include in Prometheus timing metrics. + min_iter_size: If the request body is larger than this value, it will be passed to + aiohttp as an async iterable to stop it from copying the whole thing + in memory. Returns: The parsed response JSON. @@ -351,12 +369,19 @@ async def request( if retry_count is None: retry_count = self.default_retry_count + if inspect.isasyncgen(content): + # Can't retry with non-static body + retry_count = 0 + do_fake_iter = content and hasattr(content, "__len__") and len(content) > min_iter_size + if do_fake_iter: + headers["Content-Length"] = str(len(content)) backoff = 4 while True: - self._log_request(method, path, content, orig_content, query_params, req_id) + self._log_request(method, path, content, orig_content, query_params, headers, req_id) API_CALLS.labels(method=metrics_method).inc() + req_content = _async_iter_bytes(content) if do_fake_iter else content try: - return await self._send(method, full_url, content, query_params, headers or {}) + return await self._send(method, full_url, req_content, query_params, headers or {}) except MatrixRequestError as e: API_CALLS_FAILED.labels(method=metrics_method).inc() if retry_count > 0 and e.http_status in (502, 503, 504): diff --git a/mautrix/client/api/modules/media_repository.py b/mautrix/client/api/modules/media_repository.py index 7e293152..3b9abb1e 100644 --- a/mautrix/client/api/modules/media_repository.py +++ b/mautrix/client/api/modules/media_repository.py @@ -32,7 +32,7 @@ class MediaRepositoryMethods(BaseClientAPI): async def upload_media( self, - data: bytes | AsyncIterable[bytes], + data: bytes | bytearray | AsyncIterable[bytes], mime_type: str | None = None, filename: str | None = None, size: int | None = None, From 3972728deed0406dc62fa83edcd022d1df99a7a1 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 8 Mar 2022 14:34:17 +0200 Subject: [PATCH 029/456] Use with statement for memoryview --- mautrix/api.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/mautrix/api.py b/mautrix/api.py index 097b44e4..7a7f881c 100644 --- a/mautrix/api.py +++ b/mautrix/api.py @@ -165,10 +165,9 @@ def _next_global_req_id() -> int: async def _async_iter_bytes(data: bytearray | bytes, chunk_size: int = 1024**2) -> AsyncBody: - mv = memoryview(data) - for i in range(0, len(data), chunk_size): - yield mv[i : i + chunk_size] - mv.release() + with memoryview(data) as mv: + for i in range(0, len(data), chunk_size): + yield mv[i : i + chunk_size] class HTTPAPI: From 6b2fd64d0f9f1aa16b7283477a6ac043ce792728 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 8 Mar 2022 19:58:07 +0200 Subject: [PATCH 030/456] Update changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a99aa7cd..745b4118 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,11 +20,11 @@ into `types.crypto`. * **Breaking change *(bridge)*** Made `User.get_puppet` abstract and added new abstract `User.get_portal_with` and `Portal.get_dm_puppet` methods. -* Added support for creating DM portals with minimal bridge-specific code. * Added a redundant `__all__` to various `__init__.py` files to appease pyright. * *(api)* Reduced aiohttp memory usage when uploading large files by making an in-memory async iterable instead of passing the bytes directly. * *(bridge)* Removed legacy community utilities. +* *(bridge)* Added support for creating DM portals with minimal bridge-specific code. * *(util.async_db)* Fixed counting number of db upgrades. * *(util.async_db)* Added support for schema migrations that jump versions. * *(util.db)* Module deprecated. The async_db module is recommended. However, From 7f895eb61a25e5bb1a19ae33312e1bc17f539b4e Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 9 Mar 2022 01:53:27 +0200 Subject: [PATCH 031/456] Update AppServiceAPI.request docstring --- mautrix/appservice/api/appservice.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/mautrix/appservice/api/appservice.py b/mautrix/appservice/api/appservice.py index 62253626..f59d5907 100644 --- a/mautrix/appservice/api/appservice.py +++ b/mautrix/appservice/api/appservice.py @@ -199,24 +199,31 @@ def request( query_params: dict[str, Any] | None = None, retry_count: int | None = None, metrics_method: str | None = "", + min_iter_size: int = 25 * 1024 * 1024, ) -> Awaitable[dict]: """ - Make a raw HTTP request, with optional AppService timestamp massaging and external_url - setting. + Make a raw Matrix API request, acting as the appservice user assigned to this AppServiceAPI + instance and optionally including timestamp massaging. Args: method: The HTTP method to use. - path: The API endpoint to call. - Does not include the base path (e.g. /_matrix/client/r0). - content: The content to post as a dict (json) or bytes/str (raw). + path: The full API endpoint to call (including the _matrix/... prefix) + content: The content to post as a dict/list (will be serialized as JSON) + or bytes/str (will be sent as-is). timestamp: The timestamp query param used for timestamp massaging. - headers: The dict of HTTP headers to send. - query_params: The dict of query parameters to send. + headers: A dict of HTTP headers to send. If the headers don't contain ``Content-Type``, + it'll be set to ``application/json``. The ``Authorization`` header is always + overridden if :attr:`token` is set. + query_params: A dict of query parameters to send. retry_count: Number of times to retry if the homeserver isn't reachable. + Defaults to :attr:`default_retry_count`. metrics_method: Name of the method to include in Prometheus timing metrics. + min_iter_size: If the request body is larger than this value, it will be passed to + aiohttp as an async iterable to stop it from copying the whole thing + in memory. Returns: - The response as a dict. + The parsed response JSON. """ query_params = query_params or {} if timestamp is not None: From 6ff238647ef6c7c20c8a0b5c0106d4cc07a63fba Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 16 Mar 2022 01:16:26 +0200 Subject: [PATCH 032/456] Check database owner and foreign tables before starting --- CHANGELOG.md | 7 ++++ mautrix/bridge/bridge.py | 48 +++++++++++++++++++++---- mautrix/bridge/e2ee.py | 13 +++---- mautrix/util/async_db/__init__.py | 10 ++++++ mautrix/util/async_db/aiosqlite.py | 11 +++++- mautrix/util/async_db/asyncpg.py | 11 +++++- mautrix/util/async_db/connection.py | 12 +++++++ mautrix/util/async_db/connection.pyi | 1 + mautrix/util/async_db/database.py | 53 ++++++++++++++++++++++++---- mautrix/util/async_db/errors.py | 26 ++++++++++++++ mautrix/util/async_db/upgrade.py | 14 +++----- 11 files changed, 175 insertions(+), 31 deletions(-) create mode 100644 mautrix/util/async_db/errors.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 745b4118..02b8b417 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,13 @@ * *(bridge)* Added support for creating DM portals with minimal bridge-specific code. * *(util.async_db)* Fixed counting number of db upgrades. * *(util.async_db)* Added support for schema migrations that jump versions. +* *(util.async_db)* Added system for preventing using the same database for + multiple programs. + * To enable it, provide an unique program name as the `owner_name` parameter + in `Database.create`. + * Additionally, if `ignore_foreign_tables` is set to `True`, it will check + for tables of some known software like Synapse and Dendrite. + * The `bridge` module enables both options by default. * *(util.db)* Module deprecated. The async_db module is recommended. However, the SQLAlchemy helpers will remain until maubot has switched to asyncpg. * *(util.magic)* Allowed `bytearray` as an input type for the `mimetype` method. diff --git a/mautrix/bridge/bridge.py b/mautrix/bridge/bridge.py index 90cb827b..74f916c4 100644 --- a/mautrix/bridge/bridge.py +++ b/mautrix/bridge/bridge.py @@ -17,7 +17,14 @@ from mautrix.client.state_store.asyncpg import PgStateStore as PgClientStateStore from mautrix.errors import MExclusive, MUnknownToken from mautrix.types import RoomID, UserID -from mautrix.util.async_db import Database, UpgradeTable +from mautrix.util.async_db import ( + Database, + DatabaseException, + DatabaseNotOwned, + ForeignTablesFound, + UnsupportedDatabaseVersion, + UpgradeTable, +) from mautrix.util.bridge_state import BridgeState, BridgeStateEvent, GlobalBridgeState from mautrix.util.program import Program @@ -81,6 +88,16 @@ def prepare_arg_parser(self) -> None: "(not needed for running the bridge)" ), ) + self.parser.add_argument( + "--ignore-unsupported-database", + action="store_true", + help="Run even if the database schema is too new", + ) + self.parser.add_argument( + "--ignore-foreign-tables", + action="store_true", + help="Run even if the database contains tables from other programs (like Synapse)", + ) def preinit(self) -> None: super().preinit() @@ -152,19 +169,38 @@ def prepare_db(self) -> None: self.config["appservice.database"], upgrade_table=self.upgrade_table, db_args=self.config["appservice.database_opts"], + owner_name=self.name, + ignore_foreign_tables=self.args.ignore_foreign_tables, ) def prepare_bridge(self) -> None: self.matrix = self.matrix_class(bridge=self) + def _log_db_error(self, e: DatabaseException) -> None: + self.log.critical("Failed to initialize database", exc_info=e) + if isinstance(e, DatabaseNotOwned): + self.log.info("Sharing the same database with different programs is not supported") + elif isinstance(e, ForeignTablesFound): + self.log.info("You can use --ignore-foreign-tables to ignore this error") + elif isinstance(e, UnsupportedDatabaseVersion): + self.log.info("Downgrading the bridge is not supported") + sys.exit(25) + async def start_db(self) -> None: if hasattr(self, "db") and isinstance(self.db, Database): self.log.debug("Starting database...") - await self.db.start() - if isinstance(self.state_store, PgClientStateStore): - await self.state_store.upgrade_table.upgrade(self.db) - if self.matrix.e2ee: - self.matrix.e2ee.crypto_db.override_pool(self.db) + ignore_unsupported = self.args.ignore_unsupported_database + self.db.upgrade_table.allow_unsupported = ignore_unsupported + try: + await self.db.start() + if isinstance(self.state_store, PgClientStateStore): + self.state_store.upgrade_table.allow_unsupported = ignore_unsupported + await self.state_store.upgrade_table.upgrade(self.db) + if self.matrix.e2ee: + self.matrix.e2ee.crypto_db.allow_unsupported = ignore_unsupported + self.matrix.e2ee.crypto_db.override_pool(self.db) + except DatabaseException as e: + self._log_db_error(e) async def stop_db(self) -> None: if hasattr(self, "db") and isinstance(self.db, Database): diff --git a/mautrix/bridge/e2ee.py b/mautrix/bridge/e2ee.py index 958d46a5..ea802c5c 100644 --- a/mautrix/bridge/e2ee.py +++ b/mautrix/bridge/e2ee.py @@ -34,6 +34,7 @@ StateFilter, TrustState, ) +from mautrix.util.async_db import Database, DatabaseException from mautrix.util.logging import TraceLogger from .. import bridge as br @@ -46,13 +47,6 @@ raise UserProfile = None -try: - from mautrix.util.async_db import Database -except ImportError: - if __optional_imports__: - raise - Database = None - class EncryptionManager: loop: asyncio.AbstractEventLoop @@ -231,7 +225,10 @@ async def start(self) -> None: sys.exit(30) self.log.debug(f"Logging in with bridge bot user (using login type {flow.type.value})") if self.crypto_db: - await self.crypto_db.start() + try: + await self.crypto_db.start() + except DatabaseException as e: + self.bridge._log_db_error(e) await self.crypto_store.open() device_id = await self.crypto_store.get_device_id() if device_id: diff --git a/mautrix/util/async_db/__init__.py b/mautrix/util/async_db/__init__.py index 0fa484bb..913725a2 100644 --- a/mautrix/util/async_db/__init__.py +++ b/mautrix/util/async_db/__init__.py @@ -2,6 +2,12 @@ from .connection import LoggingConnection as Connection from .database import Database +from .errors import ( + DatabaseException, + DatabaseNotOwned, + ForeignTablesFound, + UnsupportedDatabaseVersion, +) from .scheme import Scheme from .upgrade import UpgradeTable, register_upgrade @@ -27,4 +33,8 @@ "SQLiteDatabase", "Connection", "Scheme", + "DatabaseException", + "DatabaseNotOwned", + "UnsupportedDatabaseVersion", + "ForeignTablesFound", ] diff --git a/mautrix/util/async_db/aiosqlite.py b/mautrix/util/async_db/aiosqlite.py index 71320cee..99085dcb 100644 --- a/mautrix/util/async_db/aiosqlite.py +++ b/mautrix/util/async_db/aiosqlite.py @@ -85,8 +85,17 @@ def __init__( upgrade_table: UpgradeTable, db_args: dict[str, Any] | None = None, log: logging.Logger | None = None, + owner_name: str | None = None, + ignore_foreign_tables: bool = True, ) -> None: - super().__init__(url, db_args=db_args, upgrade_table=upgrade_table, log=log) + super().__init__( + url, + db_args=db_args, + upgrade_table=upgrade_table, + log=log, + owner_name=owner_name, + ignore_foreign_tables=ignore_foreign_tables, + ) self._path = url.path if self._path.startswith("/"): self._path = self._path[1:] diff --git a/mautrix/util/async_db/asyncpg.py b/mautrix/util/async_db/asyncpg.py index aad9ef45..dbec68b4 100644 --- a/mautrix/util/async_db/asyncpg.py +++ b/mautrix/util/async_db/asyncpg.py @@ -30,12 +30,21 @@ def __init__( upgrade_table: UpgradeTable, db_args: dict[str, Any] = None, log: logging.Logger | None = None, + owner_name: str | None = None, + ignore_foreign_tables: bool = True, ) -> None: if url.scheme in ("cockroach", "cockroachdb"): self.scheme = Scheme.COCKROACH # Send postgres scheme to asyncpg url = url.with_scheme("postgres") - super().__init__(url, db_args=db_args, upgrade_table=upgrade_table, log=log) + super().__init__( + url, + db_args=db_args, + upgrade_table=upgrade_table, + log=log, + owner_name=owner_name, + ignore_foreign_tables=ignore_foreign_tables, + ) self._pool = None self._pool_override = False diff --git a/mautrix/util/async_db/connection.py b/mautrix/util/async_db/connection.py index 6d726e5d..32a96fbd 100644 --- a/mautrix/util/async_db/connection.py +++ b/mautrix/util/async_db/connection.py @@ -92,6 +92,18 @@ async def fetchval( async def fetchrow(self, query: str, *args: Any, timeout: float | None = None) -> Row | Record: return await self.wrapped.fetchrow(query, *args, timeout=timeout) + async def table_exists(self, name: str) -> bool: + if self.scheme == Scheme.SQLITE: + return await self.fetchval( + "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name=?1)", name + ) + elif self.scheme in (Scheme.POSTGRES, Scheme.COCKROACH): + return await self.fetchval( + "SELECT EXISTS(SELECT 1 FROM information_schema.tables WHERE table_name=$1)", name + ) + else: + raise RuntimeError(f"Unknown scheme {self.scheme}") + @log_duration async def copy_records_to_table( self, diff --git a/mautrix/util/async_db/connection.pyi b/mautrix/util/async_db/connection.pyi index 2d967cef..2a305d74 100644 --- a/mautrix/util/async_db/connection.pyi +++ b/mautrix/util/async_db/connection.pyi @@ -36,6 +36,7 @@ class LoggingConnection: async def fetchrow( self, query: str, *args: Any, timeout: float | None = None ) -> Row | Record: ... + async def table_exists(self, name: str) -> bool: ... async def copy_records_to_table( self, table_name: str, diff --git a/mautrix/util/async_db/database.py b/mautrix/util/async_db/database.py index 420f093d..2c4981a7 100644 --- a/mautrix/util/async_db/database.py +++ b/mautrix/util/async_db/database.py @@ -16,6 +16,7 @@ from mautrix.util.logging import TraceLogger from .connection import LoggingConnection +from .errors import DatabaseNotOwned, ForeignTablesFound from .scheme import Scheme from .upgrade import UpgradeTable, upgrade_tables @@ -31,6 +32,8 @@ class Database(ABC): url: URL _db_args: dict[str, Any] upgrade_table: UpgradeTable + owner_name: str | None + ignore_foreign_tables: bool def __init__( self, @@ -38,10 +41,14 @@ def __init__( upgrade_table: UpgradeTable, db_args: dict[str, Any] | None = None, log: TraceLogger | None = None, + owner_name: str | None = None, + ignore_foreign_tables: bool = True, ) -> None: self.url = url self._db_args = {**db_args} if db_args else {} self.upgrade_table = upgrade_table + self.owner_name = owner_name + self.ignore_foreign_tables = ignore_foreign_tables self.log = log or logging.getLogger("mau.db") assert isinstance(self.log, TraceLogger) @@ -53,6 +60,8 @@ def create( db_args: dict[str, Any] | None = None, upgrade_table: UpgradeTable | str | None = None, log: logging.Logger | TraceLogger | None = None, + owner_name: str | None = None, + ignore_foreign_tables: bool = True, ) -> Database: url = URL(url) try: @@ -75,17 +84,45 @@ def create( upgrade_table = UpgradeTable() elif not isinstance(upgrade_table, UpgradeTable): raise ValueError(f"Can't use {type(upgrade_table)} as the upgrade table") - return impl(url, db_args=db_args, upgrade_table=upgrade_table, log=log) + return impl( + url, + db_args=db_args, + upgrade_table=upgrade_table, + log=log, + owner_name=owner_name, + ignore_foreign_tables=ignore_foreign_tables, + ) def override_pool(self, db: Database) -> None: pass async def start(self) -> None: - try: - await self.upgrade_table.upgrade(self) - except Exception: - self.log.critical("Failed to upgrade database", exc_info=True) - sys.exit(25) + if not self.ignore_foreign_tables: + await self._check_foreign_tables() + if self.owner_name: + await self._check_owner() + await self.upgrade_table.upgrade(self) + + async def _check_foreign_tables(self) -> None: + if await self.table_exists("state_groups_state"): + raise ForeignTablesFound("found state_groups_state likely belonging to Synapse") + elif await self.table_exists("goose_db_version"): + raise ForeignTablesFound("found goose_db_version possibly belonging to Dendrite") + + async def _check_owner(self) -> None: + await self.execute( + """ + CREATE TABLE IF NOT EXISTS database_owner ( + key INTEGER PRIMARY KEY DEFAULT 0, + owner TEXT NOT NULL + ) + """ + ) + owner = await self.fetchval("SELECT owner FROM database_owner WHERE key=0") + if not owner: + await self.execute("INSERT INTO database_owner (owner) VALUES ($1)", self.owner_name) + elif owner != self.owner_name: + raise DatabaseNotOwned(owner) @abstractmethod async def stop(self) -> None: @@ -116,3 +153,7 @@ async def fetchval( async def fetchrow(self, query: str, *args: Any, timeout: float | None = None) -> Record: async with self.acquire() as conn: return await conn.fetchrow(query, *args, timeout=timeout) + + async def table_exists(self, name: str) -> bool: + async with self.acquire() as conn: + return await conn.table_exists(name) diff --git a/mautrix/util/async_db/errors.py b/mautrix/util/async_db/errors.py new file mode 100644 index 00000000..e15064a6 --- /dev/null +++ b/mautrix/util/async_db/errors.py @@ -0,0 +1,26 @@ +# Copyright (c) 2022 Tulir Asokan +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +class DatabaseException(RuntimeError): + pass + + +class UnsupportedDatabaseVersion(DatabaseException): + def __init__(self, name: str, version: int, latest: int) -> None: + super().__init__( + f"Unsupported {name} schema version v{version} (latest known is v{latest})" + ) + + +class ForeignTablesFound(DatabaseException): + def __init__(self, explanation: str) -> None: + super().__init__(f"The database contains foreign tables ({explanation})") + + +class DatabaseNotOwned(DatabaseException): + def __init__(self, owner: str) -> None: + super().__init__(f"The database is owned by {owner}") diff --git a/mautrix/util/async_db/upgrade.py b/mautrix/util/async_db/upgrade.py index 6ee2807a..5d2eb42a 100644 --- a/mautrix/util/async_db/upgrade.py +++ b/mautrix/util/async_db/upgrade.py @@ -14,16 +14,13 @@ from .. import async_db from .connection import LoggingConnection +from .errors import UnsupportedDatabaseVersion from .scheme import Scheme Upgrade = Callable[[LoggingConnection, Scheme], Awaitable[Optional[int]]] UpgradeWithoutScheme = Callable[[LoggingConnection], Awaitable[Optional[int]]] -class UnsupportedDatabaseVersion(Exception): - pass - - async def noop_upgrade(_: LoggingConnection) -> None: pass @@ -108,14 +105,13 @@ async def upgrade(self, db: async_db.Database) -> None: version = row["version"] if row else 0 if len(self.upgrades) < version: - error = ( - f"Unsupported database version v{version} " - f"(latest known is v{len(self.upgrades) - 1})" + unsupported_version_error = UnsupportedDatabaseVersion( + self.database_name, version, len(self.upgrades) ) if not self.allow_unsupported: - raise UnsupportedDatabaseVersion(error) + raise unsupported_version_error else: - self.log.warning(error) + self.log.warning(str(unsupported_version_error)) return elif len(self.upgrades) == version: self.log.debug(f"Database at v{version}, not upgrading") From bb3f1975dc4153706b6f62b299d517a2a19a2e2a Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 16 Mar 2022 01:16:33 +0200 Subject: [PATCH 033/456] Adjust bridge command permission error message --- mautrix/bridge/commands/handler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mautrix/bridge/commands/handler.py b/mautrix/bridge/commands/handler.py index efe556ee..d636e673 100644 --- a/mautrix/bridge/commands/handler.py +++ b/mautrix/bridge/commands/handler.py @@ -287,9 +287,9 @@ async def get_permission_error(self, evt: CommandEvent) -> str | None: "you may only run it in management rooms." ) elif self.needs_admin and not evt.sender.is_admin: - return "This command requires administrator privileges." + return "That command is limited to bridge administrators." elif self.needs_auth and not await evt.sender.is_logged_in(): - return "This command requires you to be logged in." + return "That command requires you to be logged in." return None def has_permission(self, key: HelpCacheKey) -> bool: From e098d309abec2643fad146ba2111fda8a308b587 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 16 Mar 2022 01:18:20 +0200 Subject: [PATCH 034/456] Fix string indentation --- mautrix/util/async_db/database.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/mautrix/util/async_db/database.py b/mautrix/util/async_db/database.py index 2c4981a7..8f87aabf 100644 --- a/mautrix/util/async_db/database.py +++ b/mautrix/util/async_db/database.py @@ -111,12 +111,10 @@ async def _check_foreign_tables(self) -> None: async def _check_owner(self) -> None: await self.execute( - """ - CREATE TABLE IF NOT EXISTS database_owner ( + """CREATE TABLE IF NOT EXISTS database_owner ( key INTEGER PRIMARY KEY DEFAULT 0, owner TEXT NOT NULL - ) - """ + )""" ) owner = await self.fetchval("SELECT owner FROM database_owner WHERE key=0") if not owner: From 7dfc0beb3ec1a17be374bf7c55ca8f4874526c52 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 16 Mar 2022 17:21:41 +0200 Subject: [PATCH 035/456] Bump version to 0.15.0 --- CHANGELOG.md | 2 +- mautrix/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 02b8b417..530c0b3e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -## v0.15.0 (unreleased) +## v0.15.0 (2022-03-16) * **Breaking change** Removed Python 3.7 support. * **Breaking change *(api)*** Removed `r0` from default path builders in order diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 41b8dc7b..401a2d72 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.15.0rc7" +__version__ = "0.15.0" __author__ = "Tulir Asokan " __all__ = [ "api", From f7ff483291579679a2128c9508b785494a45f73a Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 22 Mar 2022 14:17:37 +0200 Subject: [PATCH 036/456] Include error type in sync error log --- mautrix/client/syncer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mautrix/client/syncer.py b/mautrix/client/syncer.py index 4a052691..3ade8492 100644 --- a/mautrix/client/syncer.py +++ b/mautrix/client/syncer.py @@ -403,7 +403,8 @@ async def _start(self, filter_id: FilterID | None) -> None: raise except Exception as e: self.log.warning( - f"Sync request errored: {e}, waiting {fail_sleep} seconds before continuing" + f"Sync request errored: {type(e).__name__}: {e}, waiting {fail_sleep}" + " seconds before continuing" ) await self.run_internal_event( InternalEventType.SYNC_ERRORED, error=e, sleep_for=fail_sleep From 01a82f6dcd4ec1ebed3b38091caa84f6b0b4b282 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 23 Mar 2022 19:56:39 +0200 Subject: [PATCH 037/456] Add method for generating formatted_body from body correctly --- CHANGELOG.md | 6 ++++++ mautrix/__init__.py | 2 +- mautrix/types/event/message.py | 9 ++++++--- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 530c0b3e..edf70259 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## v0.15.1 (2022-03-23) + +* Added `ensure_has_html` method for `TextMessageEventContent` to generate + a HTML `formatted_body` from the plaintext `body` correctly (i.e. escaping + HTML and replacing newlines). + ## v0.15.0 (2022-03-16) * **Breaking change** Removed Python 3.7 support. diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 401a2d72..2446a694 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.15.0" +__version__ = "0.15.1" __author__ = "Tulir Asokan " __all__ = [ "api", diff --git a/mautrix/types/event/message.py b/mautrix/types/event/message.py index 2fcbcdf9..6285b2e0 100644 --- a/mautrix/types/event/message.py +++ b/mautrix/types/event/message.py @@ -342,15 +342,18 @@ def set_reply( super().set_reply(reply_to) if isinstance(reply_to, str): return - if not self.formatted_body or len(self.formatted_body) == 0 or self.format != Format.HTML: - self.format = Format.HTML - self.formatted_body = escape(self.body).replace("\n", "
") if isinstance(reply_to, MessageEvent): + self.ensure_has_html() self.formatted_body = ( reply_to.make_reply_fallback_html(displayname) + self.formatted_body ) self.body = reply_to.make_reply_fallback_text(displayname) + self.body + def ensure_has_html(self) -> None: + if not self.formatted_body or self.format != Format.HTML: + self.format = Format.HTML + self.formatted_body = escape(self.body).replace("\n", "
") + def formatted(self, format: Format) -> Optional[str]: if self.format == format: return self.formatted_body From 22dddf859deaf820e184629e398b11bc9def0011 Mon Sep 17 00:00:00 2001 From: Sumner Evans Date: Fri, 25 Mar 2022 02:33:02 -0600 Subject: [PATCH 038/456] asynchronous uploads: implement methods for async upload --- mautrix/api.py | 23 ++++- mautrix/appservice/api/intent.py | 1 + .../client/api/modules/media_repository.py | 85 +++++++++++++++++-- 3 files changed, 101 insertions(+), 8 deletions(-) diff --git a/mautrix/api.py b/mautrix/api.py index 7a7f881c..c7864f4c 100644 --- a/mautrix/api.py +++ b/mautrix/api.py @@ -431,8 +431,27 @@ def get_download_url( >>> api.get_download_url("mxc://matrix.org/pqjkOuKZ1ZKRULWXgz2IVZV6") "https://matrix.org/_matrix/media/v3/download/matrix.org/pqjkOuKZ1ZKRULWXgz2IVZV6" """ + server_name, media_id = self.parse_mxc_uri(mxc_uri) + version = "r0" if self.hacky_replace_v3_with_r0 else "v3" + return ( + self.base_url / str(APIPath.MEDIA) / version / download_type / server_name / media_id + ) + + def parse_mxc_uri(self, mxc_uri: str) -> tuple[str, str]: + """ + Parse a ``mxc://`` URI. + + Args: + mxc_uri: The MXC URI to parse. + + Returns: + A tuple containing the server and media ID of the MXC URI. + + Raises: + ValueError: If `mxc_uri` doesn't begin with ``mxc://``. + """ if mxc_uri.startswith("mxc://"): - version = "r0" if self.hacky_replace_v3_with_r0 else "v3" - return self.base_url / str(APIPath.MEDIA) / version / download_type / mxc_uri[6:] + server_name, media_id = mxc_uri[6:].split("/") + return server_name, media_id else: raise ValueError("MXC URI did not begin with `mxc://`") diff --git a/mautrix/appservice/api/intent.py b/mautrix/appservice/api/intent.py index 62cc2509..3e194b9d 100644 --- a/mautrix/appservice/api/intent.py +++ b/mautrix/appservice/api/intent.py @@ -69,6 +69,7 @@ def quote(*args, **kwargs): ClientAPI.search_users, ClientAPI.set_displayname, ClientAPI.set_avatar_url, + ClientAPI.create_media_id, ClientAPI.upload_media, ClientAPI.send_receipt, ClientAPI.set_fully_read_marker, diff --git a/mautrix/client/api/modules/media_repository.py b/mautrix/client/api/modules/media_repository.py index 3b9abb1e..763336fd 100644 --- a/mautrix/client/api/modules/media_repository.py +++ b/mautrix/client/api/modules/media_repository.py @@ -5,7 +5,8 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import annotations -from typing import AsyncIterable, Literal +from typing import Any, AsyncIterable, Literal +import asyncio from mautrix import __optional_imports__ from mautrix.api import MediaPath, Method @@ -28,7 +29,57 @@ class MediaRepositoryMethods(BaseClientAPI): downloading content from the media repository and for getting URL previews without leaking client IPs. - See also: `API reference `__""" + See also: `API reference `__ + + There are also methods for supporting `MSC2246 + `__ which allows asynchronous + uploads of media. + """ + + async def create_mxc(self) -> ContentURI: + """ + Create a media ID for uploading media to the homeserver. Requires the homeserver to have + `MSC2246 `__ support. + + Returns: + The MXC URI that can be used to upload a file to later. + + Raises: + MatrixResponseError: If the response does not contain a ``content_uri`` field. + """ + resp = await self.api.request(Method.PUT, MediaPath.unstable["fi.mau.msc2246"].create) + try: + return resp["content_uri"] + except KeyError: + raise MatrixResponseError("`content_uri` not in response.") + + async def upload_async( + self, + data: bytes | bytearray | AsyncIterable[bytes], + mime_type: str | None = None, + filename: str | None = None, + size: int | None = None, + ) -> ContentURI: + """ + Create a blank content URI with create_mxc, then start uploading the data in the background + and returns the created MXC immediately. Requires the homeserver to have `MSC2246 + `__ support. + + Args: + data: The data to upload. + mime_type: The MIME type to send with the upload request. + filename: The filename to send with the upload request. + size: The file size to send with the upload request. + + Returns: + The MXC URI of the file being uploaded asynchronously. + + Raises: + MatrixResponseError: If the response does not contain a ``content_uri`` field. + """ + content_uri = await self.create_mxc() + asyncio.create_task(self.upload_media(data, mime_type, filename, size, content_uri)) + return content_uri async def upload_media( self, @@ -36,6 +87,7 @@ async def upload_media( mime_type: str | None = None, filename: str | None = None, size: int | None = None, + mxc: ContentURI | None = None, ) -> ContentURI: """ Upload a file to the content repository. @@ -47,6 +99,9 @@ async def upload_media( mime_type: The MIME type to send with the upload request. filename: The filename to send with the upload request. size: The file size to send with the upload request. + mxc: An existing MXC URI which doesn't have content yet to upload into. Requires the + homesrver to have `MSC2246 + `__ support. Returns: The MXC URI to the uploaded file. @@ -64,15 +119,21 @@ async def upload_media( query = {} if filename: query["filename"] = filename + + path = MediaPath.v3.upload + if mxc: + server_name, media_id = self.api.parse_mxc_uri(mxc) + path = MediaPath.unstable["fi.mau.msc2246"].upload[server_name][media_id] + resp = await self.api.request( - Method.POST, MediaPath.v3.upload, content=data, headers=headers, query_params=query + Method.POST, path, content=data, headers=headers, query_params=query ) try: return resp["content_uri"] except KeyError: raise MatrixResponseError("`content_uri` not in response.") - async def download_media(self, url: ContentURI) -> bytes: + async def download_media(self, url: ContentURI, max_stall_ms: int | None = None) -> bytes: """ Download a file from the content repository. @@ -80,12 +141,18 @@ async def download_media(self, url: ContentURI) -> bytes: Args: url: The MXC URI to download. + max_stall_ms: The maximum number of milliseconds that the client is willing to wait to + start receiving data. Used for MSC2246 Asynchronous Uploads. Returns: The raw downloaded data. """ url = self.api.get_download_url(url) - async with self.api.session.get(url) as response: + query_params: dict[str, Any] = {} + if max_stall_ms is not None: + query_params["max_stall_ms"] = max_stall_ms + query_params["fi.mau.msc2246.max_stall_ms"] = max_stall_ms + async with self.api.session.get(url, params=query_params) as response: return await response.read() async def download_thumbnail( @@ -95,6 +162,7 @@ async def download_thumbnail( height: int | None = None, resize_method: Literal["crop", "scale"] = None, allow_remote: bool = True, + max_stall_ms: int | None = None, ): """ Download a thumbnail for a file in the content repository. @@ -111,12 +179,14 @@ async def download_thumbnail( allow_remote: Indicates to the server that it should not attempt to fetch the media if it is deemed remote. This is to prevent routing loops where the server contacts itself. + max_stall_ms: The maximum number of milliseconds that the client is willing to wait to + start receiving data. Used for MSC2246 Asynchronous Uploads. Returns: The raw downloaded data. """ url = self.api.get_download_url(url, download_type="thumbnail") - query_params = {} + query_params: dict[str, Any] = {} if width is not None: query_params["width"] = width if height is not None: @@ -125,6 +195,9 @@ async def download_thumbnail( query_params["method"] = resize_method if allow_remote is not None: query_params["allow_remote"] = allow_remote + if max_stall_ms is not None: + query_params["max_stall_ms"] = max_stall_ms + query_params["fi.mau.msc2246.max_stall_ms"] = max_stall_ms async with self.api.session.get(url, params=query_params) as response: return await response.read() From ce6baf2983fd569619a8ac9833043c0a28c208b1 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 25 Mar 2022 13:20:27 +0200 Subject: [PATCH 039/456] Merge upload_async with upload_media and rename create_mxc --- mautrix/api.py | 3 +- mautrix/appservice/api/intent.py | 2 +- .../client/api/modules/media_repository.py | 70 +++++++++---------- 3 files changed, 36 insertions(+), 39 deletions(-) diff --git a/mautrix/api.py b/mautrix/api.py index c7864f4c..c6db9cab 100644 --- a/mautrix/api.py +++ b/mautrix/api.py @@ -437,7 +437,8 @@ def get_download_url( self.base_url / str(APIPath.MEDIA) / version / download_type / server_name / media_id ) - def parse_mxc_uri(self, mxc_uri: str) -> tuple[str, str]: + @staticmethod + def parse_mxc_uri(mxc_uri: str) -> tuple[str, str]: """ Parse a ``mxc://`` URI. diff --git a/mautrix/appservice/api/intent.py b/mautrix/appservice/api/intent.py index 3e194b9d..f9c991b7 100644 --- a/mautrix/appservice/api/intent.py +++ b/mautrix/appservice/api/intent.py @@ -69,7 +69,7 @@ def quote(*args, **kwargs): ClientAPI.search_users, ClientAPI.set_displayname, ClientAPI.set_avatar_url, - ClientAPI.create_media_id, + ClientAPI.unstable_create_mxc, ClientAPI.upload_media, ClientAPI.send_receipt, ClientAPI.set_fully_read_marker, diff --git a/mautrix/client/api/modules/media_repository.py b/mautrix/client/api/modules/media_repository.py index 763336fd..9febffdd 100644 --- a/mautrix/client/api/modules/media_repository.py +++ b/mautrix/client/api/modules/media_repository.py @@ -36,7 +36,7 @@ class MediaRepositoryMethods(BaseClientAPI): uploads of media. """ - async def create_mxc(self) -> ContentURI: + async def unstable_create_mxc(self) -> ContentURI: """ Create a media ID for uploading media to the homeserver. Requires the homeserver to have `MSC2246 `__ support. @@ -53,34 +53,6 @@ async def create_mxc(self) -> ContentURI: except KeyError: raise MatrixResponseError("`content_uri` not in response.") - async def upload_async( - self, - data: bytes | bytearray | AsyncIterable[bytes], - mime_type: str | None = None, - filename: str | None = None, - size: int | None = None, - ) -> ContentURI: - """ - Create a blank content URI with create_mxc, then start uploading the data in the background - and returns the created MXC immediately. Requires the homeserver to have `MSC2246 - `__ support. - - Args: - data: The data to upload. - mime_type: The MIME type to send with the upload request. - filename: The filename to send with the upload request. - size: The file size to send with the upload request. - - Returns: - The MXC URI of the file being uploaded asynchronously. - - Raises: - MatrixResponseError: If the response does not contain a ``content_uri`` field. - """ - content_uri = await self.create_mxc() - asyncio.create_task(self.upload_media(data, mime_type, filename, size, content_uri)) - return content_uri - async def upload_media( self, data: bytes | bytearray | AsyncIterable[bytes], @@ -88,11 +60,12 @@ async def upload_media( filename: str | None = None, size: int | None = None, mxc: ContentURI | None = None, + async_upload: bool = False, ) -> ContentURI: """ Upload a file to the content repository. - See also: `API reference `__ + See also: `API reference `__ Args: data: The data to upload. @@ -100,14 +73,20 @@ async def upload_media( filename: The filename to send with the upload request. size: The file size to send with the upload request. mxc: An existing MXC URI which doesn't have content yet to upload into. Requires the - homesrver to have `MSC2246 - `__ support. + homeserver to have MSC2246_ support. + async_upload: Should the media be uploaded in the background (using MSC2246_)? + If ``True``, this will create a MXC URI, start uploading in the background and then + immediately return the created URI. This is mutually exclusive with manually + passing the ``mxc`` parameter. + + .. _MSC2246: https://github.com/matrix-org/matrix-spec-proposals/pull/2246 Returns: The MXC URI to the uploaded file. Raises: MatrixResponseError: If the response does not contain a ``content_uri`` field. + ValueError: if both ``async_upload`` and ``mxc`` are provided at the same time. """ if magic and isinstance(data, bytes): mime_type = mime_type or magic.mimetype(data) @@ -120,18 +99,35 @@ async def upload_media( if filename: query["filename"] = filename + if async_upload: + if mxc: + raise ValueError("async_upload and mxc can't be provided simultaneously") + mxc = await self.unstable_create_mxc() + path = MediaPath.v3.upload if mxc: server_name, media_id = self.api.parse_mxc_uri(mxc) path = MediaPath.unstable["fi.mau.msc2246"].upload[server_name][media_id] - resp = await self.api.request( + task = self.api.request( Method.POST, path, content=data, headers=headers, query_params=query ) - try: - return resp["content_uri"] - except KeyError: - raise MatrixResponseError("`content_uri` not in response.") + if async_upload: + + async def _try_upload(): + try: + await task + except Exception as e: + self.log.error(f"Failed to upload {mxc}: {type(e).__name__}: {e}") + + asyncio.create_task(_try_upload()) + return mxc + else: + resp = await task + try: + return resp["content_uri"] + except KeyError: + raise MatrixResponseError("`content_uri` not in response.") async def download_media(self, url: ContentURI, max_stall_ms: int | None = None) -> bytes: """ From 607e8e75db3c6c0c02a32388c1f947ff6edb6856 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 23 Mar 2022 19:59:34 +0200 Subject: [PATCH 040/456] Update actions to checkout@v3 and setup-python@v3 --- .github/workflows/python-package.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index c1ee7705..5f8da295 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -12,9 +12,9 @@ jobs: python-version: ["3.8", "3.9", "3.10"] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} - name: Install dependencies @@ -44,8 +44,8 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - uses: actions/checkout@v3 + - uses: actions/setup-python@v3 with: python-version: "3.10" - uses: isort/isort-action@master From 2cee3c7d28cff4ab34b1a7595221b356732beadb Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 25 Mar 2022 17:29:33 +0200 Subject: [PATCH 041/456] Move async_getter_lock to mautrix.util --- mautrix/bridge/__init__.py | 2 +- mautrix/bridge/async_getter_lock.py | 22 ---------- mautrix/util/async_getter_lock.py | 62 +++++++++++++++++++++++++++++ 3 files changed, 63 insertions(+), 23 deletions(-) delete mode 100644 mautrix/bridge/async_getter_lock.py create mode 100644 mautrix/util/async_getter_lock.py diff --git a/mautrix/bridge/__init__.py b/mautrix/bridge/__init__.py index 3ee081f4..4e31784a 100644 --- a/mautrix/bridge/__init__.py +++ b/mautrix/bridge/__init__.py @@ -1,4 +1,4 @@ -from .async_getter_lock import async_getter_lock +from ..util.async_getter_lock import async_getter_lock from .bridge import Bridge from .config import BaseBridgeConfig from .custom_puppet import ( diff --git a/mautrix/bridge/async_getter_lock.py b/mautrix/bridge/async_getter_lock.py deleted file mode 100644 index 4a246c8f..00000000 --- a/mautrix/bridge/async_getter_lock.py +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright (c) 2022 Tulir Asokan -# -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. -from typing import TYPE_CHECKING, Any -import functools - -if TYPE_CHECKING: - from typing import Awaitable, Callable, ParamSpec - - Param = ParamSpec("Param") - Func = Callable[Param, Awaitable[Any]] - - -def async_getter_lock(fn: "Func") -> "Func": - @functools.wraps(fn) - async def wrapper(cls, *args, **kwargs) -> Any: - async with cls._async_get_locks[args]: - return await fn(cls, *args, **kwargs) - - return wrapper diff --git a/mautrix/util/async_getter_lock.py b/mautrix/util/async_getter_lock.py new file mode 100644 index 00000000..31766976 --- /dev/null +++ b/mautrix/util/async_getter_lock.py @@ -0,0 +1,62 @@ +# Copyright (c) 2022 Tulir Asokan +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +from __future__ import annotations + +from typing import Any +import functools + +from mautrix import __optional_imports__ + +if __optional_imports__: + from typing import Awaitable, Callable, ParamSpec + + Param = ParamSpec("Param") + Func = Callable[Param, Awaitable[Any]] + + +def async_getter_lock(fn: Func) -> Func: + """ + A utility decorator for locking async getters that have caches + (preventing race conditions between cache check and e.g. async database actions). + + The class must have an ```_async_get_locks`` defaultdict that contains :class:`asyncio.Lock`s + (see example for exact definition). Non-cache-affecting arguments should be only passed as + keyword args. + + Args: + fn: The function to decorate. + + Returns: + The decorated function. + + Examples: + >>> import asyncio + >>> from collections import defaultdict + >>> class User: + ... _async_get_locks: dict[Any, asyncio.Lock] = defaultdict(lambda: asyncio.Lock()) + ... db: Any + ... cache: dict[str, User] + ... @classmethod + ... @async_getter_lock + ... async def get(cls, id: str, *, create: bool = False) -> User | None: + ... try: + ... return cls.cache[id] + ... except KeyError: + ... pass + ... user = await cls.db.fetch_user(id) + ... if user: + ... return user + ... elif create: + ... return await cls.db.create_user(id) + ... return None + """ + + @functools.wraps(fn) + async def wrapper(cls, *args, **kwargs) -> Any: + async with cls._async_get_locks[args]: + return await fn(cls, *args, **kwargs) + + return wrapper From 14586c31c9d751d0df9a01230a1cee5a5e75373b Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 25 Mar 2022 17:30:02 +0200 Subject: [PATCH 042/456] Make DatabaseException explanations easier to access for other things --- mautrix/bridge/bridge.py | 8 ++------ mautrix/util/async_db/errors.py | 17 +++++++++++++++++ 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/mautrix/bridge/bridge.py b/mautrix/bridge/bridge.py index 74f916c4..9ca56a19 100644 --- a/mautrix/bridge/bridge.py +++ b/mautrix/bridge/bridge.py @@ -178,12 +178,8 @@ def prepare_bridge(self) -> None: def _log_db_error(self, e: DatabaseException) -> None: self.log.critical("Failed to initialize database", exc_info=e) - if isinstance(e, DatabaseNotOwned): - self.log.info("Sharing the same database with different programs is not supported") - elif isinstance(e, ForeignTablesFound): - self.log.info("You can use --ignore-foreign-tables to ignore this error") - elif isinstance(e, UnsupportedDatabaseVersion): - self.log.info("Downgrading the bridge is not supported") + if e.explanation: + self.log.info(e.explanation) sys.exit(25) async def start_db(self) -> None: diff --git a/mautrix/util/async_db/errors.py b/mautrix/util/async_db/errors.py index e15064a6..082d3b8e 100644 --- a/mautrix/util/async_db/errors.py +++ b/mautrix/util/async_db/errors.py @@ -3,11 +3,16 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. +from __future__ import annotations class DatabaseException(RuntimeError): pass + @property + def explanation(self) -> str | None: + return None + class UnsupportedDatabaseVersion(DatabaseException): def __init__(self, name: str, version: int, latest: int) -> None: @@ -15,12 +20,24 @@ def __init__(self, name: str, version: int, latest: int) -> None: f"Unsupported {name} schema version v{version} (latest known is v{latest})" ) + @property + def explanation(self) -> str: + return "Downgrading is not supported" + class ForeignTablesFound(DatabaseException): def __init__(self, explanation: str) -> None: super().__init__(f"The database contains foreign tables ({explanation})") + @property + def explanation(self) -> str: + return "You can use --ignore-foreign-tables to ignore this error" + class DatabaseNotOwned(DatabaseException): def __init__(self, owner: str) -> None: super().__init__(f"The database is owned by {owner}") + + @property + def explanation(self) -> str: + return "Sharing the same database with different programs is not supported" From 5470788bbb2d822364fc873914698b3fd4c0ad1a Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 25 Mar 2022 17:30:24 +0200 Subject: [PATCH 043/456] Allow setting crypto_log in EncryptingAPI --- mautrix/client/encryption_manager.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mautrix/client/encryption_manager.py b/mautrix/client/encryption_manager.py index 3749da37..60a9b908 100644 --- a/mautrix/client/encryption_manager.py +++ b/mautrix/client/encryption_manager.py @@ -40,8 +40,10 @@ class EncryptingAPI(store_updater.StoreUpdatingAPI): """The logger to use for crypto-related things.""" _share_session_events: dict[RoomID, asyncio.Event] - def __init__(self, *args, **kwargs) -> None: + def __init__(self, *args, crypto_log: TraceLogger | None = None, **kwargs) -> None: super().__init__(*args, **kwargs) + if crypto_log: + self.crypto_log = crypto_log self._crypto = None self._share_session_events = {} From fc1b6f0e3a86e6b026d0369a9426d6ecd8477d66 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 25 Mar 2022 17:30:30 +0200 Subject: [PATCH 044/456] Fix type hint --- mautrix/util/config/proxy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/util/config/proxy.py b/mautrix/util/config/proxy.py index 5a1828ab..5b9324b0 100644 --- a/mautrix/util/config/proxy.py +++ b/mautrix/util/config/proxy.py @@ -30,7 +30,7 @@ def __init__( def load(self) -> None: self._data = self._load_proxy() or CommentedMap() - def load_base(self) -> Optional[RecursiveDict[CommentedMap]]: + def load_base(self) -> RecursiveDict[CommentedMap] | None: return self._load_base_proxy() def save(self) -> None: From dd59b3b0e094ba5cea4cde96dea84b7b97ca211c Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 25 Mar 2022 17:34:06 +0200 Subject: [PATCH 045/456] Update changelog --- CHANGELOG.md | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index edf70259..af8413ab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,11 @@ +## v0.15.2 (unreleased) + +* Added support for async media uploads ([MSC2246]). +* Moved `async_getter_lock` decorator to `mautrix.util` (from `mautrix.bridge`). + * The old import path will keep working. + +[MSC2246]: https://github.com/matrix-org/matrix-spec-proposals/pull/2246 + ## v0.15.1 (2022-03-23) * Added `ensure_has_html` method for `TextMessageEventContent` to generate @@ -211,7 +219,7 @@ * Added utility method for adding variation selector 16 to emoji strings the same way as Element does (using emojibase data). -[MSC2716]: https://github.com/matrix-org/matrix-doc/pull/2716 +[MSC2716]: https://github.com/matrix-org/matrix-spec-proposals/pull/2716 ## v0.12.4 (2021-11-25) @@ -381,8 +389,8 @@ * Fixed receiving appservice transactions with `Authorization` header (i.e. fixed [MSC2832] support). -[MSC3202]: https://github.com/matrix-org/matrix-doc/pull/3202 -[MSC2832]: https://github.com/matrix-org/matrix-doc/pull/2832 +[MSC3202]: https://github.com/matrix-org/matrix-spec-proposals/pull/3202 +[MSC2832]: https://github.com/matrix-org/matrix-spec-proposals/pull/2832 [@sumnerevans]: https://github.com/sumnerevans [#49]: https://github.com/mautrix/python/pull/49 @@ -615,8 +623,8 @@ `EventType.Class.UNKNOWN` as the type class. * Fixed regex escaping in bridge registration generation. -[MSC2778]: https://github.com/matrix-org/matrix-doc/pull/2778 -[MSC2409]: https://github.com/matrix-org/matrix-doc/pull/2409 +[MSC2778]: https://github.com/matrix-org/matrix-spec-proposals/pull/2778 +[MSC2409]: https://github.com/matrix-org/matrix-spec-proposals/pull/2409 [@ShadowJonathan]: https://github.com/ShadowJonathan [@witchent]: https://github.com/witchent [#26]: https://github.com/mautrix/python/pull/26 From f4a2a59cfdc6a95986e0abddc7b65f153aacea8a Mon Sep 17 00:00:00 2001 From: Sumner Evans Date: Fri, 25 Mar 2022 09:56:14 -0600 Subject: [PATCH 046/456] async media uploads: add bridge config option for enabling Config name: homeserver.async_media --- mautrix/bridge/config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mautrix/bridge/config.py b/mautrix/bridge/config.py index fbe64c5e..3399c459 100644 --- a/mautrix/bridge/config.py +++ b/mautrix/bridge/config.py @@ -73,6 +73,7 @@ def do_update(self, helper: ConfigUpdateHelper) -> None: copy("homeserver.connection_limit") copy("homeserver.status_endpoint") copy("homeserver.message_send_checkpoint_endpoint") + copy("homeserver.async_media") copy("appservice.address") copy("appservice.hostname") From 87719762ef0ae35314e1ec8f41f3af9406df3b54 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 25 Mar 2022 18:24:19 +0200 Subject: [PATCH 047/456] Call system_exit on critical error --- mautrix/util/program.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mautrix/util/program.py b/mautrix/util/program.py index cc4d7845..0d4883ab 100644 --- a/mautrix/util/program.py +++ b/mautrix/util/program.py @@ -229,6 +229,7 @@ def _run(self) -> None: self.log.debug("Interrupt received, stopping...") except Exception: self.log.critical("Unexpected error in main event loop", exc_info=True) + self.loop.run_until_complete(self.system_exit()) sys.exit(2) except SystemExit: self.loop.run_until_complete(self.system_exit()) From 0b3da381d06907efd7758e8952f47e551c0e87d8 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 25 Mar 2022 18:33:00 +0200 Subject: [PATCH 048/456] Add CHECK constraint for membership column on SQLite --- mautrix/client/state_store/asyncpg/upgrade.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/mautrix/client/state_store/asyncpg/upgrade.py b/mautrix/client/state_store/asyncpg/upgrade.py index fd03e2b2..0a489aae 100644 --- a/mautrix/client/state_store/asyncpg/upgrade.py +++ b/mautrix/client/state_store/asyncpg/upgrade.py @@ -25,15 +25,18 @@ async def upgrade_blank_to_v2(conn: Connection, scheme: Scheme) -> None: power_levels TEXT )""" ) + membership_check = "" if scheme != Scheme.SQLITE: await conn.execute( "CREATE TYPE membership AS ENUM ('join', 'leave', 'invite', 'ban', 'knock')" ) + else: + membership_check = "CHECK (membership IN ('join', 'leave', 'invite', 'ban', 'knock'))" await conn.execute( - """CREATE TABLE mx_user_profile ( + f"""CREATE TABLE mx_user_profile ( room_id TEXT, user_id TEXT, - membership membership NOT NULL, + membership membership NOT NULL {membership_check}, displayname TEXT, avatar_url TEXT, PRIMARY KEY (room_id, user_id) From 76006d5ec04170ea6dac386788d5c386b714c69d Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 25 Mar 2022 19:38:32 +0200 Subject: [PATCH 049/456] Bump version to 0.15.2 --- CHANGELOG.md | 2 +- mautrix/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index af8413ab..6b7b37da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -## v0.15.2 (unreleased) +## v0.15.2 (2022-03-25) * Added support for async media uploads ([MSC2246]). * Moved `async_getter_lock` decorator to `mautrix.util` (from `mautrix.bridge`). diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 2446a694..576f2ef4 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.15.1" +__version__ = "0.15.2" __author__ = "Tulir Asokan " __all__ = [ "api", From 4c93c46922bba43fb86c10c74f6c3224ce0f7066 Mon Sep 17 00:00:00 2001 From: Sumner Evans Date: Fri, 25 Mar 2022 13:54:15 -0600 Subject: [PATCH 050/456] async upload: fix upload method --- mautrix/client/api/modules/media_repository.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mautrix/client/api/modules/media_repository.py b/mautrix/client/api/modules/media_repository.py index 9febffdd..6f690c9b 100644 --- a/mautrix/client/api/modules/media_repository.py +++ b/mautrix/client/api/modules/media_repository.py @@ -105,13 +105,13 @@ async def upload_media( mxc = await self.unstable_create_mxc() path = MediaPath.v3.upload + method = Method.POST if mxc: server_name, media_id = self.api.parse_mxc_uri(mxc) path = MediaPath.unstable["fi.mau.msc2246"].upload[server_name][media_id] + method = Method.PUT - task = self.api.request( - Method.POST, path, content=data, headers=headers, query_params=query - ) + task = self.api.request(method, path, content=data, headers=headers, query_params=query) if async_upload: async def _try_upload(): From aace0c1131290093949f56b7a1f5b631694b1c6d Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 25 Mar 2022 21:14:01 +0200 Subject: [PATCH 051/456] Don't run upgrades if there are none --- mautrix/util/async_db/database.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mautrix/util/async_db/database.py b/mautrix/util/async_db/database.py index 8f87aabf..97cb94bf 100644 --- a/mautrix/util/async_db/database.py +++ b/mautrix/util/async_db/database.py @@ -101,7 +101,8 @@ async def start(self) -> None: await self._check_foreign_tables() if self.owner_name: await self._check_owner() - await self.upgrade_table.upgrade(self) + if self.upgrade_table and len(self.upgrade_table.upgrades) > 0: + await self.upgrade_table.upgrade(self) async def _check_foreign_tables(self) -> None: if await self.table_exists("state_groups_state"): From ea272b1ed34cea3ec8cf5ec68fabd00b83b79473 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 25 Mar 2022 22:24:26 +0200 Subject: [PATCH 052/456] Bump version to 0.15.3 --- CHANGELOG.md | 4 ++++ mautrix/__init__.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b7b37da..334c1d09 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## v0.15.3 (2022-03-25) + +* Fixed incorrect HTTP method in async media uploads. + ## v0.15.2 (2022-03-25) * Added support for async media uploads ([MSC2246]). diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 576f2ef4..eff91a98 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.15.2" +__version__ = "0.15.3" __author__ = "Tulir Asokan " __all__ = [ "api", From e248573c74efd1448f0423198d5bf546d2cde8ca Mon Sep 17 00:00:00 2001 From: Sumner Evans Date: Fri, 25 Mar 2022 14:59:36 -0600 Subject: [PATCH 053/456] async media: fix method on /create --- mautrix/client/api/modules/media_repository.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/client/api/modules/media_repository.py b/mautrix/client/api/modules/media_repository.py index 6f690c9b..e181b148 100644 --- a/mautrix/client/api/modules/media_repository.py +++ b/mautrix/client/api/modules/media_repository.py @@ -47,7 +47,7 @@ async def unstable_create_mxc(self) -> ContentURI: Raises: MatrixResponseError: If the response does not contain a ``content_uri`` field. """ - resp = await self.api.request(Method.PUT, MediaPath.unstable["fi.mau.msc2246"].create) + resp = await self.api.request(Method.POST, MediaPath.unstable["fi.mau.msc2246"].create) try: return resp["content_uri"] except KeyError: From 9e76f365cc55fa716e6a42cd485b08bc53729f8a Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 25 Mar 2022 23:21:35 +0200 Subject: [PATCH 054/456] Bump version to 0.15.4 --- CHANGELOG.md | 4 ++-- mautrix/__init__.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 334c1d09..bd4eb089 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ -## v0.15.3 (2022-03-25) +## v0.15.3 & v0.15.4 (2022-03-25) -* Fixed incorrect HTTP method in async media uploads. +* Fixed incorrect HTTP methods in async media uploads. ## v0.15.2 (2022-03-25) diff --git a/mautrix/__init__.py b/mautrix/__init__.py index eff91a98..94226522 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.15.3" +__version__ = "0.15.4" __author__ = "Tulir Asokan " __all__ = [ "api", From 7aca37db440b046a7bcd205ace5ad3e8fd4dbdc2 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 26 Mar 2022 12:03:46 +0200 Subject: [PATCH 055/456] Allow None in upgrade_table type hint --- mautrix/util/async_db/database.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mautrix/util/async_db/database.py b/mautrix/util/async_db/database.py index 97cb94bf..351dda3b 100644 --- a/mautrix/util/async_db/database.py +++ b/mautrix/util/async_db/database.py @@ -31,14 +31,14 @@ class Database(ABC): scheme: Scheme url: URL _db_args: dict[str, Any] - upgrade_table: UpgradeTable + upgrade_table: UpgradeTable | None owner_name: str | None ignore_foreign_tables: bool def __init__( self, url: URL, - upgrade_table: UpgradeTable, + upgrade_table: UpgradeTable | None, db_args: dict[str, Any] | None = None, log: TraceLogger | None = None, owner_name: str | None = None, From 0495ade75d91ab368a161f049aa2a69fc723b451 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 26 Mar 2022 16:47:25 +0200 Subject: [PATCH 056/456] Fix some async_db type hints --- mautrix/util/async_db/aiosqlite.py | 16 +++++++++++----- mautrix/util/async_db/connection.py | 4 +++- mautrix/util/async_db/connection.pyi | 2 +- mautrix/util/async_db/database.py | 4 +++- 4 files changed, 18 insertions(+), 8 deletions(-) diff --git a/mautrix/util/async_db/aiosqlite.py b/mautrix/util/async_db/aiosqlite.py index 99085dcb..fa902b7c 100644 --- a/mautrix/util/async_db/aiosqlite.py +++ b/mautrix/util/async_db/aiosqlite.py @@ -47,12 +47,16 @@ def __execute(self, query: str, *args: Any): query = POSITIONAL_PARAM_PATTERN.sub(r"?\1", query) return super().execute(query, args) - async def execute(self, query: str, *args: Any, timeout: float | None = None) -> None: - await self.__execute(query, *args) + async def execute( + self, query: str, *args: Any, timeout: float | None = None + ) -> aiosqlite.Cursor: + return await self.__execute(query, *args) - async def executemany(self, query: str, *args: Any, timeout: float | None = None) -> None: + async def executemany( + self, query: str, *args: Any, timeout: float | None = None + ) -> aiosqlite.Cursor: query = POSITIONAL_PARAM_PATTERN.sub(r"?\1", query) - await super().executemany(query, *args) + return await super().executemany(query, *args) async def fetch( self, query: str, *args: Any, timeout: float | None = None @@ -60,7 +64,9 @@ async def fetch( async with self.__execute(query, *args) as cursor: return list(await cursor.fetchall()) - async def fetchrow(self, query: str, *args: Any, timeout: float | None = None) -> sqlite3.Row: + async def fetchrow( + self, query: str, *args: Any, timeout: float | None = None + ) -> sqlite3.Row | None: async with self.__execute(query, *args) as cursor: return await cursor.fetchone() diff --git a/mautrix/util/async_db/connection.py b/mautrix/util/async_db/connection.py index 32a96fbd..4dd4980e 100644 --- a/mautrix/util/async_db/connection.py +++ b/mautrix/util/async_db/connection.py @@ -89,7 +89,9 @@ async def fetchval( return await self.wrapped.fetchval(query, *args, column=column, timeout=timeout) @log_duration - async def fetchrow(self, query: str, *args: Any, timeout: float | None = None) -> Row | Record: + async def fetchrow( + self, query: str, *args: Any, timeout: float | None = None + ) -> Row | Record | None: return await self.wrapped.fetchrow(query, *args, timeout=timeout) async def table_exists(self, name: str) -> bool: diff --git a/mautrix/util/async_db/connection.pyi b/mautrix/util/async_db/connection.pyi index 2a305d74..e4219e50 100644 --- a/mautrix/util/async_db/connection.pyi +++ b/mautrix/util/async_db/connection.pyi @@ -35,7 +35,7 @@ class LoggingConnection: ) -> Any: ... async def fetchrow( self, query: str, *args: Any, timeout: float | None = None - ) -> Row | Record: ... + ) -> Row | Record | None: ... async def table_exists(self, name: str) -> bool: ... async def copy_records_to_table( self, diff --git a/mautrix/util/async_db/database.py b/mautrix/util/async_db/database.py index 351dda3b..d6e26eed 100644 --- a/mautrix/util/async_db/database.py +++ b/mautrix/util/async_db/database.py @@ -149,7 +149,9 @@ async def fetchval( async with self.acquire() as conn: return await conn.fetchval(query, *args, column=column, timeout=timeout) - async def fetchrow(self, query: str, *args: Any, timeout: float | None = None) -> Record: + async def fetchrow( + self, query: str, *args: Any, timeout: float | None = None + ) -> Record | None: async with self.acquire() as conn: return await conn.fetchrow(query, *args, timeout=timeout) From 1a04fd6dda49599ab1098e54a202f5b59627ef2e Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 28 Mar 2022 16:28:44 +0300 Subject: [PATCH 057/456] Default to ignoring instead of rejecting key requests --- mautrix/crypto/key_share.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/mautrix/crypto/key_share.py b/mautrix/crypto/key_share.py index ca2d984f..0221e5fb 100644 --- a/mautrix/crypto/key_share.py +++ b/mautrix/crypto/key_share.py @@ -65,9 +65,7 @@ async def default_allow_key_share( """ if device.user_id != self.client.mxid: raise RejectKeyShare( - f"Rejecting key request from a different user ({device.user_id})", - code=RoomKeyWithheldCode.UNAUTHORIZED, - reason="This device does not share keys to other users", + f"Ignoring key request from a different user ({device.user_id})", code=None ) elif device.device_id == self.client.device_id: raise RejectKeyShare("Ignoring key request from ourselves", code=None) From 94d1a6baa6553f20b51a6f771c53076ed9b23c2a Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 28 Mar 2022 16:32:27 +0300 Subject: [PATCH 058/456] Bump version to 0.15.5 --- mautrix/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 94226522..b553f745 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.15.4" +__version__ = "0.15.5" __author__ = "Tulir Asokan " __all__ = [ "api", From 2877d7f2822ca4a60bed52118bce6f86dd5cf37d Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 28 Mar 2022 22:24:29 +0300 Subject: [PATCH 059/456] Update and unpin black It's stable now, so there shouldn't be too many changes --- .github/workflows/python-package.yml | 2 +- .pre-commit-config.yaml | 13 +++++-------- dev-requirements.txt | 2 +- pyproject.toml | 1 - 4 files changed, 7 insertions(+), 11 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 5f8da295..70e8cf65 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -54,7 +54,7 @@ jobs: - uses: psf/black@stable with: src: "./mautrix" - version: "22.1.0" + version: "22.3.0" - name: pre-commit run: | pip install pre-commit diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5d9f684a..56919be3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,17 +7,14 @@ repos: - id: end-of-file-fixer - id: check-yaml - id: check-added-large-files - # TODO convert to use the upstream psf/black when - # https://github.com/psf/black/issues/2493 gets fixed - - repo: local + - repo: https://github.com/psf/black + rev: 22.3.0 hooks: - id: black - name: black - entry: black --check - language: system - files: ^mautrix/.*\.py$ + language_version: python3 + files: ^mautrix/.*\.pyi?$ - repo: https://github.com/PyCQA/isort rev: 5.10.1 hooks: - id: isort - files: ^mautrix/.*$ + files: ^mautrix/.*\.pyi?$ diff --git a/dev-requirements.txt b/dev-requirements.txt index 232f7249..e513c0df 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,3 +1,3 @@ pre-commit>=2.10.1,<3 isort>=5.10.1,<6 -black==22.1.0 +black>=22.3,<23 diff --git a/pyproject.toml b/pyproject.toml index 64b48c44..134042b1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,4 +8,3 @@ line_length = 99 [tool.black] line-length = 99 target-version = ["py38"] -required-version = "22.1.0" From 94c78995a3e81b22152954fe0d50f5e6b47ea90f Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 30 Mar 2022 15:47:51 +0300 Subject: [PATCH 060/456] Remove old reply fallback in set_reply Also changed the HTML reply fallback regex to be greedy to avoid issues when removing nested reply fallbacks --- mautrix/types/event/message.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mautrix/types/event/message.py b/mautrix/types/event/message.py index 6285b2e0..eee1250e 100644 --- a/mautrix/types/event/message.py +++ b/mautrix/types/event/message.py @@ -326,7 +326,7 @@ class LocationMessageEventContent(BaseMessageEventContent, SerializableAttrs): info: LocationInfo = None -html_reply_fallback_regex: Pattern = re.compile("^" r"[\s\S]+?") +html_reply_fallback_regex: Pattern = re.compile(r"^[\s\S]+") @dataclass @@ -344,6 +344,8 @@ def set_reply( return if isinstance(reply_to, MessageEvent): self.ensure_has_html() + if isinstance(reply_to.content, TextMessageEventContent): + reply_to.content.trim_reply_fallback() self.formatted_body = ( reply_to.make_reply_fallback_html(displayname) + self.formatted_body ) From 566ea0556c82e53f6919a6a59b0002b886c5d9d4 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 30 Mar 2022 15:50:30 +0300 Subject: [PATCH 061/456] Bump version to 0.15.6 --- CHANGELOG.md | 13 +++++++++++++ mautrix/__init__.py | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bd4eb089..e9179fda 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,16 @@ +## v0.15.6 (2022-03-30) + +* Fixed removing nested (i.e. malformed) reply fallbacks generated by + some clients. +* Added automatic reply fallback trimming to `set_reply()` to prevent + accidentally creating nested reply fallbacks. + +## v0.15.5 (2022-03-28) + +* Changed default behavior of OlmMachine to ignore instead of reject + key requests from other users. +* Fixed some type hints + ## v0.15.3 & v0.15.4 (2022-03-25) * Fixed incorrect HTTP methods in async media uploads. diff --git a/mautrix/__init__.py b/mautrix/__init__.py index b553f745..9e88bd8e 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.15.5" +__version__ = "0.15.6" __author__ = "Tulir Asokan " __all__ = [ "api", From ef8d16d792fd3e8459725e012db8e64bb9fb47b3 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 3 Apr 2022 12:05:42 +0300 Subject: [PATCH 062/456] Add file_name parameter to get_download_url --- mautrix/api.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/mautrix/api.py b/mautrix/api.py index c6db9cab..cacb5f39 100644 --- a/mautrix/api.py +++ b/mautrix/api.py @@ -411,7 +411,10 @@ def get_txn_id(self) -> str: return f"mautrix-python_R{self.txn_id}@T{int(time() * 1000)}" def get_download_url( - self, mxc_uri: str, download_type: Literal["download", "thumbnail"] = "download" + self, + mxc_uri: str, + download_type: Literal["download", "thumbnail"] = "download", + file_name: str | None = None, ) -> URL: """ Get the full HTTP URL to download a ``mxc://`` URI. @@ -419,6 +422,7 @@ def get_download_url( Args: mxc_uri: The MXC URI whose full URL to get. download_type: The type of download ("download" or "thumbnail"). + file_name: Optionally, a file name to include in the download URL. Returns: The full HTTP URL. @@ -427,15 +431,18 @@ def get_download_url( ValueError: If `mxc_uri` doesn't begin with ``mxc://``. Examples: - >>> api = HTTPAPI(...) + >>> api = HTTPAPI(base_url="https://matrix-client.matrix.org", ...) >>> api.get_download_url("mxc://matrix.org/pqjkOuKZ1ZKRULWXgz2IVZV6") - "https://matrix.org/_matrix/media/v3/download/matrix.org/pqjkOuKZ1ZKRULWXgz2IVZV6" + "https://matrix-client.matrix.org/_matrix/media/v3/download/matrix.org/pqjkOuKZ1ZKRULWXgz2IVZV6" + >>> api.get_download_url("mxc://matrix.org/pqjkOuKZ1ZKRULWXgz2IVZV6", file_name="hello.png") + "https://matrix-client.matrix.org/_matrix/media/v3/download/matrix.org/pqjkOuKZ1ZKRULWXgz2IVZV6/hello.png" """ server_name, media_id = self.parse_mxc_uri(mxc_uri) version = "r0" if self.hacky_replace_v3_with_r0 else "v3" - return ( - self.base_url / str(APIPath.MEDIA) / version / download_type / server_name / media_id - ) + url = self.base_url / str(APIPath.MEDIA) / version / download_type / server_name / media_id + if file_name: + url /= file_name + return url @staticmethod def parse_mxc_uri(mxc_uri: str) -> tuple[str, str]: From 4c5df89fc0bad78456c378679475499ab286ad03 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 5 Apr 2022 21:37:35 +0300 Subject: [PATCH 063/456] Add info and reason to bridge state things --- mautrix/bridge/user.py | 4 ++++ mautrix/util/bridge_state.py | 4 +++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/mautrix/bridge/user.py b/mautrix/bridge/user.py index 36d7bdc6..9948c286 100644 --- a/mautrix/bridge/user.py +++ b/mautrix/bridge/user.py @@ -178,6 +178,8 @@ async def push_bridge_state( message: str | None = None, ttl: int | None = None, remote_id: str | None = None, + info: dict[str, Any] | None = None, + reason: str | None = None, ) -> None: if not self.bridge.config["homeserver.status_endpoint"]: return @@ -188,6 +190,8 @@ async def push_bridge_state( message=message, ttl=ttl, remote_id=remote_id, + info=info, + reason=reason, ) await self.fill_bridge_state(state) if state.should_deduplicate(self._prev_bridge_status): diff --git a/mautrix/util/bridge_state.py b/mautrix/util/bridge_state.py index 9ff3cadd..82df9c5b 100644 --- a/mautrix/util/bridge_state.py +++ b/mautrix/util/bridge_state.py @@ -3,7 +3,7 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from typing import ClassVar, Dict, Optional +from typing import Any, ClassVar, Dict, Optional import logging import time @@ -74,6 +74,8 @@ class BridgeState(SerializableAttrs): source: Optional[str] = None error: Optional[str] = None message: Optional[str] = None + info: Optional[Dict[str, Any]] = None + reason: Optional[str] = None send_attempts_: int = field(default=0, hidden=True) From 3c9aea745e5a6691d229af59a9907153b3c4aa3a Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 5 Apr 2022 21:38:50 +0300 Subject: [PATCH 064/456] Bump version to 0.15.7 --- CHANGELOG.md | 4 ++++ mautrix/__init__.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e9179fda..9d8f5be6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## v0.15.7 (2022-04-05) + +* Added `file_name` parameter to `HTTPAPI.get_download_url`. + ## v0.15.6 (2022-03-30) * Fixed removing nested (i.e. malformed) reply fallbacks generated by diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 9e88bd8e..08c44be3 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.15.6" +__version__ = "0.15.7" __author__ = "Tulir Asokan " __all__ = [ "api", From 9896fff1326cdbf23f2dcf1f1fdc5206434cf0dc Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 6 Apr 2022 23:21:27 +0300 Subject: [PATCH 065/456] Add experimental metric for upload speed --- .../client/api/modules/media_repository.py | 28 +++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/mautrix/client/api/modules/media_repository.py b/mautrix/client/api/modules/media_repository.py index e181b148..45720506 100644 --- a/mautrix/client/api/modules/media_repository.py +++ b/mautrix/client/api/modules/media_repository.py @@ -6,12 +6,15 @@ from __future__ import annotations from typing import Any, AsyncIterable, Literal +from contextlib import contextmanager import asyncio +import time from mautrix import __optional_imports__ from mautrix.api import MediaPath, Method from mautrix.errors import MatrixResponseError from mautrix.types import ContentURI, MediaRepoConfig, MXOpenGraph, SerializerError +from mautrix.util.opt_prometheus import Histogram from ..base import BaseClientAPI @@ -22,6 +25,12 @@ raise magic = None # type: ignore +UPLOAD_TIME = Histogram( + "bridge_media_upload_time", + "Time spent uploading media (milliseconds per megabyte)", + buckets=[1, 5, 10, 25, 50, 100, 250, 500, 750, 1000, 2500, 5000, 10000], +) + class MediaRepositoryMethods(BaseClientAPI): """ @@ -53,6 +62,17 @@ async def unstable_create_mxc(self) -> ContentURI: except KeyError: raise MatrixResponseError("`content_uri` not in response.") + @contextmanager + def _observe_upload_time(self, size: int | None, mxc: ContentURI | None = None) -> None: + start = time.monotonic_ns() + yield + duration = time.monotonic_ns() - start + if mxc: + duration_sec = duration / 1000**3 + self.log.debug(f"Completed asynchronous upload of {mxc} in {duration_sec:.3f} seconds") + if size: + UPLOAD_TIME.observe(duration / size) + async def upload_media( self, data: bytes | bytearray | AsyncIterable[bytes], @@ -95,6 +115,8 @@ async def upload_media( headers["Content-Type"] = mime_type if size: headers["Content-Length"] = str(size) + elif isinstance(data, (bytes, bytearray)): + size = len(data) query = {} if filename: query["filename"] = filename @@ -116,14 +138,16 @@ async def upload_media( async def _try_upload(): try: - await task + with self._observe_upload_time(size, mxc): + await task except Exception as e: self.log.error(f"Failed to upload {mxc}: {type(e).__name__}: {e}") asyncio.create_task(_try_upload()) return mxc else: - resp = await task + with self._observe_upload_time(size): + resp = await task try: return resp["content_uri"] except KeyError: From b2400a11efb694f9eea1bd600e7ebe303532cf86 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 7 Apr 2022 21:14:58 +0300 Subject: [PATCH 066/456] Remove unnecessary buckets --- mautrix/client/api/modules/media_repository.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/client/api/modules/media_repository.py b/mautrix/client/api/modules/media_repository.py index 45720506..8e592679 100644 --- a/mautrix/client/api/modules/media_repository.py +++ b/mautrix/client/api/modules/media_repository.py @@ -28,7 +28,7 @@ UPLOAD_TIME = Histogram( "bridge_media_upload_time", "Time spent uploading media (milliseconds per megabyte)", - buckets=[1, 5, 10, 25, 50, 100, 250, 500, 750, 1000, 2500, 5000, 10000], + buckets=[10, 25, 50, 100, 250, 500, 750, 1000, 2500, 5000, 10000], ) From 62f513eda2739cffbf4d06b1d36632096261a3a3 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 8 Apr 2022 17:53:49 +0300 Subject: [PATCH 067/456] Fix UpgradeTable.register type hint --- mautrix/util/async_db/upgrade.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/mautrix/util/async_db/upgrade.py b/mautrix/util/async_db/upgrade.py index 5d2eb42a..d69dac56 100644 --- a/mautrix/util/async_db/upgrade.py +++ b/mautrix/util/async_db/upgrade.py @@ -61,17 +61,18 @@ def __init__( def register( self, + _outer_fn: Upgrade | UpgradeWithoutScheme | None = None, + *, index: int = -1, description: str = "", - _outer_fn: Upgrade | None = None, transaction: bool = True, upgrades_to: int | Upgrade | None = None, - ) -> Upgrade | Callable[[Upgrade], Upgrade] | None: + ) -> Upgrade | Callable[[Upgrade | UpgradeWithoutScheme], Upgrade]: if isinstance(index, str): description = index index = -1 - def actually_register(fn: Upgrade) -> Upgrade: + def actually_register(fn: Upgrade | UpgradeWithoutScheme) -> Upgrade: fn = _wrap_upgrade(fn) fn.__mau_db_upgrade_description__ = description fn.__mau_db_upgrade_transaction__ = transaction From 0b79ec60e7fb4992a9557acb7f2548e2bf6aece1 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 8 Apr 2022 17:53:59 +0300 Subject: [PATCH 068/456] Redact password when logging DB URL --- mautrix/util/async_db/asyncpg.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mautrix/util/async_db/asyncpg.py b/mautrix/util/async_db/asyncpg.py index dbec68b4..40def15e 100644 --- a/mautrix/util/async_db/asyncpg.py +++ b/mautrix/util/async_db/asyncpg.py @@ -55,7 +55,10 @@ def override_pool(self, db: PostgresDatabase) -> None: async def start(self) -> None: if not self._pool_override: self._db_args["loop"] = asyncio.get_running_loop() - self.log.debug(f"Connecting to {self.url}") + log_url = self.url + if log_url.password: + log_url = log_url.with_password("password-redacted") + self.log.debug(f"Connecting to {log_url}") self._pool = await asyncpg.create_pool(str(self.url), **self._db_args) await super().start() From 3de690b7ca2fc0a91b4933fa9ee6652aa642ed31 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 8 Apr 2022 17:58:13 +0300 Subject: [PATCH 069/456] Bump version to 0.15.8 --- CHANGELOG.md | 23 ++++++++++++++--------- mautrix/__init__.py | 2 +- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d8f5be6..7dc2542e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,27 +1,32 @@ +## v0.15.8 (2022-04-08) + +* *(client.api)* Added experimental prometheus metric for file upload speed. +* *(util.async_db)* Improved type hints for `UpgradeTable.register` + ## v0.15.7 (2022-04-05) -* Added `file_name` parameter to `HTTPAPI.get_download_url`. +* *(api)* Added `file_name` parameter to `HTTPAPI.get_download_url`. ## v0.15.6 (2022-03-30) -* Fixed removing nested (i.e. malformed) reply fallbacks generated by +* *(types)* Fixed removing nested (i.e. malformed) reply fallbacks generated by some clients. -* Added automatic reply fallback trimming to `set_reply()` to prevent +* *(types)* Added automatic reply fallback trimming to `set_reply()` to prevent accidentally creating nested reply fallbacks. ## v0.15.5 (2022-03-28) -* Changed default behavior of OlmMachine to ignore instead of reject +* *(crypto)* Changed default behavior of OlmMachine to ignore instead of reject key requests from other users. * Fixed some type hints ## v0.15.3 & v0.15.4 (2022-03-25) -* Fixed incorrect HTTP methods in async media uploads. +* *(client.api)* Fixed incorrect HTTP methods in async media uploads. ## v0.15.2 (2022-03-25) -* Added support for async media uploads ([MSC2246]). +* *(client.api)* Added support for async media uploads ([MSC2246]). * Moved `async_getter_lock` decorator to `mautrix.util` (from `mautrix.bridge`). * The old import path will keep working. @@ -29,9 +34,9 @@ ## v0.15.1 (2022-03-23) -* Added `ensure_has_html` method for `TextMessageEventContent` to generate - a HTML `formatted_body` from the plaintext `body` correctly (i.e. escaping - HTML and replacing newlines). +* *(types)* Added `ensure_has_html` method for `TextMessageEventContent` to + generate a HTML `formatted_body` from the plaintext `body` correctly (i.e. + escaping HTML and replacing newlines). ## v0.15.0 (2022-03-16) diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 08c44be3..6692e329 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.15.7" +__version__ = "0.15.8" __author__ = "Tulir Asokan " __all__ = [ "api", From 8f1d81e755263d7fcc9706d4fcc3cfd6b5e5af05 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 8 Apr 2022 18:02:21 +0300 Subject: [PATCH 070/456] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7dc2542e..8998095d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ * *(client.api)* Added experimental prometheus metric for file upload speed. * *(util.async_db)* Improved type hints for `UpgradeTable.register` +* *(util.async_db)* Changed connection string log to redact database password. ## v0.15.7 (2022-04-05) From ab167bdd512a459f123f29da1d9211d7872655a9 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 9 Apr 2022 20:43:23 +0300 Subject: [PATCH 071/456] Remove black version check in genall.py --- mautrix/genall.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/mautrix/genall.py b/mautrix/genall.py index 60bc01a8..14fda230 100644 --- a/mautrix/genall.py +++ b/mautrix/genall.py @@ -9,9 +9,6 @@ root_module = Path(__file__).parent black_cfg = black.parse_pyproject_toml(str(root_module.parent / "pyproject.toml")) -assert ( - black.__version__ == black_cfg["required_version"] -), f"Incorrect Black version {black.__version__}" black_mode = black.Mode( target_versions={black.TargetVersion[ver.upper()] for ver in black_cfg["target_version"]}, line_length=black_cfg["line_length"], From 4a979e8bb55668eef89b47cfa7f0011c3e10ab14 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 9 Apr 2022 20:51:49 +0300 Subject: [PATCH 072/456] Remove custom reply relation type --- CHANGELOG.md | 9 ++++ mautrix/types/__init__.py | 2 + mautrix/types/event/__init__.py | 1 + mautrix/types/event/message.py | 82 ++++++--------------------------- 4 files changed, 26 insertions(+), 68 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8998095d..8c19b53b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,12 @@ +## v0.16.0 (unreleased) + +* **Breaking change *(types)*** Removed custom `REPLY` relation type and + changed `RelatesTo` structure to match the actual event content. + * Applications using `content.get_reply_to()` and `content.set_reply()` will + keep working with no changes. +* *(types)* Added `THREAD` relation type and `is_falling_back` field to + `RelatesTo`. + ## v0.15.8 (2022-04-08) * *(client.api)* Added experimental prometheus metric for file upload speed. diff --git a/mautrix/types/__init__.py b/mautrix/types/__init__.py index 352ea4bd..a53e965c 100644 --- a/mautrix/types/__init__.py +++ b/mautrix/types/__init__.py @@ -64,6 +64,7 @@ ForwardedRoomKeyEventContent, GenericEvent, ImageInfo, + InReplyTo, JoinRule, JoinRulesStateEventContent, JSONWebKey, @@ -243,6 +244,7 @@ "ForwardedRoomKeyEventContent", "GenericEvent", "ImageInfo", + "InReplyTo", "JoinRule", "JoinRulesStateEventContent", "JSONWebKey", diff --git a/mautrix/types/event/__init__.py b/mautrix/types/event/__init__.py index 5cc6cf91..a81358b0 100644 --- a/mautrix/types/event/__init__.py +++ b/mautrix/types/event/__init__.py @@ -42,6 +42,7 @@ FileInfo, Format, ImageInfo, + InReplyTo, JSONWebKey, LocationInfo, LocationMessageEventContent, diff --git a/mautrix/types/event/message.py b/mautrix/types/event/message.py index eee1250e..bad471b8 100644 --- a/mautrix/types/event/message.py +++ b/mautrix/types/event/message.py @@ -54,90 +54,35 @@ def is_media(self) -> bool: # region Relations -class InReplyTo: - def __init__( - self, event_id: Optional[EventID] = None, proxy_target: Optional["RelatesTo"] = None - ) -> None: - self._event_id = event_id - self._proxy_target = proxy_target - - @property - def event_id(self) -> EventID: - if self._proxy_target: - return self._proxy_target.event_id - return self._event_id - - @event_id.setter - def event_id(self, event_id: EventID) -> None: - if self._proxy_target: - self._proxy_target.rel_type = RelationType.REPLY - self._proxy_target.event_id = event_id - else: - self._event_id = event_id +@dataclass +class InReplyTo(SerializableAttrs): + event_id: EventID class RelationType(ExtensibleEnum): ANNOTATION: "RelationType" = "m.annotation" REFERENCE: "RelationType" = "m.reference" REPLACE: "RelationType" = "m.replace" - REPLY: "RelationType" = "net.maunium.reply" + THREAD: "RelationType" = "m.thread" @dataclass -class RelatesTo(Serializable): +class RelatesTo(SerializableAttrs): """Message relations. Used for reactions, edits and replies.""" rel_type: RelationType = None event_id: Optional[EventID] = None key: Optional[str] = None - _extra: Dict[str, Any] = attr.ib(factory=lambda: {}) + is_falling_back: Optional[bool] = None + in_reply_to: Optional[InReplyTo] = field(default=None, json="m.in_reply_to") def __bool__(self) -> bool: - return bool(self.rel_type) and bool(self.event_id) - - @classmethod - def deserialize(cls, data: JSON) -> Optional["RelatesTo"]: - if not data: - return None - try: - return cls( - rel_type=RelationType.deserialize(data.pop("rel_type")), - event_id=data.pop("event_id", None), - key=data.pop("key", None), - extra=data, - ) - except KeyError: - pass - try: - return cls(rel_type=RelationType.REPLY, event_id=data["m.in_reply_to"]["event_id"]) - except KeyError: - pass - return None + return (bool(self.rel_type) and bool(self.event_id)) or bool(self.in_reply_to) def serialize(self) -> JSON: if not self: return attr.NOTHING - data = { - **self._extra, - "rel_type": self.rel_type.serialize(), - } - if self.rel_type == RelationType.REPLY: - data["m.in_reply_to"] = {"event_id": self.event_id} - if self.event_id: - data["event_id"] = self.event_id - if self.key: - data["key"] = self.key - return data - - def __setitem__(self, key: str, value: Any) -> None: - if key in ("rel_type", "event_id", "key"): - return setattr(self, key, value) - self._extra[key] = value - - def __getitem__(self, item: str) -> None: - if item in ("rel_type", "event_id", "key"): - return getattr(self, item) - return self._extra[item] + return super().serialize() # endregion @@ -152,8 +97,9 @@ class BaseMessageEventContentFuncs: _relates_to: Optional[RelatesTo] def set_reply(self, reply_to: Union[EventID, "MessageEvent"], **kwargs) -> None: - self.relates_to.rel_type = RelationType.REPLY - self.relates_to.event_id = reply_to if isinstance(reply_to, str) else reply_to.event_id + self.relates_to.in_reply_to = InReplyTo( + event_id=reply_to if isinstance(reply_to, str) else reply_to.event_id + ) def set_edit(self, edits: Union[EventID, "MessageEvent"]) -> None: self.relates_to.rel_type = RelationType.REPLACE @@ -183,8 +129,8 @@ def relates_to(self, relates_to: RelatesTo) -> None: self._relates_to = relates_to def get_reply_to(self) -> Optional[EventID]: - if self._relates_to and self._relates_to.rel_type == RelationType.REPLY: - return self._relates_to.event_id + if self._relates_to and self._relates_to.in_reply_to: + return self._relates_to.in_reply_to.event_id return None def get_edit(self) -> Optional[EventID]: From 70762c438fba200f39c46e92e6c0200bb76523d1 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 11 Apr 2022 13:54:19 +0300 Subject: [PATCH 073/456] Bump version to 0.16.0 --- CHANGELOG.md | 2 +- mautrix/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c19b53b..5ca3555c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -## v0.16.0 (unreleased) +## v0.16.0 (2022-04-11) * **Breaking change *(types)*** Removed custom `REPLY` relation type and changed `RelatesTo` structure to match the actual event content. diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 6692e329..d033fb13 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.15.8" +__version__ = "0.16.0" __author__ = "Tulir Asokan " __all__ = [ "api", From c115520cec26bed4104b7a3f6a22cf11cf1bd151 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 16 Apr 2022 13:44:10 +0300 Subject: [PATCH 074/456] Remove r0 path support --- CHANGELOG.md | 6 ++++++ mautrix/api.py | 12 ++---------- mautrix/appservice/api/appservice.py | 2 -- mautrix/bridge/custom_puppet.py | 2 -- mautrix/bridge/e2ee.py | 1 - mautrix/client/api/base.py | 5 +---- 6 files changed, 9 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ca3555c..47de1c1d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## v0.16.1 (unreleased) + +* **Breaking change** Removed `r0` path support. + * The new `v3` paths are implemented since Synapse 1.48, Dendrite 0.6.5, + and Conduit 0.4.0. Servers older than these are no longer supported. + ## v0.16.0 (2022-04-11) * **Breaking change *(types)*** Removed custom `REPLY` relation type and diff --git a/mautrix/api.py b/mautrix/api.py index cacb5f39..27255ee6 100644 --- a/mautrix/api.py +++ b/mautrix/api.py @@ -199,9 +199,6 @@ class HTTPAPI: default_retry_count: int """The default retry count to use if a custom value is not passed to :meth:`request`""" - hacky_replace_v3_with_r0: bool = False - """A hacky flag to replace /v3 with /r0 in all API paths.""" - def __init__( self, base_url: URL | str, @@ -283,7 +280,7 @@ def _log_request( else: log_content = content as_user = query_params.get("user_id", None) - level = 1 if path == Path.v3.sync or path == Path.r0.sync else 5 + level = 1 if path == Path.v3.sync else 5 self.log.log( level, f"{method}#{req_id} /{path} {log_content}".strip(" "), @@ -342,10 +339,6 @@ async def request( Returns: The parsed response JSON. """ - if self.hacky_replace_v3_with_r0: - path = path.replace("_matrix/client/v3", "_matrix/client/r0") - path = path.replace("_matrix/media/v3", "_matrix/media/r0") - headers = headers or {} if self.token: headers["Authorization"] = f"Bearer {self.token}" @@ -438,8 +431,7 @@ def get_download_url( "https://matrix-client.matrix.org/_matrix/media/v3/download/matrix.org/pqjkOuKZ1ZKRULWXgz2IVZV6/hello.png" """ server_name, media_id = self.parse_mxc_uri(mxc_uri) - version = "r0" if self.hacky_replace_v3_with_r0 else "v3" - url = self.base_url / str(APIPath.MEDIA) / version / download_type / server_name / media_id + url = self.base_url / str(APIPath.MEDIA) / "v3" / download_type / server_name / media_id if file_name: url /= file_name return url diff --git a/mautrix/appservice/api/appservice.py b/mautrix/appservice/api/appservice.py index f59d5907..af6ae48d 100644 --- a/mautrix/appservice/api/appservice.py +++ b/mautrix/appservice/api/appservice.py @@ -148,7 +148,6 @@ def real_user(self, mxid: UserID, token: str, base_url: URL | None = None) -> Ap bridge_name=self.bridge_name, default_retry_count=self.default_retry_count, ) - child.hacky_replace_v3_with_r0 = self.hacky_replace_v3_with_r0 self.real_users[mxid] = child return child @@ -263,7 +262,6 @@ def __init__(self, user: UserID, parent: AppServiceAPI) -> None: bridge_name=parent.bridge_name, default_retry_count=parent.default_retry_count, ) - self.hacky_replace_v3_with_r0 = parent.hacky_replace_v3_with_r0 self.parent = parent @property diff --git a/mautrix/bridge/custom_puppet.py b/mautrix/bridge/custom_puppet.py index db0027d6..c5c77f25 100644 --- a/mautrix/bridge/custom_puppet.py +++ b/mautrix/bridge/custom_puppet.py @@ -171,8 +171,6 @@ async def _login_with_shared_secret(cls, mxid: UserID) -> str: raise AutologinError(f"No homeserver URL configured for {server}") password = hmac.new(secret, mxid.encode("utf-8"), hashlib.sha512).hexdigest() url = base_url / str(Path.v3.login) - if cls.az.intent.api.hacky_replace_v3_with_r0: - url = base_url / str(Path.r0.login) resp = await cls.az.http_session.post( url, data=json.dumps( diff --git a/mautrix/bridge/e2ee.py b/mautrix/bridge/e2ee.py index ea802c5c..dc213642 100644 --- a/mautrix/bridge/e2ee.py +++ b/mautrix/bridge/e2ee.py @@ -214,7 +214,6 @@ async def decrypt(self, evt: EncryptedEvent, wait_session_timeout: int = 5) -> M return decrypted async def start(self) -> None: - self.client.api.hacky_replace_v3_with_r0 = self.az.intent.api.hacky_replace_v3_with_r0 flows = await self.client.get_login_flows() flow = flows.get_first_of_type(LoginType.APPSERVICE, LoginType.UNSTABLE_APPSERVICE) if flow is None: diff --git a/mautrix/client/api/base.py b/mautrix/client/api/base.py index b913f08d..f1c93eff 100644 --- a/mautrix/client/api/base.py +++ b/mautrix/client/api/base.py @@ -118,10 +118,7 @@ async def versions(self, no_cache: bool = False) -> VersionsResponse: resp = await self.api.request(Method.GET, Path.versions) vers = self.versions_cache = VersionsResponse.deserialize(resp) if not vers.has_modern_versions and vers.has_legacy_versions: - self.log.warning( - "Server isn't advertising modern spec versions, falling back to /r0 endpoints" - ) - self.api.hacky_replace_v3_with_r0 = True + self.log.warning("Server isn't advertising modern spec versions") return self.versions_cache @classmethod From cd937544259c2544e6820714d553a01feeb98324 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 17 Apr 2022 13:11:41 +0300 Subject: [PATCH 075/456] Bump version to 0.16.1 --- CHANGELOG.md | 2 +- mautrix/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 47de1c1d..4b53db04 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -## v0.16.1 (unreleased) +## v0.16.1 (2022-04-17) * **Breaking change** Removed `r0` path support. * The new `v3` paths are implemented since Synapse 1.48, Dendrite 0.6.5, diff --git a/mautrix/__init__.py b/mautrix/__init__.py index d033fb13..dfc96351 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.16.0" +__version__ = "0.16.1" __author__ = "Tulir Asokan " __all__ = [ "api", From 357684cb2221a6b03fa41d94b2931cd108466d58 Mon Sep 17 00:00:00 2001 From: Nick Mills-Barrett Date: Wed, 20 Apr 2022 10:48:36 +0100 Subject: [PATCH 076/456] Add default `None` for `GlobalBridgeState.remote_states` field --- mautrix/util/bridge_state.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/util/bridge_state.py b/mautrix/util/bridge_state.py index 82df9c5b..22830320 100644 --- a/mautrix/util/bridge_state.py +++ b/mautrix/util/bridge_state.py @@ -135,5 +135,5 @@ async def send(self, url: str, token: str, log: logging.Logger, log_sent: bool = @dataclass(kw_only=True) class GlobalBridgeState(SerializableAttrs): - remote_states: Optional[Dict[str, BridgeState]] = field(json="remoteState") + remote_states: Optional[Dict[str, BridgeState]] = field(json="remoteState", default=None) bridge_state: BridgeState = field(json="bridgeState") From 726225dd86b3ab42e71a2c212189b9f8c85f0239 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 17 Apr 2022 13:11:41 +0300 Subject: [PATCH 077/456] Add utility method for setting thread parent --- mautrix/types/event/message.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/mautrix/types/event/message.py b/mautrix/types/event/message.py index bad471b8..dea84aef 100644 --- a/mautrix/types/event/message.py +++ b/mautrix/types/event/message.py @@ -101,6 +101,22 @@ def set_reply(self, reply_to: Union[EventID, "MessageEvent"], **kwargs) -> None: event_id=reply_to if isinstance(reply_to, str) else reply_to.event_id ) + def set_thread_parent( + self, + thread_parent: Union[EventID, "MessageEvent"], + reply_to: Union[EventID, "MessageEvent", None] = None, + **kwargs, + ) -> None: + self.relates_to.rel_type = RelationType.THREAD + self.relates_to.event_id = ( + thread_parent if isinstance(thread_parent, str) else thread_parent.event_id + ) + if reply_to is None: + self.set_reply(thread_parent, **kwargs) + self.relates_to.is_falling_back = True + else: + self.set_reply(reply_to, **kwargs) + def set_edit(self, edits: Union[EventID, "MessageEvent"]) -> None: self.relates_to.rel_type = RelationType.REPLACE self.relates_to.event_id = edits if isinstance(edits, str) else edits.event_id @@ -138,6 +154,11 @@ def get_edit(self) -> Optional[EventID]: return self._relates_to.event_id return None + def get_thread_parent(self) -> Optional[EventID]: + if self._relates_to and self._relates_to.rel_type == RelationType.THREAD: + return self._relates_to.event_id + return None + def trim_reply_fallback(self) -> None: pass From e11c3bdc85b38b3f76108d2feebfa272e2c59f76 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 21 Apr 2022 15:44:12 +0300 Subject: [PATCH 078/456] Increase timeout for sending message checkpoints --- mautrix/util/message_send_checkpoint.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/mautrix/util/message_send_checkpoint.py b/mautrix/util/message_send_checkpoint.py index eb83c6df..8d9a3333 100644 --- a/mautrix/util/message_send_checkpoint.py +++ b/mautrix/util/message_send_checkpoint.py @@ -60,22 +60,23 @@ async def send(self, endpoint: str, as_token: str, log: logging.Logger) -> None: endpoint, json={"checkpoints": [self.serialize()]}, headers=headers, - timeout=ClientTimeout(5), + timeout=ClientTimeout(30), ) as resp: if not 200 <= resp.status < 300: text = await resp.text() text = text.replace("\n", "\\n") log.warning( - f"Unexpected status code {resp.status} sending message send checkpoints " + f"Unexpected status code {resp.status} sending checkpoints " f"for {self.event_id}: {text}" ) else: log.info( - f"Successfully sent message send checkpoints for {self.event_id} " - f"(step: {self.step})" + f"Successfully sent checkpoint for {self.event_id} (step: {self.step})" ) except Exception as e: - log.warning(f"Failed to send message send checkpoints for {self.event_id}: {e}") + log.warning( + f"Failed to send checkpoint for {self.event_id}: " f"{type(e).__name__}: {e}" + ) CHECKPOINT_TYPES = { From 4f2f11faf16bab6b8380aceab3611b4c404d7aac Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 21 Apr 2022 15:47:00 +0300 Subject: [PATCH 079/456] Bump version to 0.16.2 --- CHANGELOG.md | 5 +++++ mautrix/__init__.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b53db04..ebdde79f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +## v0.16.2 (2022-04-21) + +* Added `get_thread_parent` and `set_thread_parent` helper methods for `MessageEventContent`. +* Increased timeout for `MessageSendCheckpoint.send`. + ## v0.16.1 (2022-04-17) * **Breaking change** Removed `r0` path support. diff --git a/mautrix/__init__.py b/mautrix/__init__.py index dfc96351..4345ea5b 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.16.1" +__version__ = "0.16.2" __author__ = "Tulir Asokan " __all__ = [ "api", From e1a4c2a3799f77e171bf5d890bc2087c466f096f Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 21 Apr 2022 15:59:58 +0300 Subject: [PATCH 080/456] Adjust set_thread_parent parameters --- CHANGELOG.md | 5 +++++ mautrix/__init__.py | 2 +- mautrix/types/event/message.py | 9 ++++----- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ebdde79f..6daff067 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +## v0.16.3 (2022-04-21) + +* Changed `set_thread_parent` to have an explicit option for disabling the + thread-as-reply fallback. + ## v0.16.2 (2022-04-21) * Added `get_thread_parent` and `set_thread_parent` helper methods for `MessageEventContent`. diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 4345ea5b..bab94ad7 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.16.2" +__version__ = "0.16.3" __author__ = "Tulir Asokan " __all__ = [ "api", diff --git a/mautrix/types/event/message.py b/mautrix/types/event/message.py index dea84aef..fd95a24a 100644 --- a/mautrix/types/event/message.py +++ b/mautrix/types/event/message.py @@ -104,18 +104,17 @@ def set_reply(self, reply_to: Union[EventID, "MessageEvent"], **kwargs) -> None: def set_thread_parent( self, thread_parent: Union[EventID, "MessageEvent"], - reply_to: Union[EventID, "MessageEvent", None] = None, + last_event_in_thread: Union[EventID, "MessageEvent", None] = None, + disable_reply_fallback: bool = False, **kwargs, ) -> None: self.relates_to.rel_type = RelationType.THREAD self.relates_to.event_id = ( thread_parent if isinstance(thread_parent, str) else thread_parent.event_id ) - if reply_to is None: - self.set_reply(thread_parent, **kwargs) + if not disable_reply_fallback: + self.set_reply(last_event_in_thread or thread_parent, **kwargs) self.relates_to.is_falling_back = True - else: - self.set_reply(reply_to, **kwargs) def set_edit(self, edits: Union[EventID, "MessageEvent"]) -> None: self.relates_to.rel_type = RelationType.REPLACE From a8cdaa758ce1b04d733965f816ac89dda2fed970 Mon Sep 17 00:00:00 2001 From: Alejandro Herrera Date: Mon, 25 Apr 2022 14:36:27 -0500 Subject: [PATCH 081/456] Pytest warnigs enhancements are added --- pytest.ini | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 pytest.ini diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 00000000..d280de04 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +asyncio_mode = auto \ No newline at end of file From 8f0f1b246cbc8307a24597995eafccaea52d36ac Mon Sep 17 00:00:00 2001 From: Alejandro Herrera Date: Mon, 25 Apr 2022 14:50:23 -0500 Subject: [PATCH 082/456] Pytest warnigs enhancements are added --- pytest.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytest.ini b/pytest.ini index d280de04..2f4c80e3 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,2 +1,2 @@ [pytest] -asyncio_mode = auto \ No newline at end of file +asyncio_mode = auto From 50b5a9f17505d5c458c188a643782dac01c86a1a Mon Sep 17 00:00:00 2001 From: Alejandro Herrera Date: Tue, 26 Apr 2022 07:51:48 -0500 Subject: [PATCH 083/456] Added pytest configuration in pyproject.toml --- pyproject.toml | 3 +++ pytest.ini | 2 -- 2 files changed, 3 insertions(+), 2 deletions(-) delete mode 100644 pytest.ini diff --git a/pyproject.toml b/pyproject.toml index 134042b1..c0e41313 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,3 +8,6 @@ line_length = 99 [tool.black] line-length = 99 target-version = ["py38"] + +[tool.pytest.ini_options] +asyncio_mode = "auto" diff --git a/pytest.ini b/pytest.ini deleted file mode 100644 index 2f4c80e3..00000000 --- a/pytest.ini +++ /dev/null @@ -1,2 +0,0 @@ -[pytest] -asyncio_mode = auto From e295a278a1b932ecef3074ea300a720c2c51f4cb Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 29 Apr 2022 22:07:43 +0300 Subject: [PATCH 084/456] Drop support for appservice login with unstable prefix --- mautrix/bridge/e2ee.py | 10 ++++------ mautrix/types/auth.py | 1 - 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/mautrix/bridge/e2ee.py b/mautrix/bridge/e2ee.py index dc213642..20cb9895 100644 --- a/mautrix/bridge/e2ee.py +++ b/mautrix/bridge/e2ee.py @@ -215,14 +215,12 @@ async def decrypt(self, evt: EncryptedEvent, wait_session_timeout: int = 5) -> M async def start(self) -> None: flows = await self.client.get_login_flows() - flow = flows.get_first_of_type(LoginType.APPSERVICE, LoginType.UNSTABLE_APPSERVICE) - if flow is None: + if not flows.supports_type(LoginType.APPSERVICE): self.log.critical( - "Encryption enabled in config, but homeserver does not " - "advertise appservice login" + "Encryption enabled in config, but homeserver does not advertise appservice login" ) sys.exit(30) - self.log.debug(f"Logging in with bridge bot user (using login type {flow.type.value})") + self.log.debug("Logging in with bridge bot user") if self.crypto_db: try: await self.crypto_db.start() @@ -236,7 +234,7 @@ async def start(self) -> None: # It'll get overridden after the login self.client.api.token = self.az.as_token await self.client.login( - login_type=flow.type, + login_type=LoginType.APPSERVICE, device_name=self.device_name, device_id=device_id, store_access_token=True, diff --git a/mautrix/types/auth.py b/mautrix/types/auth.py index 3fd1b2d3..198d2f53 100644 --- a/mautrix/types/auth.py +++ b/mautrix/types/auth.py @@ -25,7 +25,6 @@ class LoginType(ExtensibleEnum): APPSERVICE: "LoginType" = "m.login.application_service" UNSTABLE_JWT: "LoginType" = "org.matrix.login.jwt" - UNSTABLE_APPSERVICE: "LoginType" = "uk.half-shot.msc2778.login.application_service" @dataclass From c3d0f156a92e209d9266d48a529e604199d69af9 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 30 Apr 2022 21:24:30 +0300 Subject: [PATCH 085/456] Fix async db execute type hint --- mautrix/util/async_db/__init__.py | 4 ++++ mautrix/util/async_db/connection.py | 7 +++++-- mautrix/util/async_db/database.py | 8 +++++--- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/mautrix/util/async_db/__init__.py b/mautrix/util/async_db/__init__.py index 913725a2..500ffe0e 100644 --- a/mautrix/util/async_db/__init__.py +++ b/mautrix/util/async_db/__init__.py @@ -19,11 +19,14 @@ PostgresDatabase = None try: + from aiosqlite import Cursor as SQLiteCursor + from .aiosqlite import SQLiteDatabase except ImportError: if __optional_imports__: raise SQLiteDatabase = None + SQLiteCursor = None __all__ = [ "Database", @@ -31,6 +34,7 @@ "register_upgrade", "PostgresDatabase", "SQLiteDatabase", + "SQLiteCursor", "Connection", "Scheme", "DatabaseException", diff --git a/mautrix/util/async_db/connection.py b/mautrix/util/async_db/connection.py index 4dd4980e..f8677207 100644 --- a/mautrix/util/async_db/connection.py +++ b/mautrix/util/async_db/connection.py @@ -19,6 +19,7 @@ if __optional_imports__: from sqlite3 import Row + from aiosqlite import Cursor from asyncpg import Record import asyncpg @@ -69,11 +70,13 @@ async def transaction(self) -> None: yield @log_duration - async def execute(self, query: str, *args: Any, timeout: float | None = None) -> str: + async def execute(self, query: str, *args: Any, timeout: float | None = None) -> str | Cursor: return await self.wrapped.execute(query, *args, timeout=timeout) @log_duration - async def executemany(self, query: str, *args: Any, timeout: float | None = None) -> str: + async def executemany( + self, query: str, *args: Any, timeout: float | None = None + ) -> str | Cursor: return await self.wrapped.executemany(query, *args, timeout=timeout) @log_duration diff --git a/mautrix/util/async_db/database.py b/mautrix/util/async_db/database.py index d6e26eed..a04392df 100644 --- a/mautrix/util/async_db/database.py +++ b/mautrix/util/async_db/database.py @@ -8,7 +8,6 @@ from typing import Any, AsyncContextManager, Type from abc import ABC, abstractmethod import logging -import sys from yarl import URL @@ -21,6 +20,7 @@ from .upgrade import UpgradeTable, upgrade_tables if __optional_imports__: + from aiosqlite import Cursor from asyncpg import Record @@ -131,11 +131,13 @@ async def stop(self) -> None: def acquire(self) -> AsyncContextManager[LoggingConnection]: pass - async def execute(self, query: str, *args: Any, timeout: float | None = None) -> str: + async def execute(self, query: str, *args: Any, timeout: float | None = None) -> str | Cursor: async with self.acquire() as conn: return await conn.execute(query, *args, timeout=timeout) - async def executemany(self, query: str, *args: Any, timeout: float | None = None) -> str: + async def executemany( + self, query: str, *args: Any, timeout: float | None = None + ) -> str | Cursor: async with self.acquire() as conn: return await conn.executemany(query, *args, timeout=timeout) From 93bd52893ab5da1cfe3297d0789b743e28f06d47 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 6 May 2022 15:24:52 +0300 Subject: [PATCH 086/456] Change wording in warning log --- mautrix/bridge/e2ee.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/bridge/e2ee.py b/mautrix/bridge/e2ee.py index 20cb9895..4a6647fb 100644 --- a/mautrix/bridge/e2ee.py +++ b/mautrix/bridge/e2ee.py @@ -217,7 +217,7 @@ async def start(self) -> None: flows = await self.client.get_login_flows() if not flows.supports_type(LoginType.APPSERVICE): self.log.critical( - "Encryption enabled in config, but homeserver does not advertise appservice login" + "Encryption enabled in config, but homeserver does not support appservice login" ) sys.exit(30) self.log.debug("Logging in with bridge bot user") From 9459577d2d7cd01242ce7d5d60dd4dd724974512 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 7 May 2022 19:53:12 +0300 Subject: [PATCH 087/456] Don't try to stop asyncpg connection if it wasn't started --- mautrix/util/async_db/asyncpg.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mautrix/util/async_db/asyncpg.py b/mautrix/util/async_db/asyncpg.py index 40def15e..28d3dcab 100644 --- a/mautrix/util/async_db/asyncpg.py +++ b/mautrix/util/async_db/asyncpg.py @@ -69,8 +69,8 @@ def pool(self) -> asyncpg.pool.Pool: return self._pool async def stop(self) -> None: - if not self._pool_override: - await self.pool.close() + if not self._pool_override and self._pool is not None: + await self._pool.close() @asynccontextmanager async def acquire(self) -> LoggingConnection: From 9a2752c05fc7ba62112eaed5108b29f3e1fdc594 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 7 May 2022 20:01:52 +0300 Subject: [PATCH 088/456] Log all errors starting the database --- mautrix/bridge/bridge.py | 6 +++--- mautrix/bridge/e2ee.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mautrix/bridge/bridge.py b/mautrix/bridge/bridge.py index 9ca56a19..b09b0d11 100644 --- a/mautrix/bridge/bridge.py +++ b/mautrix/bridge/bridge.py @@ -176,9 +176,9 @@ def prepare_db(self) -> None: def prepare_bridge(self) -> None: self.matrix = self.matrix_class(bridge=self) - def _log_db_error(self, e: DatabaseException) -> None: + def _log_db_error(self, e: Exception) -> None: self.log.critical("Failed to initialize database", exc_info=e) - if e.explanation: + if isinstance(e, DatabaseException) and e.explanation: self.log.info(e.explanation) sys.exit(25) @@ -195,7 +195,7 @@ async def start_db(self) -> None: if self.matrix.e2ee: self.matrix.e2ee.crypto_db.allow_unsupported = ignore_unsupported self.matrix.e2ee.crypto_db.override_pool(self.db) - except DatabaseException as e: + except Exception as e: self._log_db_error(e) async def stop_db(self) -> None: diff --git a/mautrix/bridge/e2ee.py b/mautrix/bridge/e2ee.py index 4a6647fb..2586398e 100644 --- a/mautrix/bridge/e2ee.py +++ b/mautrix/bridge/e2ee.py @@ -224,7 +224,7 @@ async def start(self) -> None: if self.crypto_db: try: await self.crypto_db.start() - except DatabaseException as e: + except Exception as e: self.bridge._log_db_error(e) await self.crypto_store.open() device_id = await self.crypto_store.get_device_id() From 702e605b5be273a9c48d25aca133031e7e1ca444 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 10 May 2022 17:19:09 +0300 Subject: [PATCH 089/456] Add helper method to redact bridge command --- mautrix/bridge/commands/handler.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/mautrix/bridge/commands/handler.py b/mautrix/bridge/commands/handler.py index d636e673..be0fbd17 100644 --- a/mautrix/bridge/commands/handler.py +++ b/mautrix/bridge/commands/handler.py @@ -12,6 +12,7 @@ import traceback from mautrix.appservice import AppService, IntentAPI +from mautrix.errors import MForbidden from mautrix.types import EventID, MessageEventContent, RoomID from mautrix.util import markdown from mautrix.util.logging import TraceLogger @@ -144,6 +145,20 @@ def print_error_traceback(self) -> bool: def main_intent(self) -> IntentAPI: return self.portal.main_intent if self.portal else self.az.intent + async def redact(self) -> None: + """ + Try to redact the command. + + If the redaction fails with M_FORBIDDEN, the error will be logged and ignored. + """ + try: + if self.has_bridge_bot: + await self.az.intent.redact(self.room_id, self.event_id) + else: + await self.main_intent.redact(self.room_id, self.event_id) + except MForbidden as e: + self.log.warning(f"Failed to redact command {self.command}: {e}") + def reply( self, message: str, allow_html: bool = False, render_markdown: bool = True ) -> Awaitable[EventID]: From 4ba953d7180bfc0b2d8b822c0701111796b9d6df Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 10 May 2022 17:21:10 +0300 Subject: [PATCH 090/456] Bump version to 0.16.4 --- CHANGELOG.md | 6 ++++++ mautrix/__init__.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6daff067..034bf43f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## v0.16.4 (2022-05-10) + +* Dropped support for appservice login with unstable prefix. +* Fixed some database start errors causing unnecessary noise in logs. +* Added helper method to redact bridge commands. + ## v0.16.3 (2022-04-21) * Changed `set_thread_parent` to have an explicit option for disabling the diff --git a/mautrix/__init__.py b/mautrix/__init__.py index bab94ad7..6ddb7ca1 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.16.3" +__version__ = "0.16.4" __author__ = "Tulir Asokan " __all__ = [ "api", From 224132430b432a599bff739fbc9ce927844d9680 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 13 May 2022 16:29:29 +0300 Subject: [PATCH 091/456] Add reason field for CommandEvent.redact --- mautrix/bridge/commands/handler.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mautrix/bridge/commands/handler.py b/mautrix/bridge/commands/handler.py index be0fbd17..fe461937 100644 --- a/mautrix/bridge/commands/handler.py +++ b/mautrix/bridge/commands/handler.py @@ -145,7 +145,7 @@ def print_error_traceback(self) -> bool: def main_intent(self) -> IntentAPI: return self.portal.main_intent if self.portal else self.az.intent - async def redact(self) -> None: + async def redact(self, reason: str | None = None) -> None: """ Try to redact the command. @@ -153,9 +153,9 @@ async def redact(self) -> None: """ try: if self.has_bridge_bot: - await self.az.intent.redact(self.room_id, self.event_id) + await self.az.intent.redact(self.room_id, self.event_id, reason=reason) else: - await self.main_intent.redact(self.room_id, self.event_id) + await self.main_intent.redact(self.room_id, self.event_id, reason=reason) except MForbidden as e: self.log.warning(f"Failed to redact command {self.command}: {e}") From ee8305aae0da64980a49bf91bcd83a95a44dbce6 Mon Sep 17 00:00:00 2001 From: Malte E Date: Tue, 17 May 2022 21:27:52 +0200 Subject: [PATCH 092/456] allow providing reasons with unbans --- mautrix/client/api/rooms.py | 8 ++++++-- mautrix/client/store_updater.py | 4 ++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/mautrix/client/api/rooms.py b/mautrix/client/api/rooms.py index b8a9e6fe..700bf876 100644 --- a/mautrix/client/api/rooms.py +++ b/mautrix/client/api/rooms.py @@ -581,7 +581,7 @@ async def ban_user( Method.POST, Path.v3.rooms[room_id].ban, {"user_id": user_id, "reason": reason} ) - async def unban_user(self, room_id: RoomID, user_id: UserID) -> None: + async def unban_user(self, room_id: RoomID, user_id: UserID, reason: str = "") -> None: """ Unban a user from the room. This allows them to be invited to the room, and join if they would otherwise be allowed to join according to its join rules. The caller must have the @@ -592,8 +592,12 @@ async def unban_user(self, room_id: RoomID, user_id: UserID) -> None: Args: room_id: The ID of the room from which the user should be unbanned. user_id: The fully qualified user ID of the user being banned. + reason: The reason the user has been unbanned. This will be supplied as the ``reason`` on + the target's updated `m.room.member`_ event. """ - await self.api.request(Method.POST, Path.v3.rooms[room_id].unban, {"user_id": user_id}) + await self.api.request( + Method.POST, Path.v3.rooms[room_id].unban, {"user_id": user_id, "reason": reason} + ) # endregion diff --git a/mautrix/client/store_updater.py b/mautrix/client/store_updater.py index 8b197e64..55d649c3 100644 --- a/mautrix/client/store_updater.py +++ b/mautrix/client/store_updater.py @@ -110,8 +110,8 @@ async def ban_user( if not extra_content and self.state_store: await self.state_store.set_membership(room_id, user_id, Membership.BAN) - async def unban_user(self, room_id: RoomID, user_id: UserID) -> None: - await super().unban_user(room_id, user_id) + async def unban_user(self, room_id: RoomID, user_id: UserID, reason: str = "") -> None: + await super().unban_user(room_id, user_id, reason=reason) if self.state_store: await self.state_store.set_membership(room_id, user_id, Membership.LEAVE) From 9e766611a1046a9d7fe2fecf22fd61355a303d2e Mon Sep 17 00:00:00 2001 From: Sumner Evans Date: Tue, 12 Apr 2022 07:43:35 -0600 Subject: [PATCH 093/456] batch send: use classes without unnecessary/unknown fields The batch send endpoint now accepts events and state events that don't have room_id, event_id, etc. All batch send classes allow for the specification of the timestamp. --- mautrix/appservice/api/intent.py | 10 +++++----- mautrix/types/__init__.py | 4 ++++ mautrix/types/event/__init__.py | 1 + mautrix/types/event/batch.py | 32 ++++++++++++++++++++++++++++++++ mautrix/types/event/message.py | 4 ++-- 5 files changed, 44 insertions(+), 7 deletions(-) create mode 100644 mautrix/types/event/batch.py diff --git a/mautrix/appservice/api/intent.py b/mautrix/appservice/api/intent.py index f9c991b7..be8d87ff 100644 --- a/mautrix/appservice/api/intent.py +++ b/mautrix/appservice/api/intent.py @@ -21,7 +21,9 @@ from mautrix.types import ( JSON, BatchID, + BatchSendEvent, BatchSendResponse, + BatchSendStateEvent, ContentURI, EventContent, EventID, @@ -30,7 +32,6 @@ JoinRulesStateEventContent, Member, Membership, - MessageEvent, PowerLevelStateEventContent, PresenceState, RoomAvatarStateEventContent, @@ -38,7 +39,6 @@ RoomNameStateEventContent, RoomPinnedEventsStateEventContent, RoomTopicStateEventContent, - StateEvent, StateEventContent, UserID, ) @@ -435,8 +435,8 @@ async def batch_send( prev_event_id: EventID, *, batch_id: BatchID | None = None, - events: Iterable[MessageEvent], - state_events_at_start: Iterable[StateEvent] = None, + events: Iterable[BatchSendEvent], + state_events_at_start: Iterable[BatchSendStateEvent] = (), ) -> BatchSendResponse: """ Send a batch of historical events into a room. See `MSC2716`_ for more info. @@ -459,7 +459,7 @@ async def batch_send( All the event IDs generated, plus a batch ID that can be passed back to this method. """ path = Path.unstable["org.matrix.msc2716"].rooms[room_id].batch_send - query = {"prev_event_id": prev_event_id} + query: JSON = {"prev_event_id": prev_event_id} if batch_id: query["batch_id"] = batch_id resp = await self.api.request( diff --git a/mautrix/types/__init__.py b/mautrix/types/__init__.py index a53e965c..6040c525 100644 --- a/mautrix/types/__init__.py +++ b/mautrix/types/__init__.py @@ -34,6 +34,8 @@ BaseMessageEventContentFuncs, BaseRoomEvent, BaseUnsigned, + BatchSendEvent, + BatchSendStateEvent, CallAnswerEventContent, CallCandidate, CallCandidatesEventContent, @@ -214,6 +216,8 @@ "BaseMessageEventContentFuncs", "BaseRoomEvent", "BaseUnsigned", + "BatchSendEvent", + "BatchSendStateEvent", "CallAnswerEventContent", "CallCandidate", "CallCandidatesEventContent", diff --git a/mautrix/types/event/__init__.py b/mautrix/types/event/__init__.py index a81358b0..2d7e889c 100644 --- a/mautrix/types/event/__init__.py +++ b/mautrix/types/event/__init__.py @@ -10,6 +10,7 @@ RoomTagInfo, ) from .base import BaseEvent, BaseRoomEvent, BaseUnsigned, GenericEvent +from .batch import BatchSendEvent, BatchSendStateEvent from .encrypted import ( EncryptedEvent, EncryptedEventContent, diff --git a/mautrix/types/event/batch.py b/mautrix/types/event/batch.py new file mode 100644 index 00000000..df45b742 --- /dev/null +++ b/mautrix/types/event/batch.py @@ -0,0 +1,32 @@ +# Copyright (c) 2022 Tulir Asokan, Sumner Evans +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +from typing import Any + +from attr import dataclass +import attr + +from ..primitive import UserID +from ..util import SerializableAttrs +from .base import BaseEvent + + +@dataclass +class BatchSendEvent(BaseEvent, SerializableAttrs): + """Base event class for events sent via a batch send request.""" + + sender: UserID + timestamp: int = attr.ib(metadata={"json": "origin_server_ts"}) + content: Any + + +@dataclass +class BatchSendStateEvent(BatchSendEvent, SerializableAttrs): + """ + State events to be used as initial state events on batch send events. These never need to be + deserialized. + """ + + state_key: str diff --git a/mautrix/types/event/message.py b/mautrix/types/event/message.py index fd95a24a..45c34ce6 100644 --- a/mautrix/types/event/message.py +++ b/mautrix/types/event/message.py @@ -3,7 +3,7 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from typing import Any, Dict, List, Optional, Pattern, Union +from typing import Dict, List, Optional, Pattern, Union from html import escape import re @@ -11,7 +11,7 @@ import attr from ..primitive import JSON, ContentURI, EventID -from ..util import ExtensibleEnum, Obj, Serializable, SerializableAttrs, deserializer, field +from ..util import ExtensibleEnum, Obj, SerializableAttrs, deserializer, field from .base import BaseRoomEvent, BaseUnsigned # region Message types From ece8e8ce648f551bd12b2cc5b4eb7c5fb455c1cd Mon Sep 17 00:00:00 2001 From: Malte E Date: Sun, 22 May 2022 21:41:02 +0200 Subject: [PATCH 094/456] always create group chat portal if is_direct==False --- mautrix/bridge/matrix.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/bridge/matrix.py b/mautrix/bridge/matrix.py index bdfd922f..16e8593e 100644 --- a/mautrix/bridge/matrix.py +++ b/mautrix/bridge/matrix.py @@ -298,7 +298,7 @@ async def handle_puppet_nonportal_invite( return if create_evt.type == RoomType.SPACE: await self.handle_puppet_space_invite(room_id, puppet, invited_by, evt) - elif len(members) > 2: + elif len(members) > 2 or not evt.content.is_direct: await self.handle_puppet_group_invite(room_id, puppet, invited_by, evt, members) else: await self.handle_puppet_dm_invite(room_id, puppet, invited_by, evt) From a8f188fd05665096793c0c40d4d4d29c54403628 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 25 May 2022 13:23:48 +0300 Subject: [PATCH 095/456] Use new_event_loop + set instead of get_event_loop --- mautrix/util/program.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mautrix/util/program.py b/mautrix/util/program.py index 0d4883ab..172d36b2 100644 --- a/mautrix/util/program.py +++ b/mautrix/util/program.py @@ -192,7 +192,8 @@ def prepare_loop(self) -> None: uvloop.install() self.log.debug("Using uvloop for asyncio") - self.loop = asyncio.get_event_loop() + self.loop = asyncio.new_event_loop() + asyncio.set_event_loop(self.loop) def start_prometheus(self) -> None: try: @@ -237,6 +238,8 @@ def _run(self) -> None: self.prepare_stop() self.loop.run_until_complete(self.stop()) self.prepare_shutdown() + self.loop.close() + asyncio.set_event_loop(None) self.log.info("Everything stopped, shutting down") sys.exit(0) From c86b84ef926bf73cb51cfa61a373206ebf817616 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 25 May 2022 13:23:57 +0300 Subject: [PATCH 096/456] Change table to use to detect sharing Dendrite database They seem to be moving away from goose --- mautrix/util/async_db/database.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mautrix/util/async_db/database.py b/mautrix/util/async_db/database.py index a04392df..5ed0c7f1 100644 --- a/mautrix/util/async_db/database.py +++ b/mautrix/util/async_db/database.py @@ -107,8 +107,8 @@ async def start(self) -> None: async def _check_foreign_tables(self) -> None: if await self.table_exists("state_groups_state"): raise ForeignTablesFound("found state_groups_state likely belonging to Synapse") - elif await self.table_exists("goose_db_version"): - raise ForeignTablesFound("found goose_db_version possibly belonging to Dendrite") + elif await self.table_exists("roomserver_rooms"): + raise ForeignTablesFound("found roomserver_rooms possibly belonging to Dendrite") async def _check_owner(self) -> None: await self.execute( From 3352319db1081458afea7e78790327cbfdf923ce Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 25 May 2022 14:00:10 +0300 Subject: [PATCH 097/456] Update changelog --- CHANGELOG.md | 39 ++++++++++++++++++++++++++++++--------- 1 file changed, 30 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 034bf43f..6067f890 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,18 +1,37 @@ +## v0.16.5 (unreleased) + +* *(bridge.commands)* Added `reason` field for `CommandEvent.redact`. +* *(client.api)* Added `reason` field for the `unban_user` method + (thanks to [@maltee1] in [#101]). +* *(bridge)* Changed automatic DM portal creation to only apply when the invite + event specifies `"is_direct": true` (thanks to [@maltee1] in [#102]). +* *(util.program)* Changed `Program` to use create and set an event loop + explicitly instead of using `get_event_loop`. +* *(appservice.api)* Switched `IntentAPI.batch_send` method to use custom Event + classes instead of the default ones (since some normal event fields aren't + applicable when batch sending). + +[@maltee1]: https://github.com/maltee1 +[#101]: https://github.com/mautrix/python/pull/101 +[#102]: https://github.com/mautrix/python/pull/102 + ## v0.16.4 (2022-05-10) -* Dropped support for appservice login with unstable prefix. -* Fixed some database start errors causing unnecessary noise in logs. -* Added helper method to redact bridge commands. +* *(types, bridge)* Dropped support for appservice login with unstable prefix. +* *(util.async_db)* Fixed some database start errors causing unnecessary noise + in logs. +* *(bridge.commands)* Added helper method to redact bridge commands. ## v0.16.3 (2022-04-21) -* Changed `set_thread_parent` to have an explicit option for disabling the - thread-as-reply fallback. +* *(types)* Changed `set_thread_parent` to have an explicit option for + disabling the thread-as-reply fallback. ## v0.16.2 (2022-04-21) -* Added `get_thread_parent` and `set_thread_parent` helper methods for `MessageEventContent`. -* Increased timeout for `MessageSendCheckpoint.send`. +* *(types)* Added `get_thread_parent` and `set_thread_parent` helper methods + for `MessageEventContent`. +* *(bridge)* Increased timeout for `MessageSendCheckpoint.send`. ## v0.16.1 (2022-04-17) @@ -272,12 +291,14 @@ ## v0.12.5 (2021-11-30) -* Added wrapper for [MSC2716]'s `/batch_send` endpoint in `IntentAPI` -* Added some Matrix request metrics (thanks to @jaller94 in #68) +* Added wrapper for [MSC2716]'s `/batch_send` endpoint in `IntentAPI`. +* Added some Matrix request metrics (thanks to [@jaller94] in [#68]). * Added utility method for adding variation selector 16 to emoji strings the same way as Element does (using emojibase data). [MSC2716]: https://github.com/matrix-org/matrix-spec-proposals/pull/2716 +[@jaller94]: https://github.com/jaller94 +[#68]: https://github.com/mautrix/python/pull/68 ## v0.12.4 (2021-11-25) From f84fc20e2af75631960bbb2c65354ad696337bff Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 26 May 2022 16:29:48 +0300 Subject: [PATCH 098/456] Let programs set exit code in manual_stop --- mautrix/util/program.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/mautrix/util/program.py b/mautrix/util/program.py index 172d36b2..80d2298b 100644 --- a/mautrix/util/program.py +++ b/mautrix/util/program.py @@ -214,6 +214,7 @@ def _run(self) -> None: signal.signal(signal.SIGINT, signal.default_int_handler) signal.signal(signal.SIGTERM, signal.default_int_handler) + exit_code = 0 try: self.log.debug("Running startup actions...") start_ts = time() @@ -224,7 +225,7 @@ def _run(self) -> None: "now running forever" ) self._stop_task = self.loop.create_future() - self.loop.run_until_complete(self._stop_task) + exit_code = self.loop.run_until_complete(self._stop_task) self.log.debug("manual_stop() called, stopping...") except KeyboardInterrupt: self.log.debug("Interrupt received, stopping...") @@ -241,7 +242,7 @@ def _run(self) -> None: self.loop.close() asyncio.set_event_loop(None) self.log.info("Everything stopped, shutting down") - sys.exit(0) + sys.exit(exit_code) async def system_exit(self) -> None: """Lifecycle method that is called if the main event loop exits using ``sys.exit()``.""" @@ -271,9 +272,9 @@ async def stop(self) -> None: def prepare_shutdown(self) -> None: """Lifecycle method that is called right before ``sys.exit(0)``.""" - def manual_stop(self) -> None: + def manual_stop(self, exit_code: int = 0) -> None: """Tell the event loop to cleanly stop and run the stop lifecycle steps.""" - self._stop_task.set_result(None) + self._stop_task.set_result(exit_code) def add_startup_actions(self, *actions: NewTask) -> None: self.startup_actions = self._add_actions(self.startup_actions, actions) From db54610def352247810412f23f209db52a03b6f0 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 26 May 2022 17:32:41 +0300 Subject: [PATCH 099/456] Remove loop parameters in manhole util --- CHANGELOG.md | 3 +++ mautrix/util/manhole.py | 23 +++++------------------ 2 files changed, 8 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6067f890..7b963b91 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,9 @@ event specifies `"is_direct": true` (thanks to [@maltee1] in [#102]). * *(util.program)* Changed `Program` to use create and set an event loop explicitly instead of using `get_event_loop`. +* *(util.program)* Added optional `exit_code` parameter to `manual_stop`. +* *(util.manhole)* Removed usage of loop parameters to fix Python 3.10 + compatibility. * *(appservice.api)* Switched `IntentAPI.batch_send` method to use custom Event classes instead of the default ones (since some normal event fields aren't applicable when batch sending). diff --git a/mautrix/util/manhole.py b/mautrix/util/manhole.py index f6967336..727ae36d 100644 --- a/mautrix/util/manhole.py +++ b/mautrix/util/manhole.py @@ -104,9 +104,7 @@ def reset(self) -> None: class Interpreter(ABC): @abstractmethod - def __init__( - self, namespace: Dict[str, Any], banner: Union[bytes, str], loop: asyncio.AbstractEventLoop - ) -> None: + def __init__(self, namespace: Dict[str, Any], banner: Union[bytes, str]) -> None: pass @abstractmethod @@ -126,17 +124,13 @@ class AsyncInterpreter(Interpreter): namespace: Dict[str, Any] banner: bytes compiler: StatefulCommandCompiler - loop: asyncio.AbstractEventLoop running: bool - def __init__( - self, namespace: Dict[str, Any], banner: Union[bytes, str], loop: asyncio.AbstractEventLoop - ) -> None: - super().__init__(namespace, banner, loop) + def __init__(self, namespace: Dict[str, Any], banner: Union[bytes, str]) -> None: + super().__init__(namespace, banner) self.namespace = namespace self.banner = banner if isinstance(banner, bytes) else str(banner).encode("utf-8") self.compiler = StatefulCommandCompiler() - self.loop = loop async def send_exception(self) -> None: """When an exception has occurred, write the traceback to the user.""" @@ -261,7 +255,6 @@ async def __call__(self, reader: asyncio.StreamReader, writer: asyncio.StreamWri class InterpreterFactory: namespace: Dict[str, Any] banner: bytes - loop: asyncio.AbstractEventLoop interpreter_class: Type[Interpreter] clients: List[Interpreter] whitelist: Set[int] @@ -272,12 +265,10 @@ def __init__( namespace: Dict[str, Any], banner: Union[bytes, str], interpreter_class: Type[Interpreter], - loop: asyncio.AbstractEventLoop, whitelist: Set[int], ) -> None: self.namespace = namespace or {} self.banner = banner - self.loop = loop self.interpreter_class = interpreter_class self.clients = [] self.whitelist = whitelist @@ -304,9 +295,7 @@ async def __call__(self, reader: asyncio.StreamReader, writer: asyncio.StreamWri return namespace = {**self.namespace} - interpreter = self.interpreter_class( - namespace=namespace, banner=self.banner, loop=self.loop - ) + interpreter = self.interpreter_class(namespace=namespace, banner=self.banner) namespace["exit"] = interpreter.close self.clients.append(interpreter) conn_id = self.conn_id @@ -336,15 +325,13 @@ async def start_manhole( """ if not SO_PEERCRED: raise ValueError("SO_PEERCRED is not supported on this platform") - loop = loop or asyncio.get_event_loop() factory = InterpreterFactory( namespace=namespace, banner=banner, interpreter_class=AsyncInterpreter, - loop=loop, whitelist=whitelist, ) - server = await asyncio.start_unix_server(factory, path=path, loop=loop) + server = await asyncio.start_unix_server(factory, path=path) os.chmod(path, 0o666) def stop(): From d180603445bb0bc465a7b2ff918c4ac28a5dbfc2 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 26 May 2022 17:35:05 +0300 Subject: [PATCH 100/456] Bump version to 0.16.5 --- CHANGELOG.md | 2 +- mautrix/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b963b91..1b781e03 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -## v0.16.5 (unreleased) +## v0.16.5 (2022-05-26) * *(bridge.commands)* Added `reason` field for `CommandEvent.redact`. * *(client.api)* Added `reason` field for the `unban_user` method diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 6ddb7ca1..3c8a6b9e 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.16.4" +__version__ = "0.16.5" __author__ = "Tulir Asokan " __all__ = [ "api", From 7db926b398355b26c1c3ca7d2c1beb816f0a556c Mon Sep 17 00:00:00 2001 From: Malte E Date: Fri, 27 May 2022 09:37:15 +0200 Subject: [PATCH 101/456] add support for knocks --- mautrix/bridge/matrix.py | 47 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/mautrix/bridge/matrix.py b/mautrix/bridge/matrix.py index 16e8593e..5110062f 100644 --- a/mautrix/bridge/matrix.py +++ b/mautrix/bridge/matrix.py @@ -243,6 +243,24 @@ async def handle_unban( async def handle_join(self, room_id: RoomID, user_id: UserID, event_id: EventID) -> None: pass + async def handle_knock(self, room_id: RoomID, user_id: UserID, event_id: EventID) -> None: + pass + + async def handle_retract_knock( + self, room_id: RoomID, user_id: UserID, event_id: EventID + ) -> None: + pass + + async def handle_reject_knock( + self, room_id: RoomID, user_id: UserID, event_id: EventID + ) -> None: + pass + + async def handle_accept_knock( + self, room_id: RoomID, user_id: UserID, event_id: EventID + ) -> None: + pass + async def handle_member_info_change( self, room_id: RoomID, @@ -817,7 +835,12 @@ async def int_handle_event(self, evt: Event, send_bridge_checkpoint: bool = True prev_content = unsigned.prev_content or MemberStateEventContent() prev_membership = prev_content.membership if prev_content else Membership.JOIN if evt.content.membership == Membership.INVITE: - await self.int_handle_invite(evt) + if prev_membership == Membership.KNOCK: + await self.handle_accept_knock( + evt.room_id, UserID(evt.state_key), evt.event_id + ) + else: + await self.int_handle_invite(evt) elif evt.content.membership == Membership.LEAVE: if prev_membership == Membership.BAN: await self.handle_unban( @@ -840,6 +863,20 @@ async def int_handle_event(self, evt: Event, send_bridge_checkpoint: bool = True evt.content.reason, evt.event_id, ) + elif prev_membership == Membership.KNOCK: + if evt.sender == evt.state_key: + await self.handle_retract_knock( + evt.room_id, UserID(evt.state_key), evt.content.reason, evt.event_id + ) + else: + await self.handle_reject_knock( + evt.room_id, + UserID(evt.state_key), + evt.sender, + evt.content.reason, + evt.event_id, + ) + elif evt.sender == evt.state_key: await self.handle_leave(evt.room_id, UserID(evt.state_key), evt.event_id) else: @@ -865,6 +902,14 @@ async def int_handle_event(self, evt: Event, send_bridge_checkpoint: bool = True await self.handle_member_info_change( evt.room_id, UserID(evt.state_key), evt.content, prev_content, evt.event_id ) + elif evt.content.membership == Membership.KNOCK: + await self.handle_knock( + evt.room_id, + UserID(evt.state_key), + evt.sender, + evt.content.reason, + evt.event_id, + ) elif evt.type in (EventType.ROOM_MESSAGE, EventType.STICKER): evt: MessageEvent if evt.type != EventType.ROOM_MESSAGE: From f7ae55778c6061e1b1f6dd78fbcebc122e3cff98 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 30 May 2022 11:35:56 +0300 Subject: [PATCH 102/456] Fix error handling double puppeting errors --- mautrix/bridge/custom_puppet.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mautrix/bridge/custom_puppet.py b/mautrix/bridge/custom_puppet.py index c5c77f25..77e1f1d8 100644 --- a/mautrix/bridge/custom_puppet.py +++ b/mautrix/bridge/custom_puppet.py @@ -273,12 +273,13 @@ async def start(self, retry_auto_login: bool = False, start_sync_task: bool = Tr if not whoami or whoami.user_id != self.custom_mxid: if self.custom_mxid and self.by_custom_mxid.get(self.custom_mxid) == self: del self.by_custom_mxid[self.custom_mxid] + prev_custom_mxid = self.custom_mxid self.custom_mxid = None self.access_token = None self.next_batch = None await self.save() self.intent = self._fresh_intent() - if whoami.user_id != self.custom_mxid: + if whoami and whoami.user_id != prev_custom_mxid: raise OnlyLoginSelf() raise InvalidAccessToken() if self.sync_with_custom_puppets and start_sync_task: From e32ce1fcedc7f83aef89dce161265bcf21d3c89e Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 1 Jun 2022 14:41:28 +0300 Subject: [PATCH 103/456] Add `com.beeper.message_send_status` event type --- mautrix/types/__init__.py | 6 +++++ mautrix/types/event/__init__.py | 1 + mautrix/types/event/beeper.py | 48 +++++++++++++++++++++++++++++++++ mautrix/types/event/generic.py | 5 ++++ mautrix/types/event/type.py | 1 + mautrix/types/event/type.pyi | 2 ++ 6 files changed, 63 insertions(+) create mode 100644 mautrix/types/event/beeper.py diff --git a/mautrix/types/__init__.py b/mautrix/types/__init__.py index 6040c525..4933596d 100644 --- a/mautrix/types/__init__.py +++ b/mautrix/types/__init__.py @@ -36,6 +36,8 @@ BaseUnsigned, BatchSendEvent, BatchSendStateEvent, + BeeperMessageStatusEvent, + BeeperMessageStatusEventContent, CallAnswerEventContent, CallCandidate, CallCandidatesEventContent, @@ -79,6 +81,7 @@ MemberStateEventContent, MessageEvent, MessageEventContent, + MessageStatusReason, MessageType, MessageUnsigned, OlmCiphertext, @@ -218,6 +221,8 @@ "BaseUnsigned", "BatchSendEvent", "BatchSendStateEvent", + "BeeperMessageStatusEvent", + "BeeperMessageStatusEventContent", "CallAnswerEventContent", "CallCandidate", "CallCandidatesEventContent", @@ -261,6 +266,7 @@ "MemberStateEventContent", "MessageEvent", "MessageEventContent", + "MessageStatusReason", "MessageType", "MessageUnsigned", "OlmCiphertext", diff --git a/mautrix/types/event/__init__.py b/mautrix/types/event/__init__.py index 2d7e889c..dde5cf14 100644 --- a/mautrix/types/event/__init__.py +++ b/mautrix/types/event/__init__.py @@ -11,6 +11,7 @@ ) from .base import BaseEvent, BaseRoomEvent, BaseUnsigned, GenericEvent from .batch import BatchSendEvent, BatchSendStateEvent +from .beeper import BeeperMessageStatusEvent, BeeperMessageStatusEventContent, MessageStatusReason from .encrypted import ( EncryptedEvent, EncryptedEventContent, diff --git a/mautrix/types/event/beeper.py b/mautrix/types/event/beeper.py new file mode 100644 index 00000000..ce0b2998 --- /dev/null +++ b/mautrix/types/event/beeper.py @@ -0,0 +1,48 @@ +# Copyright (c) 2022 Tulir Asokan +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +from typing import Optional + +from attr import dataclass + +from ..util import SerializableAttrs, SerializableEnum, field +from .base import BaseRoomEvent +from .message import RelatesTo + + +class MessageStatusReason(SerializableEnum): + GENERIC_ERROR = "m.event_not_handled" + UNSUPPORTED = "com.beeper.unsupported_event" + TOO_OLD = "m.event_too_old" + NETWORK_ERROR = "m.foreign_network_error" + NO_PERMISSION = "m.no_permission" + + @property + def checkpoint_status(self): + from mautrix.util.message_send_checkpoint import MessageSendCheckpointStatus + + if self == MessageStatusReason.UNSUPPORTED: + return MessageSendCheckpointStatus.UNSUPPORTED + elif self == MessageStatusReason.TOO_OLD: + return MessageSendCheckpointStatus.TIMEOUT + return MessageSendCheckpointStatus.PERM_FAILURE + + +@dataclass +class BeeperMessageStatusEventContent(SerializableAttrs): + network: str + success: bool + relates_to: RelatesTo = field(json="m.relates_to") + + reason: Optional[MessageStatusReason] = None + error: Optional[str] = None + message: Optional[str] = None + can_retry: Optional[bool] = None + is_certain: Optional[bool] = None + + +@dataclass +class BeeperMessageStatusEvent(BaseRoomEvent, SerializableAttrs): + content: BeeperMessageStatusEventContent diff --git a/mautrix/types/event/generic.py b/mautrix/types/event/generic.py index 67f5bbad..f105f694 100644 --- a/mautrix/types/event/generic.py +++ b/mautrix/types/event/generic.py @@ -9,6 +9,7 @@ from ..util import Obj, deserializer from .account_data import AccountDataEvent, AccountDataEventContent from .base import EventType, GenericEvent +from .beeper import BeeperMessageStatusEvent, BeeperMessageStatusEventContent from .encrypted import EncryptedEvent, EncryptedEventContent from .ephemeral import ( EphemeralEvent, @@ -38,6 +39,7 @@ EncryptedEvent, ToDeviceEvent, CallEvent, + BeeperMessageStatusEvent, GenericEvent, ], ) @@ -53,6 +55,7 @@ EncryptedEventContent, ToDeviceEventContent, CallEventContent, + BeeperMessageStatusEventContent, Obj, ] @@ -81,6 +84,8 @@ def deserialize_event(data: JSON) -> Event: return AccountDataEvent.deserialize(data) elif event_type.is_ephemeral: return EphemeralEvent.deserialize(data) + elif event_type == EventType.BEEPER_MESSAGE_STATUS: + return BeeperMessageStatusEvent.deserialize(data) else: return GenericEvent.deserialize(data) diff --git a/mautrix/types/event/type.py b/mautrix/types/event/type.py index 1de04812..609f40ff 100644 --- a/mautrix/types/event/type.py +++ b/mautrix/types/event/type.py @@ -195,6 +195,7 @@ def is_to_device(self) -> bool: "m.call.hangup": "CALL_HANGUP", "m.call.reject": "CALL_REJECT", "m.call.negotiate": "CALL_NEGOTIATE", + "com.beeper.message_send_status": "BEEPER_MESSAGE_STATUS", }, EventType.Class.EPHEMERAL: { "m.receipt": "RECEIPT", diff --git a/mautrix/types/event/type.pyi b/mautrix/types/event/type.pyi index 7c573da5..6c4c6e65 100644 --- a/mautrix/types/event/type.pyi +++ b/mautrix/types/event/type.pyi @@ -50,6 +50,8 @@ class EventType(Serializable): CALL_REJECT: "EventType" CALL_NEGOTIATE: "EventType" + BEEPER_MESSAGE_STATUS: "EventType" + RECEIPT: "EventType" TYPING: "EventType" PRESENCE: "EventType" From 8a15fe44f22fa9e924430ea0e8f0a720bbbf90d7 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 2 Jun 2022 11:05:10 +0300 Subject: [PATCH 104/456] Bump version to 0.16.6 --- CHANGELOG.md | 7 +++++++ mautrix/__init__.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b781e03..8087f6ea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## v0.16.6 (2022-06-02) + +* *(bridge)* Fixed double puppeting `start` method not handling some errors + from /whoami correctly. +* *(types)* Added `com.beeper.message_send_status` event type for bridging + status. + ## v0.16.5 (2022-05-26) * *(bridge.commands)* Added `reason` field for `CommandEvent.redact`. diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 3c8a6b9e..31825f3c 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.16.5" +__version__ = "0.16.6" __author__ = "Tulir Asokan " __all__ = [ "api", From e09d3cdd3f14f467c75beb6a0d39f1d57ad6c268 Mon Sep 17 00:00:00 2001 From: Malte E Date: Thu, 2 Jun 2022 19:29:05 +0200 Subject: [PATCH 105/456] comments by tulir --- mautrix/bridge/matrix.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/mautrix/bridge/matrix.py b/mautrix/bridge/matrix.py index 5110062f..84c2bf1b 100644 --- a/mautrix/bridge/matrix.py +++ b/mautrix/bridge/matrix.py @@ -243,21 +243,23 @@ async def handle_unban( async def handle_join(self, room_id: RoomID, user_id: UserID, event_id: EventID) -> None: pass - async def handle_knock(self, room_id: RoomID, user_id: UserID, event_id: EventID) -> None: + async def handle_knock( + self, room_id: RoomID, user_id: UserID, reason: str, event_id: EventID + ) -> None: pass async def handle_retract_knock( - self, room_id: RoomID, user_id: UserID, event_id: EventID + self, room_id: RoomID, user_id: UserID, reason: str, event_id: EventID ) -> None: pass async def handle_reject_knock( - self, room_id: RoomID, user_id: UserID, event_id: EventID + self, room_id: RoomID, user_id: UserID, sender: UserID, reason: str, event_id: EventID ) -> None: pass async def handle_accept_knock( - self, room_id: RoomID, user_id: UserID, event_id: EventID + self, room_id: RoomID, user_id: UserID, sender: UserID, reason: str, event_id: EventID ) -> None: pass @@ -837,7 +839,11 @@ async def int_handle_event(self, evt: Event, send_bridge_checkpoint: bool = True if evt.content.membership == Membership.INVITE: if prev_membership == Membership.KNOCK: await self.handle_accept_knock( - evt.room_id, UserID(evt.state_key), evt.event_id + evt.room_id, + UserID(evt.state_key), + evt.sender, + evt.content.reason, + evt.event_id, ) else: await self.int_handle_invite(evt) @@ -906,7 +912,6 @@ async def int_handle_event(self, evt: Event, send_bridge_checkpoint: bool = True await self.handle_knock( evt.room_id, UserID(evt.state_key), - evt.sender, evt.content.reason, evt.event_id, ) From 4487c8eebb7eb68670357624e011e8384aafb5f0 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 3 Jun 2022 13:56:08 +0300 Subject: [PATCH 106/456] Add handle_exception to catch any errors in LoggingConnection --- mautrix/util/async_db/connection.py | 64 +++++++++++++++++++++------- mautrix/util/async_db/connection.pyi | 4 +- 2 files changed, 52 insertions(+), 16 deletions(-) diff --git a/mautrix/util/async_db/connection.py b/mautrix/util/async_db/connection.py index f8677207..fe0ecf93 100644 --- a/mautrix/util/async_db/connection.py +++ b/mautrix/util/async_db/connection.py @@ -5,7 +5,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import annotations -from typing import Any, Callable, TypeVar +from typing import Any, Callable, TypeVar, Awaitable from contextlib import asynccontextmanager from logging import WARNING import functools @@ -43,20 +43,22 @@ async def wrapper(self: LoggingConnection, arg: str, *args: Any, **kwargs: str) return wrapper -class LoggingConnection: - scheme: Scheme - wrapped: aiosqlite.TxnConnection | asyncpg.Connection - log: TraceLogger +async def handle_exception_noop() -> None: + pass + +class LoggingConnection: def __init__( self, scheme: Scheme, wrapped: aiosqlite.TxnConnection | asyncpg.Connection, log: TraceLogger, + handle_exception: Callable[[Exception], Awaitable[None]] = handle_exception_noop ) -> None: self.scheme = scheme self.wrapped = wrapped self.log = log + self._handle_exception = handle_exception self._inited = True def __setattr__(self, key: str, value: Any) -> None: @@ -66,36 +68,60 @@ def __setattr__(self, key: str, value: Any) -> None: @asynccontextmanager async def transaction(self) -> None: - async with self.wrapped.transaction(): - yield + try: + async with self.wrapped.transaction(): + yield + except Exception as e: + await self._handle_exception(e) + raise @log_duration async def execute(self, query: str, *args: Any, timeout: float | None = None) -> str | Cursor: - return await self.wrapped.execute(query, *args, timeout=timeout) + try: + return await self.wrapped.execute(query, *args, timeout=timeout) + except Exception as e: + await self._handle_exception(e) + raise @log_duration async def executemany( self, query: str, *args: Any, timeout: float | None = None ) -> str | Cursor: - return await self.wrapped.executemany(query, *args, timeout=timeout) + try: + return await self.wrapped.executemany(query, *args, timeout=timeout) + except Exception as e: + await self._handle_exception(e) + raise @log_duration async def fetch( self, query: str, *args: Any, timeout: float | None = None ) -> list[Row | Record]: - return await self.wrapped.fetch(query, *args, timeout=timeout) + try: + return await self.wrapped.fetch(query, *args, timeout=timeout) + except Exception as e: + await self._handle_exception(e) + raise @log_duration async def fetchval( self, query: str, *args: Any, column: int = 0, timeout: float | None = None ) -> Any: - return await self.wrapped.fetchval(query, *args, column=column, timeout=timeout) + try: + return await self.wrapped.fetchval(query, *args, column=column, timeout=timeout) + except Exception as e: + await self._handle_exception(e) + raise @log_duration async def fetchrow( self, query: str, *args: Any, timeout: float | None = None ) -> Row | Record | None: - return await self.wrapped.fetchrow(query, *args, timeout=timeout) + try: + return await self.wrapped.fetchrow(query, *args, timeout=timeout) + except Exception as e: + await self._handle_exception(e) + raise async def table_exists(self, name: str) -> bool: if self.scheme == Scheme.SQLITE: @@ -121,6 +147,14 @@ async def copy_records_to_table( ) -> None: if self.scheme != Scheme.POSTGRES: raise RuntimeError("copy_records_to_table is only supported on Postgres") - return await self.wrapped.copy_records_to_table( - table_name, records=records, columns=columns, schema_name=schema_name, timeout=timeout - ) + try: + return await self.wrapped.copy_records_to_table( + table_name, + records=records, + columns=columns, + schema_name=schema_name, + timeout=timeout, + ) + except Exception as e: + await self._handle_exception(e) + raise diff --git a/mautrix/util/async_db/connection.pyi b/mautrix/util/async_db/connection.pyi index e4219e50..5178dd68 100644 --- a/mautrix/util/async_db/connection.pyi +++ b/mautrix/util/async_db/connection.pyi @@ -3,7 +3,7 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from typing import Any, AsyncContextManager +from typing import Any, AsyncContextManager, Callable, Awaitable from sqlite3 import Row from asyncpg import Record @@ -17,12 +17,14 @@ from .scheme import Scheme class LoggingConnection: scheme: Scheme wrapped: aiosqlite.TxnConnection | asyncpg.Connection + _handle_exception: Callable[[Exception], Awaitable[None]] log: TraceLogger def __init__( self, scheme: Scheme, wrapped: aiosqlite.TxnConnection | asyncpg.Connection, log: TraceLogger, + handle_exception: Callable[[Exception], Awaitable[None]] = None, ) -> None: ... async def transaction(self) -> AsyncContextManager[None]: ... async def execute(self, query: str, *args: Any, timeout: float | None = None) -> str: ... From 0fd1d7016db910317673fa6fdb17f7a39354e40f Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 5 Jun 2022 11:54:11 +0300 Subject: [PATCH 107/456] Fix imports --- mautrix/util/async_db/connection.py | 4 ++-- mautrix/util/async_db/connection.pyi | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mautrix/util/async_db/connection.py b/mautrix/util/async_db/connection.py index fe0ecf93..ec4c9957 100644 --- a/mautrix/util/async_db/connection.py +++ b/mautrix/util/async_db/connection.py @@ -5,7 +5,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import annotations -from typing import Any, Callable, TypeVar, Awaitable +from typing import Any, Awaitable, Callable, TypeVar from contextlib import asynccontextmanager from logging import WARNING import functools @@ -53,7 +53,7 @@ def __init__( scheme: Scheme, wrapped: aiosqlite.TxnConnection | asyncpg.Connection, log: TraceLogger, - handle_exception: Callable[[Exception], Awaitable[None]] = handle_exception_noop + handle_exception: Callable[[Exception], Awaitable[None]] = handle_exception_noop, ) -> None: self.scheme = scheme self.wrapped = wrapped diff --git a/mautrix/util/async_db/connection.pyi b/mautrix/util/async_db/connection.pyi index 5178dd68..27b86be7 100644 --- a/mautrix/util/async_db/connection.pyi +++ b/mautrix/util/async_db/connection.pyi @@ -3,7 +3,7 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from typing import Any, AsyncContextManager, Callable, Awaitable +from typing import Any, AsyncContextManager, Awaitable, Callable from sqlite3 import Row from asyncpg import Record From 27947d467c5c209a6a0cd4a5fa3d8708f2eca4d7 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 14 Jun 2022 18:19:51 +0300 Subject: [PATCH 108/456] Add support for parsing img tags Defaults to just the alt or title attribute. Fixes mautrix/telegram#801 --- mautrix/util/formatter/parser.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/mautrix/util/formatter/parser.py b/mautrix/util/formatter/parser.py index 83f0e57f..76c3d270 100644 --- a/mautrix/util/formatter/parser.py +++ b/mautrix/util/formatter/parser.py @@ -174,6 +174,9 @@ async def event_link_to_fstring( ) -> T | None: return None + async def img_to_fstring(self, node: HTMLNode, ctx: RecursionContext) -> T: + return self.fs(node.attrib.get("alt") or node.attrib.get("title") or "") + async def custom_node_to_fstring(self, node: HTMLNode, ctx: RecursionContext) -> T | None: return None @@ -203,6 +206,8 @@ async def node_to_fstring(self, node: HTMLNode, ctx: RecursionContext) -> T: return await self.basic_format_to_fstring(node, ctx) elif node.tag == "a": return await self.link_to_fstring(node, ctx) + elif node.tag == "img": + return await self.img_to_fstring(node, ctx) elif node.tag == "p": return (await self.tag_aware_parse_node(node, ctx)).append("\n") elif node.tag in ("font", "span"): From c359c56d6462cc67584b7acc1d6bd223822fa059 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 16 Jun 2022 13:57:42 +0300 Subject: [PATCH 109/456] Make base_insertion_event_id field optional --- mautrix/types/misc.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/mautrix/types/misc.py b/mautrix/types/misc.py index 5340534e..e880334a 100644 --- a/mautrix/types/misc.py +++ b/mautrix/types/misc.py @@ -3,7 +3,7 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from typing import Dict, List, NamedTuple, NewType +from typing import Dict, List, NamedTuple, NewType, Optional from enum import Enum from attr import dataclass @@ -128,6 +128,5 @@ class BatchSendResponse(SerializableAttrs): insertion_event_id: EventID batch_event_id: EventID - base_insertion_event_id: EventID - next_batch_id: BatchID + base_insertion_event_id: Optional[EventID] = None From 89b5cae15b4c46d72d6bd8a4a4600930a521cb6f Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 16 Jun 2022 19:07:52 +0300 Subject: [PATCH 110/456] Add notifications to power levels --- mautrix/types/event/state.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/mautrix/types/event/state.py b/mautrix/types/event/state.py index f1f1053c..acb24f81 100644 --- a/mautrix/types/event/state.py +++ b/mautrix/types/event/state.py @@ -15,6 +15,11 @@ from .type import EventType, RoomType +@dataclass +class NotificationPowerLevels(SerializableAttrs): + room: int = 50 + + @dataclass class PowerLevelStateEventContent(SerializableAttrs): """The content of a power level event.""" @@ -27,6 +32,8 @@ class PowerLevelStateEventContent(SerializableAttrs): ) events_default: int = 0 + notifications: NotificationPowerLevels = attr.ib(factory=lambda: NotificationPowerLevels()) + state_default: int = 50 invite: int = 50 From bf0a73e581a2821fe95d80236a89a7f868341941 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 17 Jun 2022 16:30:45 +0300 Subject: [PATCH 111/456] Add double puppet source key to all state events --- mautrix/appservice/api/intent.py | 112 ++++++++++++++++++++++--------- mautrix/bridge/portal.py | 3 +- mautrix/client/api/rooms.py | 18 ++++- mautrix/client/store_updater.py | 10 ++- 4 files changed, 108 insertions(+), 35 deletions(-) diff --git a/mautrix/appservice/api/intent.py b/mautrix/appservice/api/intent.py index be8d87ff..e2f3f804 100644 --- a/mautrix/appservice/api/intent.py +++ b/mautrix/appservice/api/intent.py @@ -5,7 +5,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import annotations -from typing import Any, Awaitable, Iterable +from typing import Any, Awaitable, Iterable, TypeVar from urllib.parse import quote as urllib_quote from mautrix.api import Method, Path @@ -39,6 +39,7 @@ RoomNameStateEventContent, RoomPinnedEventsStateEventContent, RoomTopicStateEventContent, + SerializableAttrs, StateEventContent, UserID, ) @@ -91,6 +92,8 @@ def quote(*args, **kwargs): DOUBLE_PUPPET_SOURCE_KEY = "fi.mau.double_puppet_source" +T = TypeVar("T") + class IntentAPI(StoreUpdatingAPI): """ @@ -188,6 +191,13 @@ async def set_presence( # endregion # region Room actions + def _add_source_key(self, content: T = None) -> T: + if self.api.is_real_user and self.api.bridge_name: + if not content: + content = {} + content[DOUBLE_PUPPET_SOURCE_KEY] = self.api.bridge_name + return content + async def invite_user( self, room_id: RoomID, @@ -226,6 +236,7 @@ async def invite_user( await self.state_store.get_membership(room_id, user_id) not in ok_states ) if do_invite: + extra_content = self._add_source_key(extra_content) await super().invite_user( room_id, user_id, reason=reason, extra_content=extra_content ) @@ -236,22 +247,73 @@ async def invite_user( else: raise IntentError(f"Failed to invite {user_id} to {room_id}", e) + async def kick_user( + self, + room_id: RoomID, + user_id: UserID, + reason: str = "", + extra_content: dict[str, JSON] | None = None, + ) -> None: + extra_content = self._add_source_key(extra_content) + await super().kick_user(room_id, user_id, reason=reason, extra_content=extra_content) + + async def ban_user( + self, + room_id: RoomID, + user_id: UserID, + reason: str = "", + extra_content: dict[str, JSON] | None = None, + ) -> None: + extra_content = self._add_source_key(extra_content) + await super().ban_user(room_id, user_id, reason=reason, extra_content=extra_content) + + async def unban_user( + self, + room_id: RoomID, + user_id: UserID, + reason: str = "", + extra_content: dict[str, JSON] | None = None, + ) -> None: + extra_content = self._add_source_key(extra_content) + await super().unban_user(room_id, user_id, reason=reason, extra_content=extra_content) + + async def join_room_by_id( + self, + room_id: RoomID, + third_party_signed: JSON = None, + extra_content: dict[str, JSON] | None = None, + ) -> RoomID: + extra_content = self._add_source_key(extra_content) + return await super().join_room_by_id( + room_id, third_party_signed=third_party_signed, extra_content=extra_content + ) + + async def leave_room( + self, + room_id: RoomID, + reason: str | None = None, + extra_content: dict[str, JSON] | None = None, + raise_not_in_room: bool = False, + ) -> None: + extra_content = self._add_source_key(extra_content) + await super().leave_room(room_id, reason, extra_content, raise_not_in_room) + def set_room_avatar( self, room_id: RoomID, avatar_url: ContentURI | None, **kwargs ) -> Awaitable[EventID]: - return self.send_state_event( - room_id, EventType.ROOM_AVATAR, RoomAvatarStateEventContent(url=avatar_url), **kwargs - ) + content = RoomAvatarStateEventContent(url=avatar_url) + content = self._add_source_key(content) + return self.send_state_event(room_id, EventType.ROOM_AVATAR, content, **kwargs) def set_room_name(self, room_id: RoomID, name: str, **kwargs) -> Awaitable[EventID]: - return self.send_state_event( - room_id, EventType.ROOM_NAME, RoomNameStateEventContent(name=name), **kwargs - ) + content = RoomNameStateEventContent(name=name) + content = self._add_source_key(content) + return self.send_state_event(room_id, EventType.ROOM_NAME, content, **kwargs) def set_room_topic(self, room_id: RoomID, topic: str, **kwargs) -> Awaitable[EventID]: - return self.send_state_event( - room_id, EventType.ROOM_TOPIC, RoomTopicStateEventContent(topic=topic), **kwargs - ) + content = RoomTopicStateEventContent(topic=topic) + content = self._add_source_key(content) + return self.send_state_event(room_id, EventType.ROOM_TOPIC, content, **kwargs) async def get_power_levels( self, room_id: RoomID, ignore_cache: bool = False @@ -271,6 +333,7 @@ async def get_power_levels( async def set_power_levels( self, room_id: RoomID, content: PowerLevelStateEventContent, **kwargs ) -> EventID: + content = self._add_source_key(content) response = await self.send_state_event( room_id, EventType.ROOM_POWER_LEVELS, content, **kwargs ) @@ -289,12 +352,9 @@ async def get_pinned_messages(self, room_id: RoomID) -> list[EventID]: def set_pinned_messages( self, room_id: RoomID, events: list[EventID], **kwargs ) -> Awaitable[EventID]: - return self.send_state_event( - room_id, - EventType.ROOM_PINNED_EVENTS, - RoomPinnedEventsStateEventContent(pinned=events), - **kwargs, - ) + content = RoomPinnedEventsStateEventContent(pinned=events) + content = self._add_source_key(content) + return self.send_state_event(room_id, EventType.ROOM_PINNED_EVENTS, content, **kwargs) async def pin_message(self, room_id: RoomID, event_id: EventID) -> None: events = await self.get_pinned_messages(room_id) @@ -309,12 +369,9 @@ async def unpin_message(self, room_id: RoomID, event_id: EventID): await self.set_pinned_messages(room_id, events) async def set_join_rule(self, room_id: RoomID, join_rule: JoinRule, **kwargs): - await self.send_state_event( - room_id, - EventType.ROOM_JOIN_RULES, - JoinRulesStateEventContent(join_rule=join_rule), - **kwargs, - ) + content = JoinRulesStateEventContent(join_rule=join_rule) + content = self._add_source_key(content) + await self.send_state_event(room_id, EventType.ROOM_JOIN_RULES, content, **kwargs) async def get_room_displayname( self, room_id: RoomID, user_id: UserID, ignore_cache=False @@ -358,10 +415,7 @@ async def send_message_event( self, room_id: RoomID, event_type: EventType, content: EventContent, **kwargs ) -> EventID: await self._ensure_has_power_level_for(room_id, event_type) - - if self.api.is_real_user and self.api.bridge_name is not None: - content[DOUBLE_PUPPET_SOURCE_KEY] = self.api.bridge_name - + content = self._add_source_key(content) return await super().send_message_event(room_id, event_type, content, **kwargs) async def redact( @@ -373,10 +427,7 @@ async def redact( **kwargs, ) -> EventID: await self._ensure_has_power_level_for(room_id, EventType.ROOM_REDACTION) - if self.api.is_real_user and self.api.bridge_name: - if not extra_content: - extra_content = {} - extra_content[DOUBLE_PUPPET_SOURCE_KEY] = self.api.bridge_name + extra_content = self._add_source_key(extra_content) return await super().redact( room_id, event_id, reason, extra_content=extra_content, **kwargs ) @@ -390,6 +441,7 @@ async def send_state_event( **kwargs, ) -> EventID: await self._ensure_has_power_level_for(room_id, event_type, state_key=state_key) + content = self._add_source_key(content) return await super().send_state_event(room_id, event_type, content, state_key, **kwargs) async def get_room_members( diff --git a/mautrix/bridge/portal.py b/mautrix/bridge/portal.py index 705a1ace..bfbd63dd 100644 --- a/mautrix/bridge/portal.py +++ b/mautrix/bridge/portal.py @@ -465,8 +465,7 @@ async def cleanup_room( left = False if custom_puppet: try: - extra_content = {DOUBLE_PUPPET_SOURCE_KEY: cls.bridge.name} - await custom_puppet.intent.leave_room(room_id, extra_content=extra_content) + await custom_puppet.intent.leave_room(room_id) await custom_puppet.intent.forget_room(room_id) except MatrixError: pass diff --git a/mautrix/client/api/rooms.py b/mautrix/client/api/rooms.py index 700bf876..81e22d33 100644 --- a/mautrix/client/api/rooms.py +++ b/mautrix/client/api/rooms.py @@ -581,7 +581,13 @@ async def ban_user( Method.POST, Path.v3.rooms[room_id].ban, {"user_id": user_id, "reason": reason} ) - async def unban_user(self, room_id: RoomID, user_id: UserID, reason: str = "") -> None: + async def unban_user( + self, + room_id: RoomID, + user_id: UserID, + reason: str = "", + extra_content: dict[str, JSON] | None = None, + ) -> None: """ Unban a user from the room. This allows them to be invited to the room, and join if they would otherwise be allowed to join according to its join rules. The caller must have the @@ -594,7 +600,17 @@ async def unban_user(self, room_id: RoomID, user_id: UserID, reason: str = "") - user_id: The fully qualified user ID of the user being banned. reason: The reason the user has been unbanned. This will be supplied as the ``reason`` on the target's updated `m.room.member`_ event. + extra_content: Additional properties for the unban (leave) event content. + If a non-empty dict is passed, the unban will be created using + the ``PUT /state/m.room.member/...`` endpoint instead of ``POST /unban``. """ + if extra_content: + if reason and "reason" not in extra_content: + extra_content["reason"] = reason + await self.send_member_event( + room_id, user_id, Membership.LEAVE, extra_content=extra_content + ) + return await self.api.request( Method.POST, Path.v3.rooms[room_id].unban, {"user_id": user_id, "reason": reason} ) diff --git a/mautrix/client/store_updater.py b/mautrix/client/store_updater.py index 55d649c3..f41ed82a 100644 --- a/mautrix/client/store_updater.py +++ b/mautrix/client/store_updater.py @@ -110,8 +110,14 @@ async def ban_user( if not extra_content and self.state_store: await self.state_store.set_membership(room_id, user_id, Membership.BAN) - async def unban_user(self, room_id: RoomID, user_id: UserID, reason: str = "") -> None: - await super().unban_user(room_id, user_id, reason=reason) + async def unban_user( + self, + room_id: RoomID, + user_id: UserID, + reason: str = "", + extra_content: dict[str, JSON] | None = None, + ) -> None: + await super().unban_user(room_id, user_id, reason=reason, extra_content=extra_content) if self.state_store: await self.state_store.set_membership(room_id, user_id, Membership.LEAVE) From 11a4d4efbb2038318532ce2de781335874479903 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 17 Jun 2022 16:50:11 +0300 Subject: [PATCH 112/456] Update changelog --- CHANGELOG.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8087f6ea..ba5b36c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,13 @@ +## unreleased + +* *(util.formatter)* Added support for parsing `img` tags + * By default, the `alt` or `title` attribute will be used as plaintext. +* *(types)* Added `notifications` object to power level content class. +* *(bridge)* Added utility methods for handling incoming knocks in + `MatrixHandler` (thanks to [@maltee1] in [#103]). + +[#103]: https://github.com/mautrix/python/pull/103 + ## v0.16.6 (2022-06-02) * *(bridge)* Fixed double puppeting `start` method not handling some errors From 59c214634fb6e38b2f32291cb58a8028254709b6 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 19 Jun 2022 14:52:04 +0300 Subject: [PATCH 113/456] Bump version to 0.16.7 --- CHANGELOG.md | 5 ++++- mautrix/__init__.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ba5b36c7..00c304ca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,10 +1,13 @@ -## unreleased +## v0.16.7 (2022-06-19) * *(util.formatter)* Added support for parsing `img` tags * By default, the `alt` or `title` attribute will be used as plaintext. * *(types)* Added `notifications` object to power level content class. * *(bridge)* Added utility methods for handling incoming knocks in `MatrixHandler` (thanks to [@maltee1] in [#103]). +* *(appservice)* Updated `IntentAPI` to add the `fi.mau.double_puppet_source` + to all state events sent with double puppeted intents (previously it was only + added to non-state events). [#103]: https://github.com/mautrix/python/pull/103 diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 31825f3c..03d9d990 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.16.6" +__version__ = "0.16.7" __author__ = "Tulir Asokan " __all__ = [ "api", From 92aef6594af5a6803641fa4cd03cf0eddd30559a Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 20 Jun 2022 10:26:37 +0300 Subject: [PATCH 114/456] Stop bridge if crypto sync fails --- mautrix/bridge/e2ee.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/mautrix/bridge/e2ee.py b/mautrix/bridge/e2ee.py index 2586398e..ed4658f4 100644 --- a/mautrix/bridge/e2ee.py +++ b/mautrix/bridge/e2ee.py @@ -11,7 +11,7 @@ from mautrix import __optional_imports__ from mautrix.appservice import AppService -from mautrix.client import Client, SyncStore +from mautrix.client import Client, InternalEventType, SyncStore from mautrix.crypto import CryptoStore, OlmMachine, PgCryptoStore, RejectKeyShare, StateStore from mautrix.errors import EncryptionError, SessionNotFound from mautrix.types import ( @@ -100,8 +100,14 @@ def __init__( default_retry_count=default_http_retry_count, ) self.crypto = OlmMachine(self.client, self.crypto_store, self.state_store) + self.client.add_event_handler(InternalEventType.SYNC_STOPPED, self._exit_on_sync_fail) self.crypto.allow_key_share = self.allow_key_share + async def _exit_on_sync_fail(self, data) -> None: + if data["error"]: + self.log.critical("Exiting due to crypto sync error") + sys.exit(32) + async def allow_key_share(self, device: DeviceIdentity, request: RequestedKeyInfo) -> bool: require_verification = self.key_sharing_config.get("require_verification", True) allow = self.key_sharing_config.get("allow", False) From 75964da0eefea48d29a3deeb79f006b2acecbb72 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 20 Jun 2022 11:02:32 +0300 Subject: [PATCH 115/456] Stop program on asyncpg internal client errors --- mautrix/util/async_db/asyncpg.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/mautrix/util/async_db/asyncpg.py b/mautrix/util/async_db/asyncpg.py index 28d3dcab..e7f2102f 100644 --- a/mautrix/util/async_db/asyncpg.py +++ b/mautrix/util/async_db/asyncpg.py @@ -9,6 +9,8 @@ from contextlib import asynccontextmanager import asyncio import logging +import sys +import traceback from yarl import URL import asyncpg @@ -23,6 +25,7 @@ class PostgresDatabase(Database): scheme = Scheme.POSTGRES _pool: asyncpg.pool.Pool | None _pool_override: bool + _exit_on_ice: bool def __init__( self, @@ -37,6 +40,7 @@ def __init__( self.scheme = Scheme.COCKROACH # Send postgres scheme to asyncpg url = url.with_scheme("postgres") + self._exit_on_ice = (db_args or {}).pop("meow_exit_on_ice", True) super().__init__( url, db_args=db_args, @@ -72,10 +76,26 @@ async def stop(self) -> None: if not self._pool_override and self._pool is not None: await self._pool.close() + async def _handle_exception(self, err: Exception) -> None: + if self._exit_on_ice and isinstance(err, asyncpg.InternalClientError): + pre_stack = traceback.format_stack()[:-2] + post_stack = traceback.format_exception(err) + header = post_stack[0] + post_stack = post_stack[1:] + self.log.critical( + "Got asyncpg internal client error, exiting...\n%s%s%s", + header, + "".join(pre_stack), + "".join(post_stack), + ) + sys.exit(26) + @asynccontextmanager async def acquire(self) -> LoggingConnection: async with self.pool.acquire() as conn: - yield LoggingConnection(self.scheme, conn, self.log) + yield LoggingConnection( + self.scheme, conn, self.log, handle_exception=self._handle_exception + ) Database.schemes["postgres"] = PostgresDatabase From fa1514cee964abdb8937972ebb138227ebe561b7 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 20 Jun 2022 13:44:50 +0300 Subject: [PATCH 116/456] Bump version to 0.16.8 --- CHANGELOG.md | 9 +++++++++ mautrix/__init__.py | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 00c304ca..c086e812 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,12 @@ +## v0.16.8 (2022-06-20) + +* *(bridge)* Updated e2be helper to stop bridge if syncing fails. +* *(util.async_db)* Updated asyncpg connector to stop program if an asyncpg + `InternalClientError` is thrown. These errors usually cause everything to + get stuck. + * The behavior can be disabled by passing `meow_exit_on_ice` = `false` in + the `db_args`. + ## v0.16.7 (2022-06-19) * *(util.formatter)* Added support for parsing `img` tags diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 03d9d990..8cf0801b 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.16.7" +__version__ = "0.16.8" __author__ = "Tulir Asokan " __all__ = [ "api", From 9acc3e6387ae823302f0ac8f39a5ab4f761928be Mon Sep 17 00:00:00 2001 From: Malte E Date: Sat, 18 Jun 2022 21:38:43 +0200 Subject: [PATCH 117/456] add support for outgoing knocks --- mautrix/client/api/rooms.py | 39 +++++++++++++++++++++++++++++++++ mautrix/client/store_updater.py | 11 ++++++++++ 2 files changed, 50 insertions(+) diff --git a/mautrix/client/api/rooms.py b/mautrix/client/api/rooms.py index 81e22d33..8caef5d1 100644 --- a/mautrix/client/api/rooms.py +++ b/mautrix/client/api/rooms.py @@ -482,6 +482,45 @@ async def leave_room( if "not in room" not in e.message or raise_not_in_room: raise + async def knock_room( + self, + room_id_or_alias: RoomID | RoomAlias, + reason: str | None = None, + servers: list[str] | None = None, + ) -> RoomID: + """ + knock on a room, i.e. request to join it by its ID or alias, with an optional list of + servers to ask about the ID from. + + See also: `API reference `__ + + Args: + room_id_or_alias: The ID of the room to knock on, or an alias pointing to the room. + reason: The reason for knocking on the room. This will be supplied as the ``reason`` on + the updated `m.room.member`_ event. + servers: A list of servers to ask about the room ID to knock. Not applicable for aliases, + as aliases already contain the necessary server information. + + Returns: + The ID of the room the user knocked on. + """ + data = {} + if reason: + data["reason"] = reason + query_params = CIMultiDict() + for server_name in servers or []: + query_params.add("server_name", server_name) + content = await self.api.request( + Method.POST, + Path.v3.knock[room_id_or_alias], + content=data, + query_params=query_params, + ) + try: + return content["room_id"] + except KeyError: + raise MatrixResponseError("`room_id` not in response.") + async def forget_room(self, room_id: RoomID) -> None: """ Stop remembering a particular room, i.e. forget it. diff --git a/mautrix/client/store_updater.py b/mautrix/client/store_updater.py index f41ed82a..929b378d 100644 --- a/mautrix/client/store_updater.py +++ b/mautrix/client/store_updater.py @@ -77,6 +77,17 @@ async def leave_room( if not extra_content and self.state_store: await self.state_store.set_membership(room_id, self.mxid, Membership.LEAVE) + async def knock_room( + self, + room_id_or_alias: RoomID | RoomAlias, + reason: str | None = None, + servers: list[str] | None = None, + ) -> RoomID: + room_id = await super().knock_room(room_id_or_alias, reason, servers) + if room_id and self.state_store: + await self.state_store.set_membership(room_id, self.mxid, Membership.KNOCK) + return room_id + async def invite_user( self, room_id: RoomID, From 1f89639a9cfd275991573db1ac28bc63a96b6d86 Mon Sep 17 00:00:00 2001 From: Sumner Evans Date: Tue, 21 Jun 2022 12:57:31 -0600 Subject: [PATCH 118/456] bridgeconfig: add rotation settings for encryption Also copies the config values that the code depends on. Before, some of the values had to be copied by each bridge. Co-authored-by: Tulir Asokan --- mautrix/bridge/config.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/mautrix/bridge/config.py b/mautrix/bridge/config.py index 3399c459..64e6e7af 100644 --- a/mautrix/bridge/config.py +++ b/mautrix/bridge/config.py @@ -102,6 +102,15 @@ def do_update(self, helper: ConfigUpdateHelper) -> None: copy("bridge.management_room_text.additional_help") copy("bridge.management_room_multiple_messages") + copy("bridge.encryption.allow") + copy("bridge.encryption.default") + copy("bridge.encryption.key_sharing.allow") + copy("bridge.encryption.key_sharing.require_cross_signing") + copy("bridge.encryption.key_sharing.require_verification") + copy("bridge.encryption.rotation.enable_custom") + copy("bridge.encryption.rotation.milliseconds") + copy("bridge.encryption.rotation.messages") + copy("bridge.relay.enabled") copy_dict("bridge.relay.message_formats", override_existing_map=False) From 99644477325bc34d47ce1bfbb68c63af5ee80289 Mon Sep 17 00:00:00 2001 From: Sumner Evans Date: Tue, 21 Jun 2022 12:57:45 -0600 Subject: [PATCH 119/456] encryption: add ability to control rotation settings Signed-off-by: Sumner Evans --- mautrix/bridge/portal.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/mautrix/bridge/portal.py b/mautrix/bridge/portal.py index bfbd63dd..536c0415 100644 --- a/mautrix/bridge/portal.py +++ b/mautrix/bridge/portal.py @@ -14,9 +14,10 @@ import logging import time -from mautrix.appservice import DOUBLE_PUPPET_SOURCE_KEY, AppService, IntentAPI +from mautrix.appservice import AppService, IntentAPI from mautrix.errors import MatrixError, MatrixRequestError, MForbidden, MNotFound from mautrix.types import ( + JSON, EncryptionAlgorithm, EventID, EventType, @@ -321,6 +322,13 @@ async def check_dm_encryption(self) -> bool | None: return await self.enable_dm_encryption() return None + def get_encryption_state_event_json(self) -> JSON: + evt = RoomEncryptionStateEventContent(EncryptionAlgorithm.MEGOLM_V1) + if self.bridge.config["bridge.encryption.rotation.enable_custom"]: + evt.rotation_period_ms = self.bridge.config["bridge.encryption.rotation.milliseconds"] + evt.rotation_period_msgs = self.bridge.config["bridge.encryption.rotation.messages"] + return evt.serialize() + async def enable_dm_encryption(self) -> bool: self.log.debug("Inviting bridge bot to room for end-to-bridge encryption") try: @@ -330,7 +338,7 @@ async def enable_dm_encryption(self) -> bool: await self.main_intent.send_state_event( self.mxid, EventType.ROOM_ENCRYPTION, - RoomEncryptionStateEventContent(EncryptionAlgorithm.MEGOLM_V1), + self.get_encryption_state_event_json(), ) except Exception: self.log.warning(f"Failed to enable end-to-bridge encryption", exc_info=True) From 1e9d1359d9696e798f9e5082df300f921baffea9 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 22 Jun 2022 22:07:00 +0300 Subject: [PATCH 120/456] Bump version to 0.16.9 --- CHANGELOG.md | 7 +++++++ mautrix/__init__.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c086e812..24c0fc7e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## v0.16.9 (2022-06-22) + +* *(client)* Added support for knocking on rooms (thanks to [@maltee1] in [#105]). +* *(bridge)* Added config option to set key rotation settings with e2be. + +[#105]: https://github.com/mautrix/python/pull/105 + ## v0.16.8 (2022-06-20) * *(bridge)* Updated e2be helper to stop bridge if syncing fails. diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 8cf0801b..f0020d99 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.16.8" +__version__ = "0.16.9" __author__ = "Tulir Asokan " __all__ = [ "api", From e3d1e53f79a0463b9016b392a2d3de5c0e342271 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 23 Jun 2022 11:40:26 +0300 Subject: [PATCH 121/456] Fix typo --- mautrix/client/api/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/client/api/client.py b/mautrix/client/api/client.py index 51070a04..5c4c5338 100644 --- a/mautrix/client/api/client.py +++ b/mautrix/client/api/client.py @@ -24,7 +24,7 @@ class ClientAPI( functions for accessing the client-server API. This class can be used directly, but generally you should use the higher-level wrappers that - inherit from this class, such as :class:`mautrix.client.ClientAPI` + inherit from this class, such as :class:`mautrix.client.Client` or :class:`mautrix.appservice.IntentAPI`. Examples: From 77f51b372ef3b3359b4dfc495055509921bec2f1 Mon Sep 17 00:00:00 2001 From: Sumner Evans Date: Thu, 23 Jun 2022 14:21:36 -0600 Subject: [PATCH 122/456] portal: automatically mark read after sending message Signed-off-by: Sumner Evans Co-authored-by: Tulir Asokan --- mautrix/bridge/portal.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mautrix/bridge/portal.py b/mautrix/bridge/portal.py index 536c0415..0d4b110f 100644 --- a/mautrix/bridge/portal.py +++ b/mautrix/bridge/portal.py @@ -429,7 +429,10 @@ async def _send_message( ) -> EventID: if self.encrypted and self.matrix.e2ee: event_type, content = await self.matrix.e2ee.encrypt(self.mxid, event_type, content) - return await intent.send_message_event(self.mxid, event_type, content, **kwargs) + event_id = await intent.send_message_event(self.mxid, event_type, content, **kwargs) + if intent.api.is_real_user: + asyncio.create_task(intent.mark_read(self.mxid, event_id)) + return event_id @property @abstractmethod From 64e597bd7b5f7b5ea0ecf714e6fe58edb02f7bec Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 24 Jun 2022 13:05:13 +0300 Subject: [PATCH 123/456] Require Matrix v1.1 support in bridges --- mautrix/bridge/matrix.py | 16 ++++- mautrix/client/api/base.py | 4 +- mautrix/types/__init__.py | 7 +- mautrix/types/misc.py | 16 +---- mautrix/types/versions.py | 136 +++++++++++++++++++++++++++++++++++++ 5 files changed, 158 insertions(+), 21 deletions(-) create mode 100644 mautrix/types/versions.py diff --git a/mautrix/bridge/matrix.py b/mautrix/bridge/matrix.py index 84c2bf1b..075389f0 100644 --- a/mautrix/bridge/matrix.py +++ b/mautrix/bridge/matrix.py @@ -41,11 +41,13 @@ RoomID, RoomType, SingleReceiptEventContent, + SpecVersions, StateEvent, StateUnsigned, TextMessageEventContent, TypingEvent, UserID, + Version, VersionsResponse, ) from mautrix.util import markdown @@ -95,6 +97,7 @@ class BaseMatrixHandler: e2ee: EncryptionManager | None media_config: MediaRepoConfig versions: VersionsResponse + minimum_spec_version: Version = SpecVersions.V11 user_id_prefix: str user_id_suffix: str @@ -109,7 +112,7 @@ def __init__( self.bridge = bridge self.commands = command_processor or cmd.CommandProcessor(bridge=bridge) self.media_config = MediaRepoConfig(upload_size=50 * 1024 * 1024) - self.versions = VersionsResponse(versions=["v1.2"]) + self.versions = VersionsResponse.deserialize({"versions": ["v1.3"]}) self.az.matrix_event_handler(self.int_handle_event) self.e2ee = None @@ -149,6 +152,16 @@ def __init__( False, ) + async def check_versions(self) -> None: + if not self.versions.supports_at_least(self.minimum_spec_version): + self.log.fatal( + "Server isn't advertising modern spec versions " + "(latest supported by server: %s, minimum required by bridge: %s)", + self.versions.latest_version, + self.minimum_spec_version, + ) + sys.exit(18) + async def wait_for_connection(self) -> None: self.log.info("Ensuring connectivity to homeserver") errors = 0 @@ -156,6 +169,7 @@ async def wait_for_connection(self) -> None: while True: try: self.versions = await self.az.intent.versions() + await self.check_versions() await self.az.intent.whoami() break except (MUnknownToken, MExclusive): diff --git a/mautrix/client/api/base.py b/mautrix/client/api/base.py index f1c93eff..07659894 100644 --- a/mautrix/client/api/base.py +++ b/mautrix/client/api/base.py @@ -116,9 +116,7 @@ async def versions(self, no_cache: bool = False) -> VersionsResponse: """ if no_cache or not self.versions_cache: resp = await self.api.request(Method.GET, Path.versions) - vers = self.versions_cache = VersionsResponse.deserialize(resp) - if not vers.has_modern_versions and vers.has_legacy_versions: - self.log.warning("Server isn't advertising modern spec versions") + self.versions_cache = VersionsResponse.deserialize(resp) return self.versions_cache @classmethod diff --git a/mautrix/types/__init__.py b/mautrix/types/__init__.py index 4933596d..dd65a183 100644 --- a/mautrix/types/__init__.py +++ b/mautrix/types/__init__.py @@ -144,7 +144,6 @@ RoomCreatePreset, RoomDirectoryResponse, RoomDirectoryVisibility, - VersionsResponse, ) from .primitive import ( JSON, @@ -186,6 +185,7 @@ field, serializer, ) +from .versions import SpecVersions, Version, VersionFormat, VersionsResponse __all__ = [ "DiscoveryInformation", @@ -338,7 +338,6 @@ "RoomCreatePreset", "RoomDirectoryResponse", "RoomDirectoryVisibility", - "VersionsResponse", "JSON", "BatchID", "ContentURI", @@ -375,4 +374,8 @@ "deserializer", "field", "serializer", + "SpecVersions", + "Version", + "VersionFormat", + "VersionsResponse", ] diff --git a/mautrix/types/misc.py b/mautrix/types/misc.py index e880334a..87f5aa9a 100644 --- a/mautrix/types/misc.py +++ b/mautrix/types/misc.py @@ -3,7 +3,7 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from typing import Dict, List, NamedTuple, NewType, Optional +from typing import List, NamedTuple, NewType, Optional from enum import Enum from attr import dataclass @@ -107,20 +107,6 @@ class RoomDirectoryResponse(SerializableAttrs): ) -@dataclass -class VersionsResponse(SerializableAttrs): - versions: List[str] - unstable_features: Dict[str, bool] = attr.ib(factory=lambda: {}) - - @property - def has_legacy_versions(self) -> bool: - return any(v for v in self.versions if v.startswith("r0.")) - - @property - def has_modern_versions(self) -> bool: - return any(v for v in self.versions if v.startswith("v")) - - @dataclass class BatchSendResponse(SerializableAttrs): state_event_ids: List[EventID] diff --git a/mautrix/types/versions.py b/mautrix/types/versions.py new file mode 100644 index 00000000..8b42de39 --- /dev/null +++ b/mautrix/types/versions.py @@ -0,0 +1,136 @@ +# Copyright (c) 2022 Tulir Asokan +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +from typing import Dict, List, NamedTuple, Optional, Union +from enum import IntEnum +import re + +from attr import dataclass +import attr + +from . import JSON +from .util import Serializable, SerializableAttrs + + +class VersionFormat(IntEnum): + UNKNOWN = -1 + LEGACY = 0 + MODERN = 1 + + def __repr__(self) -> str: + return f"VersionFormat.{self.name}" + + +legacy_version_regex = re.compile(r"^r(\d+)\.(\d+)\.(\d+)$") +modern_version_regex = re.compile(r"^v(\d+)\.(\d+)$") + + +@attr.dataclass(frozen=True) +class Version(Serializable): + format: VersionFormat + major: int + minor: int + patch: int + raw: str + + def __str__(self) -> str: + if self.format == VersionFormat.MODERN: + return f"v{self.major}.{self.minor}" + elif self.format == VersionFormat.LEGACY: + return f"r{self.major}.{self.minor}.{self.patch}" + else: + return self.raw + + def serialize(self) -> JSON: + return str(self) + + @classmethod + def deserialize(cls, raw: JSON) -> "Version": + assert isinstance(raw, str), "versions must be strings" + if modern := modern_version_regex.fullmatch(raw): + major, minor = modern.groups() + return Version(VersionFormat.MODERN, int(major), int(minor), 0, raw) + elif legacy := legacy_version_regex.fullmatch(raw): + major, minor, patch = legacy.groups() + return Version(VersionFormat.LEGACY, int(major), int(minor), int(patch), raw) + else: + return Version(VersionFormat.UNKNOWN, 0, 0, 0, raw) + + +class SpecVersions: + R010 = Version.deserialize("r0.1.0") + R020 = Version.deserialize("r0.2.0") + R030 = Version.deserialize("r0.3.0") + R040 = Version.deserialize("r0.4.0") + R050 = Version.deserialize("r0.5.0") + R060 = Version.deserialize("r0.6.0") + R061 = Version.deserialize("r0.6.1") + V11 = Version.deserialize("v1.1") + V12 = Version.deserialize("v1.2") + V13 = Version.deserialize("v1.3") + + +@dataclass +class VersionsResponse(SerializableAttrs): + versions: List[Version] + unstable_features: Dict[str, bool] = attr.ib(factory=lambda: {}) + + def supports(self, thing: Union[Version, str]) -> Optional[bool]: + """ + Check if the versions response contains the given spec version or unstable feature. + + Args: + thing: The spec version (as a :class:`Version` or string) + or unstable feature name (as a string) to check. + + Returns: + ``True`` if the exact version or unstable feature is supported, + ``False`` if it's not supported, + ``None`` for unstable features which are not included in the response at all. + """ + if isinstance(thing, Version): + return thing in self.versions + elif (parsed_version := Version.deserialize(thing)).format != VersionFormat.UNKNOWN: + return parsed_version in self.versions + return self.unstable_features.get(thing) + + def supports_at_least(self, version: Union[Version, str]) -> bool: + """ + Check if the versions response contains the given spec version or any higher version. + + Args: + version: The spec version as a :class:`Version` or a string. + + Returns: + ``True`` if a version equal to or higher than the given version is found, + ``False`` otherwise. + """ + if isinstance(version, str): + version = Version.deserialize(version) + return any(v for v in self.versions if v > version) + + @property + def latest_version(self) -> Version: + return max(self.versions) + + @property + def has_legacy_versions(self) -> bool: + """ + Check if the response contains any legacy (r0.x.y) versions. + + .. deprecated:: 0.16.10 + :meth:`supports_at_least` and :meth:`supports` methods are now preferred. + """ + return any(v for v in self.versions if v.format == VersionFormat.LEGACY) + + @property + def has_modern_versions(self) -> bool: + """ + Check if the response contains any modern (v1.1 or higher) versions. + + .. deprecated:: 0.16.10 + :meth:`supports_at_least` and :meth:`supports` methods are now preferred. + """ + return self.supports_at_least(SpecVersions.V11) From 59baa23001d129f8f23a4ecca11db2a2482e3e3d Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 24 Jun 2022 19:41:38 +0300 Subject: [PATCH 124/456] Bump version to 0.16.10 --- CHANGELOG.md | 6 ++++++ mautrix/__init__.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 24c0fc7e..3b225f6a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## v0.16.10 (2022-06-24) + +* *(bridge)* Started requiring Matrix v1.1 support from homeservers. +* *(bridge)* Added hack to automatically send a read receipt for messages sent + to Matrix with double puppeting (to work around weird unread count issues). + ## v0.16.9 (2022-06-22) * *(client)* Added support for knocking on rooms (thanks to [@maltee1] in [#105]). diff --git a/mautrix/__init__.py b/mautrix/__init__.py index f0020d99..ab57635e 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.16.9" +__version__ = "0.16.10" __author__ = "Tulir Asokan " __all__ = [ "api", From 547113decbc3e6cf471827b1e35d91a2e8d210af Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 28 Jun 2022 19:24:01 +0300 Subject: [PATCH 125/456] Disable ensure_joined in send_member_event --- mautrix/appservice/api/intent.py | 15 +++++++++++---- mautrix/client/api/events.py | 3 +++ mautrix/client/api/rooms.py | 2 +- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/mautrix/appservice/api/intent.py b/mautrix/appservice/api/intent.py index e2f3f804..3fd1600b 100644 --- a/mautrix/appservice/api/intent.py +++ b/mautrix/appservice/api/intent.py @@ -134,7 +134,9 @@ async def wrapper(*args, __self=self, __method=method, **kwargs): room_id = kwargs.get("room_id", None) if not room_id: room_id = args[0] - await __self.ensure_joined(room_id) + ensure_joined = kwargs.pop("ensure_joined", True) + if ensure_joined: + await __self.ensure_joined(room_id) return await __method(*args, **kwargs) setattr(self, method.__name__, wrapper) @@ -316,9 +318,10 @@ def set_room_topic(self, room_id: RoomID, topic: str, **kwargs) -> Awaitable[Eve return self.send_state_event(room_id, EventType.ROOM_TOPIC, content, **kwargs) async def get_power_levels( - self, room_id: RoomID, ignore_cache: bool = False + self, room_id: RoomID, ignore_cache: bool = False, ensure_joined: bool = True ) -> PowerLevelStateEventContent: - await self.ensure_joined(room_id) + if ensure_joined: + await self.ensure_joined(room_id) if not ignore_cache: levels = await self.state_store.get_power_levels(room_id) if levels: @@ -327,6 +330,10 @@ async def get_power_levels( levels = await self.get_state_event(room_id, EventType.ROOM_POWER_LEVELS) except MNotFound: levels = PowerLevelStateEventContent() + except MForbidden: + if not ensure_joined: + return PowerLevelStateEventContent() + raise await self.state_store.set_power_levels(room_id, levels) return levels @@ -623,7 +630,7 @@ async def _ensure_has_power_level_for( return if not await self.state_store.has_power_levels_cached(room_id): # TODO add option to not try to fetch power levels from server - await self.get_power_levels(room_id, ignore_cache=True) + await self.get_power_levels(room_id, ignore_cache=True, ensure_joined=False) if not await self.state_store.has_power_level(room_id, self.mxid, event_type): # TODO implement something better raise IntentError( diff --git a/mautrix/client/api/events.py b/mautrix/client/api/events.py index 8839edfe..0b03e167 100644 --- a/mautrix/client/api/events.py +++ b/mautrix/client/api/events.py @@ -333,6 +333,7 @@ async def send_state_event( event_type: EventType, content: StateEventContent, state_key: str = "", + ensure_joined: bool = True, **kwargs, ) -> EventID: """ @@ -346,6 +347,8 @@ async def send_state_event( event_type: The type of state to send. content: The content to send. state_key: The key for the state to send. Defaults to empty string. + ensure_joined: Used by IntentAPI to determine if it should ensure the user is joined + before sending the event. **kwargs: Optional parameters to pass to the :meth:`HTTPAPI.request` method. Used by :class:`IntentAPI` to pass the timestamp massaging field to :meth:`AppServiceAPI.request`. diff --git a/mautrix/client/api/rooms.py b/mautrix/client/api/rooms.py index dbcc6852..726d1153 100644 --- a/mautrix/client/api/rooms.py +++ b/mautrix/client/api/rooms.py @@ -391,7 +391,7 @@ async def send_member_event( content[key] = value content = await self.fill_member_event(room_id, user_id, content) or content return await self.send_state_event( - room_id, EventType.ROOM_MEMBER, content=content, state_key=user_id + room_id, EventType.ROOM_MEMBER, content=content, state_key=user_id, ensure_joined=False ) async def invite_user( From 81fa21e7438d77060e580c6b6ec98c6f00c253be Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 28 Jun 2022 19:35:02 +0300 Subject: [PATCH 126/456] Bump version to 0.16.11 --- CHANGELOG.md | 5 +++++ mautrix/__init__.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b225f6a..3a2db57d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +## v0.16.11 (2022-06-28) + +* *(appservice)* Fixed the `extra_content` parameter in membership methods + causing duplicate join events through the `ensure_joined` mechanism. + ## v0.16.10 (2022-06-24) * *(bridge)* Started requiring Matrix v1.1 support from homeservers. diff --git a/mautrix/__init__.py b/mautrix/__init__.py index ab57635e..20dd361e 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.16.10" +__version__ = "0.16.11" __author__ = "Tulir Asokan " __all__ = [ "api", From 639fbf56deee7c54a4422abd281f688c0f60ddc2 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 29 Jun 2022 14:33:03 +0300 Subject: [PATCH 127/456] Make from_token optional in get_messages --- mautrix/client/api/events.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/mautrix/client/api/events.py b/mautrix/client/api/events.py index 0b03e167..7065146c 100644 --- a/mautrix/client/api/events.py +++ b/mautrix/client/api/events.py @@ -263,7 +263,7 @@ async def get_messages( self, room_id: RoomID, direction: PaginationDirection, - from_token: SyncToken, + from_token: SyncToken | None = None, to_token: SyncToken | None = None, limit: int | None = None, filter_json: str | dict | RoomEventFilter | None = None, @@ -272,7 +272,7 @@ async def get_messages( Get a list of message and state events for a room. Pagination parameters are used to paginate history in the room. - See also: `API reference `__ + See also: `API reference `__ Args: room_id: The ID of the room to get events from. @@ -280,6 +280,9 @@ async def get_messages( from_token: The token to start returning events from. This token can be obtained from a ``prev_batch`` token returned for each room by the `sync endpoint`_, or from a ``start`` or ``end`` token returned by a previous request to this endpoint. + + Starting from Matrix v1.3, this field can be omitted to fetch events from the + beginning or end of the room. to_token: The token to stop returning events at. limit: The maximum number of events to return. Defaults to 10. filter_json: A JSON RoomEventFilter_ to filter returned events with. @@ -287,9 +290,9 @@ async def get_messages( Returns: .. _RoomEventFilter: - https://spec.matrix.org/v1.1/client-server-api/#filtering + https://spec.matrix.org/v1.3/client-server-api/#filtering .. _sync endpoint: - https://spec.matrix.org/v1.1/client-server-api/#get_matrixclientv3sync + https://spec.matrix.org/v1.3/client-server-api/#get_matrixclientv3sync """ if isinstance(filter_json, Serializable): filter_json = filter_json.json() From 007828f9ad510f28a639038675140776c400cb76 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 1 Jul 2022 18:15:49 +0300 Subject: [PATCH 128/456] Add cross-signing key store --- mautrix/crypto/store/abstract.py | 71 +++++++++++++++++++++++++ mautrix/crypto/store/asyncpg/store.py | 70 ++++++++++++++++++++++++ mautrix/crypto/store/asyncpg/upgrade.py | 44 ++++++++++++++- mautrix/crypto/store/memory.py | 37 +++++++++++++ mautrix/types/__init__.py | 3 ++ mautrix/types/crypto.py | 32 ++++++++++- 6 files changed, 254 insertions(+), 3 deletions(-) diff --git a/mautrix/crypto/store/abstract.py b/mautrix/crypto/store/abstract.py index 85c99f2f..78691989 100644 --- a/mautrix/crypto/store/abstract.py +++ b/mautrix/crypto/store/abstract.py @@ -5,9 +5,12 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import annotations +from typing import NamedTuple from abc import ABC, abstractmethod from mautrix.types import ( + CrossSigner, + CrossSigningUsage, DeviceID, DeviceIdentity, EventID, @@ -15,6 +18,8 @@ RoomEncryptionStateEventContent, RoomID, SessionID, + SigningKey, + TOFUSigningKey, UserID, ) @@ -364,3 +369,69 @@ async def filter_tracked_users(self, users: list[UserID]) -> list[UserID]: A filtered version of the input list that only includes users who have had a previous call to :meth:`put_devices` (even if the call was with an empty dict). """ + + @abstractmethod + async def put_cross_signing_key( + self, user_id: UserID, usage: CrossSigningUsage, key: SigningKey + ) -> None: + """ + Store a single cross-signing key. + + Args: + user_id: The user whose cross-signing key is being stored. + usage: The type of key being stored. + key: The key itself. + """ + + @abstractmethod + async def get_cross_signing_keys( + self, user_id: UserID + ) -> dict[CrossSigningUsage, TOFUSigningKey]: + """ + Retrieve stored cross-signing keys for a specific user. + + Args: + user_id: The user whose cross-signing keys to get. + + Returns: + A map from the type of key to a tuple containing the current key and the key that was + seen first. If the keys are different, it should be treated as a local TOFU violation. + """ + + @abstractmethod + async def put_signature( + self, target: CrossSigner, signer: CrossSigner, signature: str + ) -> None: + """ + Store a signature for a given key from a given key. + + Args: + target: The user ID and key being signed. + signer: The user ID and key who are doing the signing. + signature: The signature. + """ + + @abstractmethod + async def is_key_signed_by(self, target: CrossSigner, signer: CrossSigner) -> bool: + """ + Check if a given key is signed by the given signer. + + Args: + target: The key to check. + signer: The signer who is expected to have signed the key. + + Returns: + ``True`` if the database contains a signature for the key, ``False`` otherwise. + """ + + @abstractmethod + async def drop_signatures_by_key(self, signer: CrossSigner) -> int: + """ + Delete signatures made by the given key. + + Args: + signer: The key whose signatures to delete. + + Returns: + The number of signatures deleted. + """ diff --git a/mautrix/crypto/store/asyncpg/store.py b/mautrix/crypto/store/asyncpg/store.py index cb9b232c..3556137b 100644 --- a/mautrix/crypto/store/asyncpg/store.py +++ b/mautrix/crypto/store/asyncpg/store.py @@ -11,13 +11,17 @@ from mautrix.client.state_store import SyncStore from mautrix.client.state_store.asyncpg import PgStateStore from mautrix.types import ( + CrossSigner, + CrossSigningUsage, DeviceID, DeviceIdentity, EventID, IdentityKey, RoomID, SessionID, + SigningKey, SyncToken, + TOFUSigningKey, TrustState, UserID, ) @@ -28,6 +32,11 @@ from ..abstract import CryptoStore, StateStore from .upgrade import upgrade_table +try: + from aiosqlite import Cursor +except ImportError: + Cursor = None + class PgCryptoStateStore(PgStateStore, StateStore): """ @@ -522,3 +531,64 @@ async def filter_tracked_users(self, users: list[UserID]) -> list[UserID]: f"SELECT user_id FROM crypto_tracked_user WHERE user_id IN ({params})", *users ) return [row["user_id"] for row in rows] + + async def put_cross_signing_key( + self, user_id: UserID, usage: CrossSigningUsage, key: SigningKey + ) -> None: + q = """ + INSERT INTO crypto_cross_signing_keys (user_id, usage, key, first_seen_key) + VALUES ($1, $2, $3, $4) + ON CONFLICT (user_id, usage) DO UPDATE SET key=excluded.key + """ + await self.db.execute(q, user_id, usage.value, key, key) + + async def get_cross_signing_keys( + self, user_id: UserID + ) -> dict[CrossSigningUsage, TOFUSigningKey]: + q = "SELECT usage, key, first_seen_key FROM crypto_cross_signing_keys WHERE user_id=$1" + return { + CrossSigningUsage(row["usage"]): TOFUSigningKey( + key=SigningKey(row["key"]), + first=SigningKey(row["first_seen_key"]), + ) + for row in await self.db.fetch(q, user_id) + } + + async def put_signature( + self, target: CrossSigner, signer: CrossSigner, signature: str + ) -> None: + q = """ + INSERT INTO crypto_cross_signing_signatures ( + signed_user_id, signed_key, signer_user_id, signer_key, signature + ) VALUES ($1, $2, $3, $4, $5) + ON CONFLICT (signed_user_id, signed_key, signer_user_id, signer_key) + DO UPDATE SET signature=excluded.signature + """ + signed_user_id, signed_key = target + signer_user_id, signer_key = signer + await self.db.execute(q, signed_user_id, signed_key, signer_user_id, signer_key, signature) + + async def is_key_signed_by(self, target: CrossSigner, signer: CrossSigner) -> bool: + q = """ + SELECT EXISTS( + SELECT 1 FROM crypto_cross_signing_signatures + WHERE signed_user_id=$1 AND signed_key=$2 AND signer_user_id=$3 AND signer_key=$4 + ) + """ + signed_user_id, signed_key = target + signer_user_id, signer_key = signer + return await self.db.fetchval(q, signed_user_id, signed_key, signer_user_id, signer_key) + + async def drop_signatures_by_key(self, signer: CrossSigner) -> int: + signer_user_id, signer_key = signer + q = "DELETE FROM crypto_cross_signing_signatures WHERE signer_user_id=$1 AND signer_key=$2" + res = await self.db.execute(q, signer_user_id, signer_key) + if Cursor is not None and isinstance(res, Cursor): + return res.rowcount + elif ( + isinstance(res, str) + and res.startswith("DELETE ") + and (intPart := res[len("DELETE ") :]).isdecimal() + ): + return int(intPart) + return -1 diff --git a/mautrix/crypto/store/asyncpg/upgrade.py b/mautrix/crypto/store/asyncpg/upgrade.py index 12b1ec54..12baf9f9 100644 --- a/mautrix/crypto/store/asyncpg/upgrade.py +++ b/mautrix/crypto/store/asyncpg/upgrade.py @@ -16,7 +16,7 @@ ) -@upgrade_table.register(description="Latest revision", upgrades_to=4) +@upgrade_table.register(description="Latest revision", upgrades_to=5) async def upgrade_blank_to_v4(conn: Connection) -> None: await conn.execute( """CREATE TABLE IF NOT EXISTS crypto_account ( @@ -93,6 +93,25 @@ async def upgrade_blank_to_v4(conn: Connection) -> None: PRIMARY KEY (account_id, room_id) )""" ) + await conn.execute( + """CREATE TABLE crypto_cross_signing_keys ( + user_id TEXT, + usage TEXT, + key CHAR(43), + first_seen_key CHAR(43), + PRIMARY KEY (user_id, usage) + )""" + ) + await conn.execute( + """CREATE TABLE crypto_cross_signing_signatures ( + signed_user_id TEXT, + signed_key TEXT, + signer_user_id TEXT, + signer_key TEXT, + signature TEXT, + PRIMARY KEY (signed_user_id, signed_key, signer_user_id, signer_key) + )""" + ) @upgrade_table.register(description="Add account_id primary key column") @@ -201,3 +220,26 @@ async def upgrade_v4(conn: Connection, scheme: Scheme) -> None: await conn.execute( "ALTER TABLE crypto_olm_session ALTER COLUMN last_encrypted SET NOT NULL" ) + + +@upgrade_table.register(description="Add cross-signing key and signature caches") +async def upgrade_v5(conn: Connection) -> None: + await conn.execute( + """CREATE TABLE crypto_cross_signing_keys ( + user_id TEXT, + usage TEXT, + key CHAR(43), + first_seen_key CHAR(43), + PRIMARY KEY (user_id, usage) + )""" + ) + await conn.execute( + """CREATE TABLE crypto_cross_signing_signatures ( + signed_user_id TEXT, + signed_key TEXT, + signer_user_id TEXT, + signer_key TEXT, + signature TEXT, + PRIMARY KEY (signed_user_id, signed_key, signer_user_id, signer_key) + )""" + ) diff --git a/mautrix/crypto/store/memory.py b/mautrix/crypto/store/memory.py index 4aef397a..ab7c492e 100644 --- a/mautrix/crypto/store/memory.py +++ b/mautrix/crypto/store/memory.py @@ -7,13 +7,17 @@ from mautrix.client.state_store import SyncStore from mautrix.types import ( + CrossSigner, + CrossSigningUsage, DeviceID, DeviceIdentity, EventID, IdentityKey, RoomID, SessionID, + SigningKey, SyncToken, + TOFUSigningKey, UserID, ) @@ -31,6 +35,8 @@ class MemoryCryptoStore(CryptoStore, SyncStore): _olm_sessions: dict[IdentityKey, list[Session]] _inbound_sessions: dict[tuple[RoomID, IdentityKey, SessionID], InboundGroupSession] _outbound_sessions: dict[RoomID, OutboundGroupSession] + _signatures: dict[CrossSigner, dict[CrossSigner, str]] + _cross_signing_keys: dict[UserID, dict[CrossSigningUsage, TOFUSigningKey]] def __init__(self, account_id: str, pickle_key: str) -> None: self.account_id = account_id @@ -44,6 +50,8 @@ def __init__(self, account_id: str, pickle_key: str) -> None: self._olm_sessions = {} self._inbound_sessions = {} self._outbound_sessions = {} + self._signatures = {} + self._cross_signing_keys = {} async def get_device_id(self) -> DeviceID | None: return self._device_id @@ -157,3 +165,32 @@ async def put_devices(self, user_id: UserID, devices: dict[DeviceID, DeviceIdent async def filter_tracked_users(self, users: list[UserID]) -> list[UserID]: return [user_id for user_id in users if user_id in self._devices] + + async def put_cross_signing_key( + self, user_id: UserID, usage: CrossSigningUsage, key: SigningKey + ) -> None: + try: + current = self._cross_signing_keys[user_id][usage] + except KeyError: + self._cross_signing_keys.setdefault(user_id, {})[usage] = TOFUSigningKey( + key=key, first=key + ) + else: + current.key = key + + async def get_cross_signing_keys( + self, user_id: UserID + ) -> dict[CrossSigningUsage, TOFUSigningKey]: + return self._cross_signing_keys.get(user_id, {}) + + async def put_signature( + self, target: CrossSigner, signer: CrossSigner, signature: str + ) -> None: + self._signatures.setdefault(signer, {})[target] = signature + + async def is_key_signed_by(self, target: CrossSigner, signer: CrossSigner) -> bool: + return target in self._signatures.get(signer, {}) + + async def drop_signatures_by_key(self, signer: CrossSigner) -> int: + deleted = self._signatures.pop(signer, None) + return len(deleted) diff --git a/mautrix/types/__init__.py b/mautrix/types/__init__.py index dd65a183..8301d237 100644 --- a/mautrix/types/__init__.py +++ b/mautrix/types/__init__.py @@ -16,11 +16,14 @@ ) from .crypto import ( ClaimKeysResponse, + CrossSigner, + CrossSigningUsage, DecryptedOlmEvent, DeviceIdentity, DeviceKeys, OlmEventKeys, QueryKeysResponse, + TOFUSigningKey, TrustState, UnsignedDeviceInfo, ) diff --git a/mautrix/types/crypto.py b/mautrix/types/crypto.py index 90c3f830..5bfe99d2 100644 --- a/mautrix/types/crypto.py +++ b/mautrix/types/crypto.py @@ -3,14 +3,14 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, NamedTuple, Optional from enum import IntEnum from attr import dataclass from .event import EncryptionAlgorithm, EncryptionKeyAlgorithm, ToDeviceEvent from .primitive import DeviceID, IdentityKey, SigningKey, UserID -from .util import SerializableAttrs, field +from .util import ExtensibleEnum, SerializableAttrs, field @dataclass @@ -89,3 +89,31 @@ class DecryptedOlmEvent(ToDeviceEvent, SerializableAttrs): recipient_keys: OlmEventKeys sender_device: Optional[DeviceID] = None sender_key: IdentityKey = field(hidden=True, default=None) + + +class CrossSigningUsage(ExtensibleEnum): + MASTER = "master" + SELF = "self_signing" + USER = "user_signing" + + +class TOFUSigningKey(NamedTuple): + """ + A tuple representing a single cross-signing key. The first value is the current key, and the + second value is the first seen key. If the values don't match, it means the key is not valid + for trust-on-first-use. + """ + + key: SigningKey + first: SigningKey + + +class CrossSigner(NamedTuple): + """ + A tuple containing a user ID and a signing key they own. + + The key can either be a device-owned signing key, or one of the user's cross-signing keys. + """ + + user_id: UserID + key: SigningKey From fe6eea2284110aae0ba00d60920fa8c8af1dcf41 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 1 Jul 2022 20:22:34 +0300 Subject: [PATCH 129/456] Add methods for validating cross-signing signatures of other users --- mautrix/crypto/base.py | 19 +++-- mautrix/crypto/device_lists.py | 108 ++++++++++++++++++++++-- mautrix/crypto/machine.py | 6 +- mautrix/crypto/store/asyncpg/upgrade.py | 7 ++ mautrix/types/__init__.py | 9 ++ mautrix/types/crypto.py | 54 ++++++++---- mautrix/types/event/__init__.py | 1 + mautrix/types/event/encrypted.py | 18 ++++ mautrix/types/primitive.py | 2 + 9 files changed, 191 insertions(+), 33 deletions(-) diff --git a/mautrix/crypto/base.py b/mautrix/crypto/base.py index e1f65a6d..016c991f 100644 --- a/mautrix/crypto/base.py +++ b/mautrix/crypto/base.py @@ -5,7 +5,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import annotations -from typing import Any, Awaitable, Callable, Dict, TypedDict +from typing import Any, Awaitable, Callable, TypedDict import asyncio import functools import json @@ -16,10 +16,12 @@ DeviceID, EncryptionKeyAlgorithm, IdentityKey, + KeyID, RequestedKeyInfo, RoomID, SessionID, SigningKey, + TrustState, UserID, ) from mautrix.util.logging import TraceLogger @@ -28,7 +30,7 @@ class SignedObject(TypedDict): - signatures: Dict[UserID, Dict[str, str]] + signatures: dict[UserID, dict[str, str]] unsigned: Any @@ -45,11 +47,13 @@ class BaseOlmMachine: allow_key_share: Callable[[crypto.DeviceIdentity, RequestedKeyInfo], Awaitable[bool]] # Futures that wait for responses to a key request - _key_request_waiters: Dict[SessionID, asyncio.Future] + _key_request_waiters: dict[SessionID, asyncio.Future] # Futures that wait for a session to be received (either normally or through a key request) - _inbound_session_waiters: Dict[SessionID, asyncio.Future] + _inbound_session_waiters: dict[SessionID, asyncio.Future] - _prev_unwedge: Dict[IdentityKey, float] + _prev_unwedge: dict[IdentityKey, float] + _fetch_keys_lock: asyncio.Lock + _cs_fetch_attempted: set[UserID] async def wait_for_session( self, room_id: RoomID, sender_key: IdentityKey, session_id: SessionID, timeout: float = 3 @@ -77,12 +81,13 @@ def _mark_session_received(self, session_id: SessionID) -> None: def verify_signature_json( - data: "SignedObject", user_id: UserID, device_id: DeviceID, key: SigningKey + data: "SignedObject", user_id: UserID, key_name: DeviceID | str, key: SigningKey ) -> bool: data_copy = {**data} data_copy.pop("unsigned", None) signatures = data_copy.pop("signatures") - signature = signatures[user_id][f"{EncryptionKeyAlgorithm.ED25519}:{device_id}"] + key_id = str(KeyID(EncryptionKeyAlgorithm.ED25519, key_name)) + signature = signatures[user_id][key_id] signed_data = canonical_json(data_copy) try: olm.ed25519_verify(key, signed_data, signature) diff --git a/mautrix/crypto/device_lists.py b/mautrix/crypto/device_lists.py index b5e4cc03..5757b7d9 100644 --- a/mautrix/crypto/device_lists.py +++ b/mautrix/crypto/device_lists.py @@ -7,10 +7,15 @@ from mautrix.errors import DeviceValidationError from mautrix.types import ( + CrossSigner, + CrossSigningUsage, DeviceID, DeviceIdentity, DeviceKeys, + EncryptionKeyAlgorithm, IdentityKey, + KeyID, + SigningKey, SyncToken, TrustState, UserID, @@ -30,13 +35,13 @@ async def _fetch_keys( users = set(users) self.log.trace(f"Querying keys for {users}") - keys = await self.client.query_keys(users, token=since) + resp = await self.client.query_keys(users, token=since) - for server, err in keys.failures.items(): + for server, err in resp.failures.items(): self.log.warning(f"Query keys failure for {server}: {err}") data = {} - for user_id, devices in keys.device_keys.items(): + for user_id, devices in resp.device_keys.items(): users.remove(user_id) new_devices = {} @@ -47,20 +52,26 @@ async def _fetch_keys( f"have {len(existing_devices)} in store" ) changed = False - for device_id, keys in devices.items(): + ssk = resp.self_signing_keys.get(user_id) + for device_id, device_keys in devices.items(): try: existing = existing_devices[device_id] except KeyError: existing = None changed = True - self.log.trace(f"Validating device {keys} of {user_id}") + self.log.trace(f"Validating device {device_keys} of {user_id}") try: - new_device = await self._validate_device(user_id, device_id, keys, existing) + new_device = await self._validate_device( + user_id, device_id, device_keys, existing + ) except DeviceValidationError as e: self.log.warning(f"Failed to validate device {device_id} of {user_id}: {e}") else: if new_device: new_devices[device_id] = new_device + await self._store_device_self_signatures( + device_keys, ssk.first_key if ssk else None + ) self.log.debug( f"Storing new device list for {user_id} containing {len(new_devices)} devices" ) @@ -75,6 +86,50 @@ async def _fetch_keys( return data + async def _store_device_self_signatures( + self, device_keys: DeviceKeys, self_signing_key: SigningKey | None + ) -> None: + device_desc = f"Device {device_keys.user_id}/{device_keys.device_id}" + try: + self_signatures = device_keys.signatures[device_keys.user_id].copy() + except KeyError: + self.log.warning(f"{device_desc} doesn't have any signatures from the user") + return + if len(device_keys.signatures) > 1: + self.log.warning( + f"{device_desc} has signatures from other users (%s)", + set(device_keys.signatures.keys()) - {device_keys.user_id}, + ) + + device_self_sig = self_signatures.pop( + KeyID(EncryptionKeyAlgorithm.ED25519, device_keys.device_id) + ) + target = CrossSigner(device_keys.user_id, device_keys.ed25519) + # This one is already validated by _validate_device + await self.crypto_store.put_signature(target, target, device_self_sig) + + try: + cs_self_sig = self_signatures.pop( + KeyID(EncryptionKeyAlgorithm.ED25519, self_signing_key) + ) + except KeyError: + self.log.warning(f"{device_desc} isn't cross-signed") + else: + is_valid_self_sig = verify_signature_json( + device_keys.serialize(), device_keys.user_id, self_signing_key, self_signing_key + ) + if is_valid_self_sig: + signer = CrossSigner(device_keys.user_id, self_signing_key) + await self.crypto_store.put_signature(target, signer, cs_self_sig) + else: + self.log.warning(f"{device_desc} doesn't have a valid cross-signing signature") + + if len(self_signatures) > 0: + self.log.warning( + f"{device_desc} has signatures from unexpected keys (%s)", + set(self_signatures.keys()), + ) + async def get_or_fetch_device( self, user_id: UserID, device_id: DeviceID ) -> Optional[DeviceIdentity]: @@ -142,3 +197,44 @@ async def _validate_device( name=name, deleted=False, ) + + async def resolve_trust(self, device: DeviceIdentity) -> TrustState: + if device.trust in (TrustState.VERIFIED, TrustState.BLACKLISTED): + return device.trust + their_keys = await self.crypto_store.get_cross_signing_keys(device.user_id) + if len(their_keys) == 0 and device.user_id not in self._cs_fetch_attempted: + self.log.debug(f"Didn't find any cross-signing keys for {device.user_id}, fetching...") + async with self._fetch_keys_lock: + if device.user_id not in self._cs_fetch_attempted: + self._cs_fetch_attempted.add(device.user_id) + await self._fetch_keys([device.user_id]) + try: + msk = their_keys[CrossSigningUsage.MASTER] + ssk = their_keys[CrossSigningUsage.SELF] + except KeyError as e: + self.log.error(f"Didn't find cross-signing key {e.args[0]} of {device.user_id}") + return TrustState.UNSET + ssk_signed = await self.crypto_store.is_key_signed_by( + target=CrossSigner(device.user_id, ssk.key), + signer=CrossSigner(device.user_id, msk.key), + ) + if not ssk_signed: + self.log.warning( + f"Self-signing key of {device.user_id} is not signed by their master key" + ) + return TrustState.UNSET + device_signed = await self.crypto_store.is_key_signed_by( + target=CrossSigner(device.user_id, device.signing_key), + signer=CrossSigner(device.user_id, ssk.key), + ) + if device_signed: + if await self.is_user_trusted(device.user_id): + return TrustState.CROSS_SIGNED_TRUSTED + elif msk.key == msk.first: + return TrustState.CROSS_SIGNED_TOFU + return TrustState.CROSS_SIGNED_UNTRUSTED + return TrustState.UNSET + + async def is_user_trusted(self, user_id: UserID) -> bool: + # TODO implement once own cross-signing key stuff is ready + return False diff --git a/mautrix/crypto/machine.py b/mautrix/crypto/machine.py index 8268ecc3..97a0bac8 100644 --- a/mautrix/crypto/machine.py +++ b/mautrix/crypto/machine.py @@ -19,6 +19,7 @@ Membership, StateEvent, ToDeviceEvent, + TrustState, ) from mautrix.util.logging import TraceLogger @@ -50,12 +51,8 @@ class OlmMachine( crypto_store: CryptoStore state_store: StateStore - _fetch_keys_lock: asyncio.Lock - account: Optional[OlmAccount] - allow_unverified_devices: bool - def __init__( self, client: cli.Client, @@ -78,6 +75,7 @@ def __init__( self._key_request_waiters = {} self._inbound_session_waiters = {} self._prev_unwedge = {} + self._cs_fetch_attempted = set() self.client.add_event_handler( cli.InternalEventType.DEVICE_OTK_COUNT, self.handle_otk_count, wait_sync=True diff --git a/mautrix/crypto/store/asyncpg/upgrade.py b/mautrix/crypto/store/asyncpg/upgrade.py index 12baf9f9..f7f50a74 100644 --- a/mautrix/crypto/store/asyncpg/upgrade.py +++ b/mautrix/crypto/store/asyncpg/upgrade.py @@ -243,3 +243,10 @@ async def upgrade_v5(conn: Connection) -> None: PRIMARY KEY (signed_user_id, signed_key, signer_user_id, signer_key) )""" ) + + +@upgrade_table.register(description="Update trust state values") +async def upgrade_v6(conn: Connection) -> None: + await conn.execute("UPDATE crypto_device SET trust=300 WHERE trust=1") # verified + await conn.execute("UPDATE crypto_device SET trust=-100 WHERE trust=2") # blacklisted + await conn.execute("UPDATE crypto_device SET trust=0 WHERE trust=3") # ignored -> unset diff --git a/mautrix/types/__init__.py b/mautrix/types/__init__.py index 8301d237..7e134626 100644 --- a/mautrix/types/__init__.py +++ b/mautrix/types/__init__.py @@ -17,6 +17,7 @@ from .crypto import ( ClaimKeysResponse, CrossSigner, + CrossSigningKeys, CrossSigningUsage, DecryptedOlmEvent, DeviceIdentity, @@ -75,6 +76,7 @@ JoinRule, JoinRulesStateEventContent, JSONWebKey, + KeyID, KeyRequestAction, LocationInfo, LocationMessageEventContent, @@ -159,6 +161,7 @@ RoomAlias, RoomID, SessionID, + Signature, SigningKey, SyncToken, UserID, @@ -206,11 +209,15 @@ "UserIdentifierType", "WhoamiResponse", "ClaimKeysResponse", + "CrossSigner", + "CrossSigningKeys", + "CrossSigningUsage", "DecryptedOlmEvent", "DeviceIdentity", "DeviceKeys", "OlmEventKeys", "QueryKeysResponse", + "TOFUSigningKey", "TrustState", "UnsignedDeviceInfo", "AccountDataEvent", @@ -260,6 +267,7 @@ "JoinRule", "JoinRulesStateEventContent", "JSONWebKey", + "KeyID", "KeyRequestAction", "LocationInfo", "LocationMessageEventContent", @@ -351,6 +359,7 @@ "RoomAlias", "RoomID", "SessionID", + "Signature", "SigningKey", "SyncToken", "UserID", diff --git a/mautrix/types/crypto.py b/mautrix/types/crypto.py index 5bfe99d2..a4350e73 100644 --- a/mautrix/types/crypto.py +++ b/mautrix/types/crypto.py @@ -8,8 +8,8 @@ from attr import dataclass -from .event import EncryptionAlgorithm, EncryptionKeyAlgorithm, ToDeviceEvent -from .primitive import DeviceID, IdentityKey, SigningKey, UserID +from .event import EncryptionAlgorithm, EncryptionKeyAlgorithm, KeyID, ToDeviceEvent +from .primitive import DeviceID, IdentityKey, Signature, SigningKey, UserID from .util import ExtensibleEnum, SerializableAttrs, field @@ -23,8 +23,8 @@ class DeviceKeys(SerializableAttrs): user_id: UserID device_id: DeviceID algorithms: List[EncryptionAlgorithm] - keys: Dict[str, str] - signatures: Dict[UserID, Dict[str, str]] + keys: Dict[KeyID, str] + signatures: Dict[UserID, Dict[KeyID, Signature]] unsigned: UnsignedDeviceInfo = None def __attrs_post_init__(self) -> None: @@ -34,35 +34,63 @@ def __attrs_post_init__(self) -> None: @property def ed25519(self) -> Optional[SigningKey]: try: - return SigningKey(self.keys[f"{EncryptionKeyAlgorithm.ED25519}:{self.device_id}"]) + return SigningKey(self.keys[KeyID(EncryptionKeyAlgorithm.ED25519, self.device_id)]) except KeyError: return None @property def curve25519(self) -> Optional[IdentityKey]: try: - return IdentityKey(self.keys[f"{EncryptionKeyAlgorithm.CURVE25519}:{self.device_id}"]) + return IdentityKey(self.keys[KeyID(EncryptionKeyAlgorithm.CURVE25519, self.device_id)]) except KeyError: return None +class CrossSigningUsage(ExtensibleEnum): + MASTER = "master" + SELF = "self_signing" + USER = "user_signing" + + +@dataclass +class CrossSigningKeys(SerializableAttrs): + user_id: UserID + usage: List[CrossSigningUsage] + keys: Dict[str, SigningKey] + signatures: Dict[UserID, Dict[KeyID, Signature]] = field(factory=lambda: {}) + + @property + def first_key(self) -> Optional[SigningKey]: + try: + return next(iter(self.keys.values())) + except StopIteration: + return None + + @dataclass class QueryKeysResponse(SerializableAttrs): failures: Dict[str, Any] device_keys: Dict[UserID, Dict[DeviceID, DeviceKeys]] + master_keys: Dict[UserID, CrossSigningKeys] + self_signing_keys: Dict[UserID, CrossSigningKeys] + user_signing_keys: Dict[UserID, CrossSigningKeys] @dataclass class ClaimKeysResponse(SerializableAttrs): failures: Dict[str, Any] - one_time_keys: Dict[UserID, Dict[DeviceID, Dict[str, Any]]] + one_time_keys: Dict[UserID, Dict[DeviceID, Dict[KeyID, Any]]] class TrustState(IntEnum): + BLACKLISTED = -100 UNSET = 0 - VERIFIED = 1 - BLACKLISTED = 2 - IGNORED = 3 + UNKNOWN_DEVICE = 10 + FORWARDED = 20 + CROSS_SIGNED_UNTRUSTED = 50 + CROSS_SIGNED_TOFU = 100 + CROSS_SIGNED_TRUSTED = 200 + VERIFIED = 300 @dataclass @@ -91,12 +119,6 @@ class DecryptedOlmEvent(ToDeviceEvent, SerializableAttrs): sender_key: IdentityKey = field(hidden=True, default=None) -class CrossSigningUsage(ExtensibleEnum): - MASTER = "master" - SELF = "self_signing" - USER = "user_signing" - - class TOFUSigningKey(NamedTuple): """ A tuple representing a single cross-signing key. The first value is the current key, and the diff --git a/mautrix/types/event/__init__.py b/mautrix/types/event/__init__.py index dde5cf14..1515bc9a 100644 --- a/mautrix/types/event/__init__.py +++ b/mautrix/types/event/__init__.py @@ -19,6 +19,7 @@ EncryptedOlmEventContent, EncryptionAlgorithm, EncryptionKeyAlgorithm, + KeyID, OlmCiphertext, OlmMsgType, ) diff --git a/mautrix/types/event/encrypted.py b/mautrix/types/event/encrypted.py index 397d8cfa..354b2acc 100644 --- a/mautrix/types/event/encrypted.py +++ b/mautrix/types/event/encrypted.py @@ -26,6 +26,24 @@ class EncryptionKeyAlgorithm(ExtensibleEnum): SIGNED_CURVE25519: "EncryptionKeyAlgorithm" = "signed_curve25519" +@dataclass(frozen=True) +class KeyID(Serializable): + algorithm: EncryptionKeyAlgorithm + key_id: str + + def serialize(self) -> JSON: + return str(self) + + @classmethod + def deserialize(cls, raw: JSON) -> "KeyID": + assert isinstance(raw, str), "key IDs must be strings" + alg, key_id = raw.split(":", 1) + return cls(EncryptionKeyAlgorithm(alg), key_id) + + def __str__(self) -> str: + return f"{self.algorithm.value}:{self.key_id}" + + class OlmMsgType(Serializable, IntEnum): PREKEY = 0 MESSAGE = 1 diff --git a/mautrix/types/primitive.py b/mautrix/types/primitive.py index f318f966..dec7dc87 100644 --- a/mautrix/types/primitive.py +++ b/mautrix/types/primitive.py @@ -53,3 +53,5 @@ SigningKey.__doc__ = "A ed25519 public key as unpadded base64" IdentityKey = NewType("IdentityKey", str) IdentityKey.__doc__ = "A curve25519 public key as unpadded base64" +Signature = NewType("Signature", str) +Signature.__doc__ = "An ed25519 signature as unpadded base64" From e3892b78d325fe9a56d85a0e1831ad8cc6f84539 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 4 Jul 2022 12:24:50 +0300 Subject: [PATCH 130/456] Add remaining cross-signing validation stuff --- mautrix/bridge/config.py | 17 +++- mautrix/bridge/e2ee.py | 10 +++ mautrix/bridge/matrix.py | 87 ++++++++++++++++---- mautrix/crypto/base.py | 4 +- mautrix/crypto/decrypt_megolm.py | 50 +++++++++--- mautrix/crypto/device_lists.py | 111 +++++++++++++++++++++++--- mautrix/crypto/encrypt_megolm.py | 8 +- mautrix/crypto/machine.py | 4 +- mautrix/crypto/sessions.py | 6 +- mautrix/crypto/store/abstract.py | 15 ++++ mautrix/crypto/store/asyncpg/store.py | 39 ++++++++- mautrix/crypto/store/memory.py | 5 ++ mautrix/errors/crypto.py | 2 +- mautrix/types/crypto.py | 34 +++++++- 14 files changed, 330 insertions(+), 62 deletions(-) diff --git a/mautrix/bridge/config.py b/mautrix/bridge/config.py index 64e6e7af..d4441d94 100644 --- a/mautrix/bridge/config.py +++ b/mautrix/bridge/config.py @@ -104,9 +104,20 @@ def do_update(self, helper: ConfigUpdateHelper) -> None: copy("bridge.encryption.allow") copy("bridge.encryption.default") - copy("bridge.encryption.key_sharing.allow") - copy("bridge.encryption.key_sharing.require_cross_signing") - copy("bridge.encryption.key_sharing.require_verification") + copy("bridge.encryption.require") + copy("bridge.encryption.verification_levels.receive") + copy("bridge.encryption.verification_levels.send") + copy("bridge.encryption.verification_levels.share") + copy("bridge.encryption.allow_key_sharing") + if self.get("bridge.encryption.key_sharing_allow", False): + helper.base["bridge.encryption.allow_key_sharing"] = True + require_verif = self.get("bridge.encryption.key_sharing.require_verification", True) + require_cs = self.get("bridge.encryption.key_sharing.require_cross_signing", False) + if require_verif: + helper.base["bridge.encryption.verification_levels.share"] = "verified" + elif not require_cs: + helper.base["bridge.encryption.verification_levels.share"] = "unverified" + # else: default (cross-signed-tofu) copy("bridge.encryption.rotation.enable_custom") copy("bridge.encryption.rotation.milliseconds") copy("bridge.encryption.rotation.messages") diff --git a/mautrix/bridge/e2ee.py b/mautrix/bridge/e2ee.py index ed4658f4..db366b3e 100644 --- a/mautrix/bridge/e2ee.py +++ b/mautrix/bridge/e2ee.py @@ -58,6 +58,10 @@ class EncryptionManager: crypto_db: Database | None state_store: StateStore + min_send_trust: TrustState + min_share_trust: TrustState + min_receive_trust: TrustState + bridge: br.Bridge az: AppService _id_prefix: str @@ -102,6 +106,12 @@ def __init__( self.crypto = OlmMachine(self.client, self.crypto_store, self.state_store) self.client.add_event_handler(InternalEventType.SYNC_STOPPED, self._exit_on_sync_fail) self.crypto.allow_key_share = self.allow_key_share + verification_levels = bridge.config["bridge.encryption.verification_levels"] + self.min_share_trust = TrustState.parse(verification_levels["share"]) + self.min_send_trust = TrustState.parse(verification_levels["send"]) + self.min_receive_trust = TrustState.parse(verification_levels["receive"]) + self.crypto.share_keys_min_trust = self.min_share_trust + self.crypto.send_keys_min_trust = self.min_receive_trust async def _exit_on_sync_fail(self, data) -> None: if data["error"]: diff --git a/mautrix/bridge/matrix.py b/mautrix/bridge/matrix.py index 075389f0..37d0d864 100644 --- a/mautrix/bridge/matrix.py +++ b/mautrix/bridge/matrix.py @@ -45,6 +45,7 @@ StateEvent, StateUnsigned, TextMessageEventContent, + TrustState, TypingEvent, UserID, Version, @@ -95,6 +96,7 @@ class BaseMatrixHandler: config: config.BaseBridgeConfig bridge: br.Bridge e2ee: EncryptionManager | None + require_e2ee: bool media_config: MediaRepoConfig versions: VersionsResponse minimum_spec_version: Version = SpecVersions.V11 @@ -116,6 +118,7 @@ def __init__( self.az.matrix_event_handler(self.int_handle_event) self.e2ee = None + self.require_e2ee = False if self.config["bridge.encryption.allow"]: if not EncryptionManager: self.log.fatal( @@ -138,6 +141,7 @@ def __init__( db_url=self.config["appservice.database"], key_sharing_config=self.config["bridge.encryption.key_sharing"], ) + self.require_e2ee = self.config["bridge.config.require"] self.management_room_text = self.config.get( "bridge.management_room_text", @@ -496,10 +500,14 @@ def is_command(self, message: MessageEventContent) -> tuple[bool, str]: return is_command, text async def handle_message( - self, room_id: RoomID, user_id: UserID, message: MessageEventContent, event_id: EventID + self, + room_id: RoomID, + user_id: UserID, + message: MessageEventContent, + event_id: EventID, + was_encrypted: bool = False, ) -> None: async def bail(error_text: str, step=MessageSendCheckpointStep.REMOTE) -> None: - self.log.debug(error_text) await MessageSendCheckpoint( event_id=event_id, room_id=room_id, @@ -516,12 +524,18 @@ async def bail(error_text: str, step=MessageSendCheckpointStep.REMOTE) -> None: self.log, ) + if not was_encrypted and self.require_e2ee: + self.log.warning(f"Dropping {event_id} from {user_id} as it's not encrypted!") + await bail("unencrypted message") + return + sender = await self.bridge.get_user(user_id) if not sender or not await self.allow_message(sender): - await bail( + self.log.debug( f"Ignoring message {event_id} from {user_id} to {room_id}:" " user is not whitelisted." ) + await bail("user not whitelisted") return self.log.debug(f"Received Matrix event {event_id} from {sender.mxid} in {room_id}") self.log.trace("Event %s content: %s", event_id, message) @@ -535,20 +549,24 @@ async def bail(error_text: str, step=MessageSendCheckpointStep.REMOTE) -> None: if await self.allow_bridging_message(sender, portal): await portal.handle_matrix_message(sender, message, event_id) else: - await bail( + self.log.debug( f"Ignoring event {event_id} from {sender.mxid}:" " not allowed to send to portal" ) + await bail("not allowed to send to portal") return if message.msgtype != MessageType.TEXT: - await bail(f"Ignoring event {event_id}: not a portal room and not a m.text message") + self.log.debug( + f"Ignoring event {event_id}: not a portal room and not a m.text message" + ) + await bail("not a portal room and not a m.text message") return elif not await self.allow_command(sender): - await bail( - f"Ignoring command {event_id} from {sender.mxid}: not allowed to perform command", - step=MessageSendCheckpointStep.COMMAND, + self.log.debug( + f"Ignoring command {event_id} from {sender.mxid}: not allowed to perform command" ) + await bail("not allowed to perform command", step=MessageSendCheckpointStep.COMMAND) return has_two_members, bridge_bot_in_room = await self._is_direct_chat(room_id) @@ -576,6 +594,7 @@ async def bail(error_text: str, step=MessageSendCheckpointStep.REMOTE) -> None: bridge_bot_in_room, ) except Exception as e: + self.log.debug(f"Error handling command {command} from {sender}: {e}") await bail(repr(e), step=MessageSendCheckpointStep.COMMAND) else: await MessageSendCheckpoint( @@ -593,10 +612,11 @@ async def bail(error_text: str, step=MessageSendCheckpointStep.REMOTE) -> None: self.log, ) else: - await bail( + self.log.debug( f"Ignoring event {event_id} from {sender.mxid}:" " not a command and not a portal room" ) + await bail("not a command and not a portal room") async def _is_direct_chat(self, room_id: RoomID) -> tuple[bool, bool]: try: @@ -651,6 +671,40 @@ async def send_encryption_error_notice( evt.room_id, f"\u26a0 Your message was not bridged: {error}" ) + @staticmethod + def _device_unverified_explanation(trust: TrustState) -> str: + explanation = { + TrustState.BLACKLISTED: "device is blacklisted", + TrustState.UNKNOWN_DEVICE: "device info not found", + TrustState.FORWARDED: "keys were forwarded from an unknown device", + TrustState.CROSS_SIGNED_UNTRUSTED: "cross-signing keys changed after setting up the bridge", + }.get(trust) + base = "your device is not trusted" + return f"{base} ({explanation})" if explanation else base + + async def _post_decrypt( + self, evt: Event, retry_num: int = 0, error_event_id: EventID | None = None + ) -> None: + trust_state = evt["mautrix"]["trust_state"] + if trust_state < self.e2ee.min_send_trust: + self.log.warning( + f"Dropping {evt.event_id} from {evt.sender} due to insufficient verification level" + f" (event: {trust_state}, required: {self.e2ee.min_send_trust})" + ) + # TODO error message + self.send_decrypted_checkpoint( + evt, + retry_num=retry_num, + permanent=True, + err=self._device_unverified_explanation(trust_state), + ) + return + + self.send_decrypted_checkpoint(evt, retry_num=retry_num) + if error_event_id: + await self.az.intent.redact(evt.room_id, error_event_id) + await self.int_handle_event(evt, was_encrypted=True) + async def handle_encrypted(self, evt: EncryptedEvent) -> None: if not self.e2ee: self.send_decrypted_checkpoint(evt, "Encryption unsupported", True) @@ -667,8 +721,7 @@ async def handle_encrypted(self, evt: EncryptedEvent) -> None: self.send_decrypted_checkpoint(evt, e, True) await self.send_encryption_error_notice(evt, e) else: - self.send_decrypted_checkpoint(decrypted) - await self.int_handle_event(decrypted, send_bridge_checkpoint=False) + await self._post_decrypt(decrypted) async def handle_encrypted_unsupported(self, evt: EncryptedEvent) -> None: self.log.debug( @@ -727,9 +780,7 @@ async def _handle_encrypted_wait( self.log.trace("%s decryption traceback:", evt.event_id, exc_info=True) msg = f"\u26a0 Your message was not bridged: {e}" else: - self.send_decrypted_checkpoint(decrypted, retry_num=1) - await self.az.intent.redact(evt.room_id, event_id) - await self.int_handle_event(decrypted, send_bridge_checkpoint=False) + await self._post_decrypt(decrypted, retry_num=1, error_event_id=event_id) return else: error_message = f"Didn't get {err.session_id}, giving up on {evt.event_id}" @@ -834,14 +885,14 @@ async def allow_matrix_event(self, evt: Event) -> bool: # For non-room events and non-bridge-originated room events, allow. return True - async def int_handle_event(self, evt: Event, send_bridge_checkpoint: bool = True) -> None: + async def int_handle_event(self, evt: Event, was_encrypted: bool = False) -> None: if isinstance(evt, StateEvent) and evt.type == EventType.ROOM_MEMBER and self.e2ee: await self.e2ee.handle_member_event(evt) if not await self.allow_matrix_event(evt): return self.log.trace("Received event: %s", evt) - if send_bridge_checkpoint: + if not was_encrypted: self.send_bridge_checkpoint(evt) start_time = time.time() @@ -933,7 +984,9 @@ async def int_handle_event(self, evt: Event, send_bridge_checkpoint: bool = True evt: MessageEvent if evt.type != EventType.ROOM_MESSAGE: evt.content.msgtype = MessageType(str(evt.type)) - await self.handle_message(evt.room_id, evt.sender, evt.content, evt.event_id) + await self.handle_message( + evt.room_id, evt.sender, evt.content, evt.event_id, was_encrypted=was_encrypted + ) elif evt.type == EventType.ROOM_ENCRYPTED: await self.handle_encrypted(evt) elif evt.type == EventType.ROOM_ENCRYPTION: diff --git a/mautrix/crypto/base.py b/mautrix/crypto/base.py index 016c991f..8dfe0a62 100644 --- a/mautrix/crypto/base.py +++ b/mautrix/crypto/base.py @@ -42,8 +42,8 @@ class BaseOlmMachine: account: account.OlmAccount - allow_unverified_devices: bool - share_to_unverified_devices: bool + send_keys_min_trust: TrustState + share_keys_min_trust: TrustState allow_key_share: Callable[[crypto.DeviceIdentity, RequestedKeyInfo], Awaitable[bool]] # Futures that wait for responses to a key request diff --git a/mautrix/crypto/decrypt_megolm.py b/mautrix/crypto/decrypt_megolm.py index c1c0100e..15eefd89 100644 --- a/mautrix/crypto/decrypt_megolm.py +++ b/mautrix/crypto/decrypt_megolm.py @@ -23,10 +23,10 @@ TrustState, ) -from .base import BaseOlmMachine +from .device_lists import DeviceListMachine -class MegolmDecryptionMachine(BaseOlmMachine): +class MegolmDecryptionMachine(DeviceListMachine): async def decrypt_megolm_event(self, evt: EncryptedEvent) -> Event: """ Decrypt an event that was encrypted using Megolm. @@ -59,23 +59,47 @@ async def decrypt_megolm_event(self, evt: EncryptedEvent) -> Event: ): raise DuplicateMessageIndex() - verified = False + forwarded_keys = False if ( evt.content.device_id == self.client.device_id and session.signing_key == self.account.signing_key - and evt.content.sender_key == self.account.identity_key + and session.sender_key == self.account.identity_key + and not session.forwarding_chain ): - verified = True + trust_level = TrustState.VERIFIED else: - device = await self.crypto_store.get_device(evt.sender, evt.content.device_id) - if device and device.trust == TrustState.VERIFIED and not session.forwarding_chain: - if ( + device = await self.get_or_fetch_device_by_key(evt.sender, session.sender_key) + if not session.forwarding_chain or ( + len(session.forwarding_chain) == 1 + and session.forwarding_chain[0] == session.sender_key + ): + if not device: + self.log.debug( + f"Couldn't resolve trust level of session {session.id}: " + f"sent by unknown device {evt.sender}/{session.sender_key}" + ) + trust_level = TrustState.UNKNOWN_DEVICE + elif ( device.signing_key != session.signing_key - or device.identity_key != evt.content.sender_key + or device.identity_key != session.sender_key ): raise VerificationError() - verified = True - # else: TODO query device keys? + else: + trust_level = await self.resolve_trust(device) + else: + forwarded_keys = True + last_chain_item = session.forwarding_chain[-1] + received_from = await self.crypto_store.find_device_by_key( + evt.sender, last_chain_item + ) + if received_from: + trust_level = await self.resolve_trust(received_from) + else: + self.log.debug( + f"Couldn't resolve trust level of session {session.id}: " + f"forwarding chain ends with unknown device {last_chain_item}" + ) + trust_level = TrustState.FORWARDED try: data = json.loads(plaintext) @@ -105,6 +129,8 @@ async def decrypt_megolm_event(self, evt: EncryptedEvent) -> Event: result.unsigned = evt.unsigned result.type = result.type.with_class(evt.type.t_class) result["mautrix"] = { - "verified": verified, + "trust_state": trust_level, + "forwarded_keys": forwarded_keys, + "was_encrypted": True, } return result diff --git a/mautrix/crypto/device_lists.py b/mautrix/crypto/device_lists.py index 5757b7d9..0cb67e5f 100644 --- a/mautrix/crypto/device_lists.py +++ b/mautrix/crypto/device_lists.py @@ -8,6 +8,7 @@ from mautrix.errors import DeviceValidationError from mautrix.types import ( CrossSigner, + CrossSigningKeys, CrossSigningUsage, DeviceID, DeviceIdentity, @@ -15,6 +16,7 @@ EncryptionKeyAlgorithm, IdentityKey, KeyID, + QueryKeysResponse, SigningKey, SyncToken, TrustState, @@ -36,13 +38,14 @@ async def _fetch_keys( self.log.trace(f"Querying keys for {users}") resp = await self.client.query_keys(users, token=since) + missing_users = users.copy() for server, err in resp.failures.items(): self.log.warning(f"Query keys failure for {server}: {err}") data = {} for user_id, devices in resp.device_keys.items(): - users.remove(user_id) + missing_users.remove(user_id) new_devices = {} existing_devices = (await self.crypto_store.get_devices(user_id)) or {} @@ -52,7 +55,8 @@ async def _fetch_keys( f"have {len(existing_devices)} in store" ) changed = False - ssk = resp.self_signing_keys.get(user_id) + ssks = resp.self_signing_keys.get(user_id) + ssk = ssks.first_ed25519_key for device_id, device_keys in devices.items(): try: existing = existing_devices[device_id] @@ -69,9 +73,7 @@ async def _fetch_keys( else: if new_device: new_devices[device_id] = new_device - await self._store_device_self_signatures( - device_keys, ssk.first_key if ssk else None - ) + await self._store_device_self_signatures(device_keys, ssk) self.log.debug( f"Storing new device list for {user_id} containing {len(new_devices)} devices" ) @@ -81,8 +83,11 @@ async def _fetch_keys( if changed or len(new_devices) != len(existing_devices): await self.on_devices_changed(user_id) + for user_id in missing_users: + self.log.warning(f"Didn't get any devices for user {user_id}") + for user_id in users: - self.log.warning(f"Didn't get any keys for user {user_id}") + await self._store_cross_signing_keys(resp, user_id) return data @@ -96,7 +101,7 @@ async def _store_device_self_signatures( self.log.warning(f"{device_desc} doesn't have any signatures from the user") return if len(device_keys.signatures) > 1: - self.log.warning( + self.log.debug( f"{device_desc} has signatures from other users (%s)", set(device_keys.signatures.keys()) - {device_keys.user_id}, ) @@ -125,11 +130,85 @@ async def _store_device_self_signatures( self.log.warning(f"{device_desc} doesn't have a valid cross-signing signature") if len(self_signatures) > 0: - self.log.warning( + self.log.debug( f"{device_desc} has signatures from unexpected keys (%s)", set(self_signatures.keys()), ) + async def _store_cross_signing_keys(self, resp: QueryKeysResponse, user_id: UserID) -> None: + new_keys: dict[CrossSigningUsage, CrossSigningKeys] = {} + try: + master = new_keys[CrossSigningUsage.MASTER] = resp.master_keys[user_id] + except KeyError: + self.log.debug(f"Didn't get a cross-signing master key for {user_id}") + return + try: + new_keys[CrossSigningUsage.SELF] = resp.self_signing_keys[user_id] + except KeyError: + self.log.debug(f"Didn't get a cross-signing self-signing key for {user_id}") + return + try: + new_keys[CrossSigningUsage.USER] = resp.user_signing_keys[user_id] + except KeyError: + pass + current_keys = await self.crypto_store.get_cross_signing_keys(user_id) + for usage, key in current_keys.items(): + if usage in new_keys and key.key != new_keys[usage].first_ed25519_key: + num = await self.crypto_store.drop_signatures_by_key(CrossSigner(user_id, key.key)) + if num >= 0: + self.log.debug( + f"Dropped {num} signatures made by key {user_id}/{key.key} ({usage})" + " as it has been replaced" + ) + for usage, key in new_keys.items(): + actual_key = key.first_ed25519_key + self.log.debug(f"Storing cross-signing key for {user_id}: {actual_key} (type {usage})") + await self.crypto_store.put_cross_signing_key(user_id, usage, actual_key) + + if usage != CrossSigningUsage.MASTER and ( + KeyID(EncryptionKeyAlgorithm.ED25519, master.first_ed25519_key) + not in key.signatures[user_id] + ): + self.log.warning( + f"Cross-signing key {user_id}/{actual_key}/{usage}" + " doesn't seem to have a signature from the master key" + ) + + for signer_user_id, signatures in key.signatures.items(): + for key_id, signature in signatures.items(): + signing_key = SigningKey(key_id.key_id) + if signer_user_id == user_id: + try: + device = resp.device_keys[signer_user_id][DeviceID(key_id.key_id)] + signing_key = device.ed25519 + except KeyError: + pass + if len(signing_key) != 43: + self.log.debug( + f"Cross-signing key {user_id}/{actual_key} has a signature from " + f"an unknown key {key_id}" + ) + continue + signing_key_log = signing_key + if signing_key != key_id.key_id: + signing_key_log = f"{signing_key} ({key_id})" + self.log.debug( + f"Verifying cross-signing key {user_id}/{actual_key} " + f"with key {signer_user_id}/{signing_key_log}" + ) + is_valid_sig = verify_signature_json( + key.serialize(), signer_user_id, key_id.key_id, signing_key + ) + if is_valid_sig: + self.log.debug(f"Signature from {signing_key_log} for {key_id} verified") + await self.crypto_store.put_signature( + target=CrossSigner(user_id, actual_key), + signer=CrossSigner(signer_user_id, signing_key), + signature=signature, + ) + else: + self.log.warning(f"Invalid signature from {signing_key_log} for {key_id}") + async def get_or_fetch_device( self, user_id: UserID, device_id: DeviceID ) -> Optional[DeviceIdentity]: @@ -193,12 +272,19 @@ async def _validate_device( device_id=device_id, identity_key=identity_key, signing_key=signing_key, - trust=TrustState.UNSET, + trust=TrustState.UNVERIFIED, name=name, deleted=False, ) async def resolve_trust(self, device: DeviceIdentity) -> TrustState: + try: + return await self._try_resolve_trust(device) + except Exception: + self.log.exception(f"Failed to resolve trust of {device.user_id}/{device.device_id}") + return TrustState.UNVERIFIED + + async def _try_resolve_trust(self, device: DeviceIdentity) -> TrustState: if device.trust in (TrustState.VERIFIED, TrustState.BLACKLISTED): return device.trust their_keys = await self.crypto_store.get_cross_signing_keys(device.user_id) @@ -208,12 +294,13 @@ async def resolve_trust(self, device: DeviceIdentity) -> TrustState: if device.user_id not in self._cs_fetch_attempted: self._cs_fetch_attempted.add(device.user_id) await self._fetch_keys([device.user_id]) + their_keys = await self.crypto_store.get_cross_signing_keys(device.user_id) try: msk = their_keys[CrossSigningUsage.MASTER] ssk = their_keys[CrossSigningUsage.SELF] except KeyError as e: self.log.error(f"Didn't find cross-signing key {e.args[0]} of {device.user_id}") - return TrustState.UNSET + return TrustState.UNVERIFIED ssk_signed = await self.crypto_store.is_key_signed_by( target=CrossSigner(device.user_id, ssk.key), signer=CrossSigner(device.user_id, msk.key), @@ -222,7 +309,7 @@ async def resolve_trust(self, device: DeviceIdentity) -> TrustState: self.log.warning( f"Self-signing key of {device.user_id} is not signed by their master key" ) - return TrustState.UNSET + return TrustState.UNVERIFIED device_signed = await self.crypto_store.is_key_signed_by( target=CrossSigner(device.user_id, device.signing_key), signer=CrossSigner(device.user_id, ssk.key), @@ -233,7 +320,7 @@ async def resolve_trust(self, device: DeviceIdentity) -> TrustState: elif msk.key == msk.first: return TrustState.CROSS_SIGNED_TOFU return TrustState.CROSS_SIGNED_UNTRUSTED - return TrustState.UNSET + return TrustState.UNVERIFIED async def is_user_trusted(self, user_id: UserID) -> bool: # TODO implement once own cross-signing key stuff is ready diff --git a/mautrix/crypto/encrypt_megolm.py b/mautrix/crypto/encrypt_megolm.py index e5830611..fa15d846 100644 --- a/mautrix/crypto/encrypt_megolm.py +++ b/mautrix/crypto/encrypt_megolm.py @@ -315,7 +315,8 @@ async def _find_olm_sessions( session.users_ignored.add(key) return already_shared - if device.trust == TrustState.BLACKLISTED: + trust = await self.resolve_trust(device) + if trust == TrustState.BLACKLISTED: self.log.debug( f"Not encrypting group session {session.id} for {device_id} " f"of {user_id}: device is blacklisted" @@ -329,10 +330,11 @@ async def _find_olm_sessions( code=RoomKeyWithheldCode.BLACKLISTED, reason="Device is blacklisted", ) - elif not self.allow_unverified_devices and device.trust == TrustState.UNSET: + elif self.send_keys_min_trust > trust: self.log.debug( f"Not encrypting group session {session.id} for {device_id} " - f"of {user_id}: device is not verified" + f"of {user_id}: device is not trusted " + f"(min: {self.send_keys_min_trust}, device: {trust})" ) session.users_ignored.add(key) return RoomKeyWithheldEventContent( diff --git a/mautrix/crypto/machine.py b/mautrix/crypto/machine.py index 97a0bac8..3d0410f0 100644 --- a/mautrix/crypto/machine.py +++ b/mautrix/crypto/machine.py @@ -67,8 +67,8 @@ def __init__( self.state_store = state_store self.account = None - self.allow_unverified_devices = True - self.share_to_unverified_devices = False + self.send_keys_min_trust = TrustState.UNVERIFIED + self.share_keys_min_trust = TrustState.CROSS_SIGNED_TOFU self.allow_key_share = self.default_allow_key_share self._fetch_keys_lock = asyncio.Lock() diff --git a/mautrix/crypto/sessions.py b/mautrix/crypto/sessions.py index 55292639..c8164b27 100644 --- a/mautrix/crypto/sessions.py +++ b/mautrix/crypto/sessions.py @@ -97,7 +97,7 @@ class InboundGroupSession(olm.InboundGroupSession): room_id: RoomID signing_key: SigningKey sender_key: IdentityKey - forwarding_chain: List[str] + forwarding_chain: List[IdentityKey] def __init__( self, @@ -105,7 +105,7 @@ def __init__( signing_key: SigningKey, sender_key: IdentityKey, room_id: RoomID, - forwarding_chain: Optional[List[str]] = None, + forwarding_chain: Optional[List[IdentityKey]] = None, ) -> None: self.signing_key = signing_key self.sender_key = sender_key @@ -124,7 +124,7 @@ def from_pickle( signing_key: SigningKey, sender_key: IdentityKey, room_id: RoomID, - forwarding_chain: Optional[List[str]] = None, + forwarding_chain: Optional[List[IdentityKey]] = None, ) -> "InboundGroupSession": session = super().from_pickle(pickle, passphrase) session.signing_key = signing_key diff --git a/mautrix/crypto/store/abstract.py b/mautrix/crypto/store/abstract.py index 78691989..a993d32d 100644 --- a/mautrix/crypto/store/abstract.py +++ b/mautrix/crypto/store/abstract.py @@ -411,6 +411,21 @@ async def put_signature( signature: The signature. """ + @abstractmethod + async def get_signatures_for_key_by( + self, target: CrossSigner, signer: UserID + ) -> dict[SigningKey, str]: + """ + Get all signatures from a given user for a given key. + + Args: + target: The key that is signed. + signer: The user whose signatures for the key to get. + + Returns: + A map of the signing key (owned by the signer) to signature. + """ + @abstractmethod async def is_key_signed_by(self, target: CrossSigner, signer: CrossSigner) -> bool: """ diff --git a/mautrix/crypto/store/asyncpg/store.py b/mautrix/crypto/store/asyncpg/store.py index 3556137b..de844f61 100644 --- a/mautrix/crypto/store/asyncpg/store.py +++ b/mautrix/crypto/store/asyncpg/store.py @@ -259,13 +259,14 @@ async def get_group_session( ) if row is None: return None + forwarding_chain = row["forwarding_chains"].split(",") if row["forwarding_chains"] else [] return InboundGroupSession.from_pickle( row["session"], passphrase=self.pickle_key, signing_key=row["signing_key"], sender_key=sender_key, room_id=room_id, - forwarding_chain=row["forwarding_chains"].split(","), + forwarding_chain=forwarding_chain, ) async def has_group_session( @@ -540,7 +541,10 @@ async def put_cross_signing_key( VALUES ($1, $2, $3, $4) ON CONFLICT (user_id, usage) DO UPDATE SET key=excluded.key """ - await self.db.execute(q, user_id, usage.value, key, key) + try: + await self.db.execute(q, user_id, usage.value, key, key) + except Exception: + self.log.exception(f"Failed to store cross-signing key {user_id}/{key}/{usage}") async def get_cross_signing_keys( self, user_id: UserID @@ -566,7 +570,28 @@ async def put_signature( """ signed_user_id, signed_key = target signer_user_id, signer_key = signer - await self.db.execute(q, signed_user_id, signed_key, signer_user_id, signer_key, signature) + try: + await self.db.execute( + q, signed_user_id, signed_key, signer_user_id, signer_key, signature + ) + except Exception: + self.log.exception( + f"Failed to store signature from {signer_user_id}/{signer_key} " + f"for {signed_user_id}/{signed_key}" + ) + + async def get_signatures_for_key_by( + self, target: CrossSigner, signer: UserID + ) -> dict[SigningKey, str]: + q = """ + SELECT signer_key, signature FROM crypto_cross_signing_signatures + WHERE signed_user_id=$1 AND signed_key=$2 AND signer_user_id=$3 + """ + signed_user_id, signed_key = target + return { + SigningKey(row["signer_key"]): row["signature"] + for row in await self.db.fetch(q, signed_user_id, signed_key, signer) + } async def is_key_signed_by(self, target: CrossSigner, signer: CrossSigner) -> bool: q = """ @@ -582,7 +607,13 @@ async def is_key_signed_by(self, target: CrossSigner, signer: CrossSigner) -> bo async def drop_signatures_by_key(self, signer: CrossSigner) -> int: signer_user_id, signer_key = signer q = "DELETE FROM crypto_cross_signing_signatures WHERE signer_user_id=$1 AND signer_key=$2" - res = await self.db.execute(q, signer_user_id, signer_key) + try: + res = await self.db.execute(q, signer_user_id, signer_key) + except Exception: + self.log.exception( + f"Failed to drop old signatures made by replaced key {signer_user_id}/{signer_key}" + ) + return -1 if Cursor is not None and isinstance(res, Cursor): return res.rowcount elif ( diff --git a/mautrix/crypto/store/memory.py b/mautrix/crypto/store/memory.py index ab7c492e..4c928d2b 100644 --- a/mautrix/crypto/store/memory.py +++ b/mautrix/crypto/store/memory.py @@ -188,6 +188,11 @@ async def put_signature( ) -> None: self._signatures.setdefault(signer, {})[target] = signature + async def get_signatures_for_key_by( + self, target: CrossSigner, signer: UserID + ) -> dict[SigningKey, str]: + raise NotImplementedError() + async def is_key_signed_by(self, target: CrossSigner, signer: CrossSigner) -> bool: return target in self._signatures.get(signer, {}) diff --git a/mautrix/errors/crypto.py b/mautrix/errors/crypto.py index 1c67e320..c45d77e7 100644 --- a/mautrix/errors/crypto.py +++ b/mautrix/errors/crypto.py @@ -46,7 +46,7 @@ def __init__(self) -> None: class VerificationError(DecryptionError): def __init__(self) -> None: - super().__init__("Device keys in event and verified device info do not match") + super().__init__("Device keys in session and cached device info do not match") class DecryptedPayloadError(DecryptionError): diff --git a/mautrix/types/crypto.py b/mautrix/types/crypto.py index a4350e73..0fa0fb69 100644 --- a/mautrix/types/crypto.py +++ b/mautrix/types/crypto.py @@ -3,7 +3,7 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from typing import Any, Dict, List, NamedTuple, Optional +from typing import Any, ClassVar, Dict, List, NamedTuple, Optional from enum import IntEnum from attr import dataclass @@ -56,7 +56,7 @@ class CrossSigningUsage(ExtensibleEnum): class CrossSigningKeys(SerializableAttrs): user_id: UserID usage: List[CrossSigningUsage] - keys: Dict[str, SigningKey] + keys: Dict[KeyID, SigningKey] signatures: Dict[UserID, Dict[KeyID, Signature]] = field(factory=lambda: {}) @property @@ -66,6 +66,16 @@ def first_key(self) -> Optional[SigningKey]: except StopIteration: return None + @property + def first_ed25519_key(self) -> Optional[SigningKey]: + return self.first_key_with_algorithm(EncryptionKeyAlgorithm.ED25519) + + def first_key_with_algorithm(self, alg: EncryptionKeyAlgorithm) -> Optional[SigningKey]: + try: + return next(key for key_id, key in self.keys.items() if key_id.algorithm == alg) + except StopIteration: + return None + @dataclass class QueryKeysResponse(SerializableAttrs): @@ -84,7 +94,7 @@ class ClaimKeysResponse(SerializableAttrs): class TrustState(IntEnum): BLACKLISTED = -100 - UNSET = 0 + UNVERIFIED = 0 UNKNOWN_DEVICE = 10 FORWARDED = 20 CROSS_SIGNED_UNTRUSTED = 50 @@ -92,6 +102,24 @@ class TrustState(IntEnum): CROSS_SIGNED_TRUSTED = 200 VERIFIED = 300 + def __str__(self) -> str: + return _trust_state_to_name[self] + + @classmethod + def parse(cls, val: str) -> "TrustState": + try: + return _name_to_trust_state[val] + except KeyError as e: + raise ValueError(f"Invalid trust state {val!r}") from e + + +_trust_state_to_name: dict[TrustState, str] = { + val: val.name.lower().replace("_", "-") for val in TrustState +} +_name_to_trust_state: dict[str, TrustState] = { + value: key for key, value in _trust_state_to_name.items() +} + @dataclass class DeviceIdentity: From 01b36c17244ed66fb8e08bee47307c7497304ce6 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 4 Jul 2022 14:06:11 +0300 Subject: [PATCH 131/456] Send error notices and status events if decryption fails --- mautrix/bridge/matrix.py | 196 ++++++++++++++++++---------------- mautrix/errors/crypto.py | 4 +- mautrix/types/event/beeper.py | 5 + 3 files changed, 109 insertions(+), 96 deletions(-) diff --git a/mautrix/bridge/matrix.py b/mautrix/bridge/matrix.py index 37d0d864..54a9bbfb 100644 --- a/mautrix/bridge/matrix.py +++ b/mautrix/bridge/matrix.py @@ -23,6 +23,7 @@ ) from mautrix.types import ( BaseRoomEvent, + BeeperMessageStatusEventContent, EncryptedEvent, Event, EventID, @@ -32,12 +33,15 @@ MemberStateEventContent, MessageEvent, MessageEventContent, + MessageStatusReason, MessageType, PresenceEvent, ReactionEvent, ReceiptEvent, ReceiptType, RedactionEvent, + RelatesTo, + RelationType, RoomID, RoomType, SingleReceiptEventContent, @@ -499,34 +503,67 @@ def is_command(self, message: MessageEventContent) -> tuple[bool, str]: text = text[len(prefix) + 1 :].lstrip() return is_command, text - async def handle_message( + async def _send_crypto_status_error( self, - room_id: RoomID, - user_id: UserID, - message: MessageEventContent, - event_id: EventID, - was_encrypted: bool = False, - ) -> None: - async def bail(error_text: str, step=MessageSendCheckpointStep.REMOTE) -> None: - await MessageSendCheckpoint( - event_id=event_id, - room_id=room_id, - step=step, - timestamp=int(time.time() * 1000), - status=MessageSendCheckpointStatus.PERM_FAILURE, - reported_by=MessageSendCheckpointReportedBy.BRIDGE, - event_type=EventType.ROOM_MESSAGE, - message_type=message.msgtype, - info=error_text, - ).send( - self.bridge.config["homeserver.message_send_checkpoint_endpoint"], - self.az.as_token, - self.log, + evt: Event, + err: Exception | str | None = None, + retry_num: int = 0, + is_final: bool = True, + edit: EventID | None = None, + wait_for: int | None = None, + ) -> EventID | None: + msg = str(err) + if isinstance(err, SessionNotFound): + msg = "the bridge hasn't received the decryption keys" + self._send_message_checkpoint( + evt, MessageSendCheckpointStep.DECRYPTED, msg, permanent=is_final, retry_num=retry_num + ) + + if wait_for: + msg += f". The bridge will retry for {wait_for} seconds" + event_id = None + if self.config.get("bridge.delivery_error_reports", True): + try: + content = TextMessageEventContent( + msgtype=MessageType.NOTICE, body=f"\u26a0 Your message was not bridged: {msg}." + ) + if edit: + content.set_edit(edit) + event_id = await self.az.intent.send_message(evt.room_id, content) + except IntentError: + self.log.debug("IntentError while sending encryption error", exc_info=True) + self.log.error( + "Got IntentError while trying to send encryption error message. " + "This likely means the bridge bot is not in the room, which can " + "happen if you force-enable e2ee on the homeserver without enabling " + "it by default on the bridge (bridge -> encryption -> default)." + ) + + if self.config.get("bridge.message_status_events", False): + status_content = BeeperMessageStatusEventContent( + network="", # TODO set network properly + relates_to=RelatesTo(rel_type=RelationType.REFERENCE, event_id=evt.event_id), + success=False, + is_certain=True, + can_retry=True, + reason=MessageStatusReason.UNDECRYPTABLE, + error=msg, ) + await self.az.intent.send_message_event( + evt.room_id, EventType.BEEPER_MESSAGE_STATUS, status_content + ) + + return event_id + + async def handle_message(self, evt: MessageEvent, was_encrypted: bool = False) -> None: + room_id = evt.room_id + user_id = evt.sender + event_id = evt.event_id + message = evt.content if not was_encrypted and self.require_e2ee: self.log.warning(f"Dropping {event_id} from {user_id} as it's not encrypted!") - await bail("unencrypted message") + await self._send_crypto_status_error(evt, "unencrypted message", 0) return sender = await self.bridge.get_user(user_id) @@ -535,7 +572,9 @@ async def bail(error_text: str, step=MessageSendCheckpointStep.REMOTE) -> None: f"Ignoring message {event_id} from {user_id} to {room_id}:" " user is not whitelisted." ) - await bail("user not whitelisted") + self._send_message_checkpoint( + evt, MessageSendCheckpointStep.BRIDGE, "user is not whitelisted" + ) return self.log.debug(f"Received Matrix event {event_id} from {sender.mxid} in {room_id}") self.log.trace("Event %s content: %s", event_id, message) @@ -553,20 +592,28 @@ async def bail(error_text: str, step=MessageSendCheckpointStep.REMOTE) -> None: f"Ignoring event {event_id} from {sender.mxid}:" " not allowed to send to portal" ) - await bail("not allowed to send to portal") + self._send_message_checkpoint( + evt, + MessageSendCheckpointStep.BRIDGE, + "user is not allowed to send to the portal", + ) return if message.msgtype != MessageType.TEXT: self.log.debug( f"Ignoring event {event_id}: not a portal room and not a m.text message" ) - await bail("not a portal room and not a m.text message") + self._send_message_checkpoint( + evt, MessageSendCheckpointStep.BRIDGE, "not a portal room and not a m.text message" + ) return elif not await self.allow_command(sender): self.log.debug( - f"Ignoring command {event_id} from {sender.mxid}: not allowed to perform command" + f"Ignoring command {event_id} from {sender.mxid}: not allowed to run commands" + ) + self._send_message_checkpoint( + evt, MessageSendCheckpointStep.COMMAND, "not allowed to run commands" ) - await bail("not allowed to perform command", step=MessageSendCheckpointStep.COMMAND) return has_two_members, bridge_bot_in_room = await self._is_direct_chat(room_id) @@ -595,7 +642,7 @@ async def bail(error_text: str, step=MessageSendCheckpointStep.REMOTE) -> None: ) except Exception as e: self.log.debug(f"Error handling command {command} from {sender}: {e}") - await bail(repr(e), step=MessageSendCheckpointStep.COMMAND) + self._send_message_checkpoint(evt, MessageSendCheckpointStep.COMMAND, e) else: await MessageSendCheckpoint( event_id=event_id, @@ -616,7 +663,9 @@ async def bail(error_text: str, step=MessageSendCheckpointStep.REMOTE) -> None: f"Ignoring event {event_id} from {sender.mxid}:" " not a command and not a portal room" ) - await bail("not a command and not a portal room") + self._send_message_checkpoint( + evt, MessageSendCheckpointStep.COMMAND, "not a command and not a portal room" + ) async def _is_direct_chat(self, room_id: RoomID) -> tuple[bool, bool]: try: @@ -664,20 +713,15 @@ async def try_handle_sync_event(self, evt: Event) -> None: except Exception: self.log.exception("Error handling manually received Matrix event") - async def send_encryption_error_notice( - self, evt: EncryptedEvent, error: DecryptionError - ) -> None: - await self.az.intent.send_notice( - evt.room_id, f"\u26a0 Your message was not bridged: {error}" - ) - @staticmethod def _device_unverified_explanation(trust: TrustState) -> str: explanation = { TrustState.BLACKLISTED: "device is blacklisted", TrustState.UNKNOWN_DEVICE: "device info not found", TrustState.FORWARDED: "keys were forwarded from an unknown device", - TrustState.CROSS_SIGNED_UNTRUSTED: "cross-signing keys changed after setting up the bridge", + TrustState.CROSS_SIGNED_UNTRUSTED: ( + "cross-signing keys changed after setting up the bridge" + ), }.get(trust) base = "your device is not trusted" return f"{base} ({explanation})" if explanation else base @@ -691,35 +735,35 @@ async def _post_decrypt( f"Dropping {evt.event_id} from {evt.sender} due to insufficient verification level" f" (event: {trust_state}, required: {self.e2ee.min_send_trust})" ) - # TODO error message - self.send_decrypted_checkpoint( + await self._send_crypto_status_error( evt, retry_num=retry_num, - permanent=True, err=self._device_unverified_explanation(trust_state), + edit=error_event_id, ) return - self.send_decrypted_checkpoint(evt, retry_num=retry_num) + self._send_message_checkpoint( + evt, MessageSendCheckpointStep.DECRYPTED, retry_num=retry_num + ) if error_event_id: await self.az.intent.redact(evt.room_id, error_event_id) await self.int_handle_event(evt, was_encrypted=True) async def handle_encrypted(self, evt: EncryptedEvent) -> None: if not self.e2ee: - self.send_decrypted_checkpoint(evt, "Encryption unsupported", True) + await self._send_crypto_status_error(evt, "encryption is not supported") + # TODO replace this with code in _send_crypto_status_error? await self.handle_encrypted_unsupported(evt) return try: - decrypted = await self.e2ee.decrypt(evt, wait_session_timeout=5) + decrypted = await self.e2ee.decrypt(evt, wait_session_timeout=3) except SessionNotFound as e: - self.send_decrypted_checkpoint(evt, e, False) - await self._handle_encrypted_wait(evt, e, wait=15) + await self._handle_encrypted_wait(evt, e, wait=6) except DecryptionError as e: self.log.warning(f"Failed to decrypt {evt.event_id}: {e}") self.log.trace("%s decryption traceback:", evt.event_id, exc_info=True) - self.send_decrypted_checkpoint(evt, e, True) - await self.send_encryption_error_notice(evt, e) + await self._send_crypto_status_error(evt, e) else: await self._post_decrypt(decrypted) @@ -740,11 +784,6 @@ async def _handle_encrypted_wait( f"Couldn't find session {err.session_id} trying to decrypt {evt.event_id}," " waiting even longer" ) - msg = ( - "\u26a0 Your message was not bridged: the bridge hasn't received the decryption " - f"keys. The bridge will retry for {wait} seconds. If this error keeps happening, " - "try restarting your client." - ) asyncio.create_task( self.e2ee.crypto.request_room_key( evt.room_id, @@ -753,17 +792,7 @@ async def _handle_encrypted_wait( from_devices={evt.sender: [evt.content.device_id]}, ) ) - try: - event_id = await self.az.intent.send_notice(evt.room_id, msg) - except IntentError: - self.log.debug("IntentError while sending encryption error", exc_info=True) - self.log.error( - "Got IntentError while trying to send encryption error message. " - "This likely means the bridge bot is not in the room, which can " - "happen if you force-enable e2ee on the homeserver without enabling " - "it by default on the bridge (bridge -> encryption -> default)." - ) - return + event_id = await self._send_crypto_status_error(evt, err, is_final=False, wait_for=wait) got_keys = await self.e2ee.crypto.wait_for_session( evt.room_id, err.sender_key, err.session_id, timeout=wait ) @@ -775,24 +804,17 @@ async def _handle_encrypted_wait( try: decrypted = await self.e2ee.decrypt(evt, wait_session_timeout=0) except DecryptionError as e: - self.send_decrypted_checkpoint(evt, e, True, retry_num=1) + await self._send_crypto_status_error(evt, e, retry_num=1, edit=event_id) self.log.warning(f"Failed to decrypt {evt.event_id}: {e}") self.log.trace("%s decryption traceback:", evt.event_id, exc_info=True) - msg = f"\u26a0 Your message was not bridged: {e}" else: await self._post_decrypt(decrypted, retry_num=1, error_event_id=event_id) return else: - error_message = f"Didn't get {err.session_id}, giving up on {evt.event_id}" - self.log.warning(error_message) - self.send_decrypted_checkpoint(evt, error_message, True, retry_num=1) - msg = ( - "\u26a0 Your message was not bridged: the bridge hasn't received the decryption" - " keys. If this error keeps happening, try restarting your client." + self.log.warning(f"Didn't get {err.session_id}, giving up on {evt.event_id}") + await self._send_crypto_status_error( + evt, SessionNotFound(err.session_id), retry_num=1, edit=event_id ) - content = TextMessageEventContent(msgtype=MessageType.NOTICE, body=msg) - content.set_edit(event_id) - await self.az.intent.send_message(evt.room_id, content) async def handle_encryption(self, evt: StateEvent) -> None: await self.az.state_store.set_encryption_info(evt.room_id, evt.content) @@ -804,12 +826,12 @@ async def handle_encryption(self, evt: StateEvent) -> None: portal.log.debug("Received encryption event in direct portal: %s", evt.content) await portal.enable_dm_encryption() - def send_message_send_checkpoint( + def _send_message_checkpoint( self, evt: Event, step: MessageSendCheckpointStep, err: Exception | str | None = None, - permanent: bool = False, + permanent: bool = True, retry_num: int = 0, ) -> None: endpoint = self.bridge.config["homeserver.message_send_checkpoint_endpoint"] @@ -841,20 +863,6 @@ def send_message_send_checkpoint( ) asyncio.create_task(checkpoint.send(endpoint, self.az.as_token, self.log)) - def send_bridge_checkpoint(self, evt: Event) -> None: - self.send_message_send_checkpoint(evt, MessageSendCheckpointStep.BRIDGE) - - def send_decrypted_checkpoint( - self, - evt: Event, - err: Exception | str | None = None, - permanent: bool = False, - retry_num: int = 0, - ) -> None: - self.send_message_send_checkpoint( - evt, MessageSendCheckpointStep.DECRYPTED, err, permanent, retry_num - ) - allowed_event_classes: tuple[type, ...] = ( MessageEvent, StateEvent, @@ -893,7 +901,7 @@ async def int_handle_event(self, evt: Event, was_encrypted: bool = False) -> Non self.log.trace("Received event: %s", evt) if not was_encrypted: - self.send_bridge_checkpoint(evt) + self._send_message_checkpoint(evt, MessageSendCheckpointStep.BRIDGE) start_time = time.time() if evt.type == EventType.ROOM_MEMBER: @@ -984,9 +992,7 @@ async def int_handle_event(self, evt: Event, was_encrypted: bool = False) -> Non evt: MessageEvent if evt.type != EventType.ROOM_MESSAGE: evt.content.msgtype = MessageType(str(evt.type)) - await self.handle_message( - evt.room_id, evt.sender, evt.content, evt.event_id, was_encrypted=was_encrypted - ) + await self.handle_message(evt, was_encrypted=was_encrypted) elif evt.type == EventType.ROOM_ENCRYPTED: await self.handle_encrypted(evt) elif evt.type == EventType.ROOM_ENCRYPTION: diff --git a/mautrix/errors/crypto.py b/mautrix/errors/crypto.py index c45d77e7..526f8ed5 100644 --- a/mautrix/errors/crypto.py +++ b/mautrix/errors/crypto.py @@ -3,6 +3,8 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. +from __future__ import annotations + from mautrix.types import IdentityKey, SessionID from .base import MatrixError @@ -31,7 +33,7 @@ class MatchingSessionDecryptionError(DecryptionError): class SessionNotFound(DecryptionError): - def __init__(self, session_id: SessionID, sender_key: IdentityKey) -> None: + def __init__(self, session_id: SessionID, sender_key: IdentityKey | None = None) -> None: super().__init__( f"Failed to decrypt megolm event: no session with given ID {session_id} found" ) diff --git a/mautrix/types/event/beeper.py b/mautrix/types/event/beeper.py index ce0b2998..6c3c6010 100644 --- a/mautrix/types/event/beeper.py +++ b/mautrix/types/event/beeper.py @@ -7,6 +7,7 @@ from attr import dataclass +from ..primitive import EventID from ..util import SerializableAttrs, SerializableEnum, field from .base import BaseRoomEvent from .message import RelatesTo @@ -15,6 +16,7 @@ class MessageStatusReason(SerializableEnum): GENERIC_ERROR = "m.event_not_handled" UNSUPPORTED = "com.beeper.unsupported_event" + UNDECRYPTABLE = "com.beeper.undecryptable_event" TOO_OLD = "m.event_too_old" NETWORK_ERROR = "m.foreign_network_error" NO_PERMISSION = "m.no_permission" @@ -42,6 +44,9 @@ class BeeperMessageStatusEventContent(SerializableAttrs): can_retry: Optional[bool] = None is_certain: Optional[bool] = None + still_working: Optional[bool] = None + last_retry: Optional[EventID] = None + @dataclass class BeeperMessageStatusEvent(BaseRoomEvent, SerializableAttrs): From 2cbf590a74a387bf61cd05feb8f0ffb6411e1492 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 4 Jul 2022 14:07:30 +0300 Subject: [PATCH 132/456] Reword server outdated error --- mautrix/bridge/matrix.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mautrix/bridge/matrix.py b/mautrix/bridge/matrix.py index 54a9bbfb..208dc31a 100644 --- a/mautrix/bridge/matrix.py +++ b/mautrix/bridge/matrix.py @@ -163,8 +163,8 @@ def __init__( async def check_versions(self) -> None: if not self.versions.supports_at_least(self.minimum_spec_version): self.log.fatal( - "Server isn't advertising modern spec versions " - "(latest supported by server: %s, minimum required by bridge: %s)", + "The homeserver is outdated " + "(server supports Matrix %s, but the bridge requires at least %s)", self.versions.latest_version, self.minimum_spec_version, ) From 4181c38deb882561a6ddc5a5b2678f8733cbc9ef Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 4 Jul 2022 14:41:04 +0300 Subject: [PATCH 133/456] Fix typo --- mautrix/bridge/matrix.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/bridge/matrix.py b/mautrix/bridge/matrix.py index 208dc31a..a968a977 100644 --- a/mautrix/bridge/matrix.py +++ b/mautrix/bridge/matrix.py @@ -145,7 +145,7 @@ def __init__( db_url=self.config["appservice.database"], key_sharing_config=self.config["bridge.encryption.key_sharing"], ) - self.require_e2ee = self.config["bridge.config.require"] + self.require_e2ee = self.config["bridge.encryption.require"] self.management_room_text = self.config.get( "bridge.management_room_text", From 82218dece16db6e99f7912b6757a499ca82cacaa Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 4 Jul 2022 14:48:28 +0300 Subject: [PATCH 134/456] Fix breaking on Python 3.8 --- mautrix/types/crypto.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mautrix/types/crypto.py b/mautrix/types/crypto.py index 0fa0fb69..08c9dd22 100644 --- a/mautrix/types/crypto.py +++ b/mautrix/types/crypto.py @@ -3,7 +3,7 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from typing import Any, ClassVar, Dict, List, NamedTuple, Optional +from typing import Any, Dict, List, NamedTuple, Optional from enum import IntEnum from attr import dataclass @@ -113,10 +113,10 @@ def parse(cls, val: str) -> "TrustState": raise ValueError(f"Invalid trust state {val!r}") from e -_trust_state_to_name: dict[TrustState, str] = { +_trust_state_to_name: Dict[TrustState, str] = { val: val.name.lower().replace("_", "-") for val in TrustState } -_name_to_trust_state: dict[str, TrustState] = { +_name_to_trust_state: Dict[str, TrustState] = { value: key for key, value in _trust_state_to_name.items() } From 3e0a39b6d820299274e1ae4c5e1324442e21b6d5 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 4 Jul 2022 15:24:53 +0300 Subject: [PATCH 135/456] Remove unused method --- mautrix/crypto/store/abstract.py | 15 --------------- mautrix/crypto/store/asyncpg/store.py | 13 ------------- mautrix/crypto/store/memory.py | 5 ----- 3 files changed, 33 deletions(-) diff --git a/mautrix/crypto/store/abstract.py b/mautrix/crypto/store/abstract.py index a993d32d..78691989 100644 --- a/mautrix/crypto/store/abstract.py +++ b/mautrix/crypto/store/abstract.py @@ -411,21 +411,6 @@ async def put_signature( signature: The signature. """ - @abstractmethod - async def get_signatures_for_key_by( - self, target: CrossSigner, signer: UserID - ) -> dict[SigningKey, str]: - """ - Get all signatures from a given user for a given key. - - Args: - target: The key that is signed. - signer: The user whose signatures for the key to get. - - Returns: - A map of the signing key (owned by the signer) to signature. - """ - @abstractmethod async def is_key_signed_by(self, target: CrossSigner, signer: CrossSigner) -> bool: """ diff --git a/mautrix/crypto/store/asyncpg/store.py b/mautrix/crypto/store/asyncpg/store.py index de844f61..0f81fc61 100644 --- a/mautrix/crypto/store/asyncpg/store.py +++ b/mautrix/crypto/store/asyncpg/store.py @@ -580,19 +580,6 @@ async def put_signature( f"for {signed_user_id}/{signed_key}" ) - async def get_signatures_for_key_by( - self, target: CrossSigner, signer: UserID - ) -> dict[SigningKey, str]: - q = """ - SELECT signer_key, signature FROM crypto_cross_signing_signatures - WHERE signed_user_id=$1 AND signed_key=$2 AND signer_user_id=$3 - """ - signed_user_id, signed_key = target - return { - SigningKey(row["signer_key"]): row["signature"] - for row in await self.db.fetch(q, signed_user_id, signed_key, signer) - } - async def is_key_signed_by(self, target: CrossSigner, signer: CrossSigner) -> bool: q = """ SELECT EXISTS( diff --git a/mautrix/crypto/store/memory.py b/mautrix/crypto/store/memory.py index 4c928d2b..ab7c492e 100644 --- a/mautrix/crypto/store/memory.py +++ b/mautrix/crypto/store/memory.py @@ -188,11 +188,6 @@ async def put_signature( ) -> None: self._signatures.setdefault(signer, {})[target] = signature - async def get_signatures_for_key_by( - self, target: CrossSigner, signer: UserID - ) -> dict[SigningKey, str]: - raise NotImplementedError() - async def is_key_signed_by(self, target: CrossSigner, signer: CrossSigner) -> bool: return target in self._signatures.get(signer, {}) From 18b0a07353dcd96230f6bb97b7657b5c23263c29 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 4 Jul 2022 15:51:41 +0300 Subject: [PATCH 136/456] Mark sender_key and device_id as deprecated in Megolm events --- mautrix/crypto/decrypt_megolm.py | 3 ++- mautrix/errors/crypto.py | 16 ++++++++++++- mautrix/types/event/encrypted.py | 39 ++++++++++++++++++++++++++------ 3 files changed, 49 insertions(+), 9 deletions(-) diff --git a/mautrix/crypto/decrypt_megolm.py b/mautrix/crypto/decrypt_megolm.py index 15eefd89..d3ccead3 100644 --- a/mautrix/crypto/decrypt_megolm.py +++ b/mautrix/crypto/decrypt_megolm.py @@ -20,6 +20,7 @@ EncryptedMegolmEventContent, EncryptionAlgorithm, Event, + SessionID, TrustState, ) @@ -55,7 +56,7 @@ async def decrypt_megolm_event(self, evt: EncryptedEvent) -> Event: except olm.OlmGroupSessionError as e: raise DecryptionError("Failed to decrypt megolm event") from e if not await self.crypto_store.validate_message_index( - evt.content.sender_key, evt.content.session_id, evt.event_id, index, evt.timestamp + session.sender_key, SessionID(session.id), evt.event_id, index, evt.timestamp ): raise DuplicateMessageIndex() diff --git a/mautrix/errors/crypto.py b/mautrix/errors/crypto.py index 526f8ed5..3e4cf5dc 100644 --- a/mautrix/errors/crypto.py +++ b/mautrix/errors/crypto.py @@ -5,6 +5,8 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import annotations +import warnings + from mautrix.types import IdentityKey, SessionID from .base import MatrixError @@ -38,7 +40,19 @@ def __init__(self, session_id: SessionID, sender_key: IdentityKey | None = None) f"Failed to decrypt megolm event: no session with given ID {session_id} found" ) self.session_id = session_id - self.sender_key = sender_key + self._sender_key = sender_key + + @property + def sender_key(self) -> IdentityKey | None: + """ + .. deprecated:: 0.17.0 + Matrix v1.3 deprecated the device_id and sender_key fields in megolm events. + """ + warnings.warn( + "The sender_key field in Megolm events was deprecated in Matrix 1.3", + DeprecationWarning, + ) + return self._sender_key class DuplicateMessageIndex(DecryptionError): diff --git a/mautrix/types/event/encrypted.py b/mautrix/types/event/encrypted.py index 354b2acc..71eb2954 100644 --- a/mautrix/types/event/encrypted.py +++ b/mautrix/types/event/encrypted.py @@ -3,14 +3,14 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from typing import Dict, NewType, Optional, Union +from typing import Annotated, Dict, NewType, Optional, Union from enum import IntEnum +import warnings from attr import dataclass -import attr from ..primitive import JSON, DeviceID, IdentityKey, SessionID -from ..util import ExtensibleEnum, Obj, Serializable, SerializableAttrs, deserializer +from ..util import ExtensibleEnum, Obj, Serializable, SerializableAttrs, deserializer, field from .base import BaseRoomEvent, BaseUnsigned from .message import RelatesTo @@ -74,12 +74,37 @@ class EncryptedMegolmEventContent(SerializableAttrs): """The content of an m.room.encrypted event""" ciphertext: str - sender_key: IdentityKey - device_id: DeviceID session_id: SessionID - _relates_to: Optional[RelatesTo] = attr.ib(default=None, metadata={"json": "m.relates_to"}) algorithm: EncryptionAlgorithm = EncryptionAlgorithm.MEGOLM_V1 + _sender_key: Optional[IdentityKey] = field(default=None, json="sender_key") + _device_id: Optional[DeviceID] = field(default=None, json="device_id") + _relates_to: Optional[RelatesTo] = field(default=None, json="m.relates_to") + + @property + def sender_key(self) -> Optional[IdentityKey]: + """ + .. deprecated:: 0.17.0 + Matrix v1.3 deprecated the device_id and sender_key fields in megolm events. + """ + warnings.warn( + "The sender_key field in Megolm events was deprecated in Matrix 1.3", + DeprecationWarning, + ) + return self._sender_key + + @property + def device_id(self) -> Optional[DeviceID]: + """ + .. deprecated:: 0.17.0 + Matrix v1.3 deprecated the device_id and sender_key fields in megolm events. + """ + warnings.warn( + "The sender_key field in Megolm events was deprecated in Matrix 1.3", + DeprecationWarning, + ) + return self._device_id + @property def relates_to(self) -> RelatesTo: if self._relates_to is None: @@ -114,7 +139,7 @@ class EncryptedEvent(BaseRoomEvent, SerializableAttrs): """A m.room.encrypted event""" content: EncryptedEventContent - _unsigned: Optional[BaseUnsigned] = attr.ib(default=None, metadata={"json": "unsigned"}) + _unsigned: Optional[BaseUnsigned] = field(default=None, json="unsigned") @property def unsigned(self) -> BaseUnsigned: From 3611da878fca9a2b35ee52850fa99164a67e608e Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 4 Jul 2022 16:14:54 +0300 Subject: [PATCH 137/456] Remove sender_key parameter from has/get_group_session --- mautrix/bridge/e2ee.py | 2 +- mautrix/bridge/matrix.py | 2 +- mautrix/crypto/base.py | 4 ++-- mautrix/crypto/decrypt_megolm.py | 4 +--- mautrix/crypto/key_request.py | 2 +- mautrix/crypto/key_share.py | 7 +++---- mautrix/crypto/store/abstract.py | 8 ++------ mautrix/crypto/store/asyncpg/store.py | 25 ++++++++++--------------- mautrix/crypto/store/memory.py | 14 ++++++-------- 9 files changed, 27 insertions(+), 41 deletions(-) diff --git a/mautrix/bridge/e2ee.py b/mautrix/bridge/e2ee.py index db366b3e..fc6db00a 100644 --- a/mautrix/bridge/e2ee.py +++ b/mautrix/bridge/e2ee.py @@ -216,7 +216,7 @@ async def decrypt(self, evt: EncryptedEvent, wait_session_timeout: int = 5) -> M f" waiting {wait_session_timeout} seconds..." ) got_keys = await self.crypto.wait_for_session( - evt.room_id, e.sender_key, e.session_id, timeout=wait_session_timeout + evt.room_id, e.session_id, timeout=wait_session_timeout ) if got_keys: self.log.debug( diff --git a/mautrix/bridge/matrix.py b/mautrix/bridge/matrix.py index a968a977..1726b4a9 100644 --- a/mautrix/bridge/matrix.py +++ b/mautrix/bridge/matrix.py @@ -794,7 +794,7 @@ async def _handle_encrypted_wait( ) event_id = await self._send_crypto_status_error(evt, err, is_final=False, wait_for=wait) got_keys = await self.e2ee.crypto.wait_for_session( - evt.room_id, err.sender_key, err.session_id, timeout=wait + evt.room_id, err.session_id, timeout=wait ) if got_keys: self.log.debug( diff --git a/mautrix/crypto/base.py b/mautrix/crypto/base.py index 8dfe0a62..d7b0f9b9 100644 --- a/mautrix/crypto/base.py +++ b/mautrix/crypto/base.py @@ -56,7 +56,7 @@ class BaseOlmMachine: _cs_fetch_attempted: set[UserID] async def wait_for_session( - self, room_id: RoomID, sender_key: IdentityKey, session_id: SessionID, timeout: float = 3 + self, room_id: RoomID, session_id: SessionID, timeout: float = 3 ) -> bool: try: fut = self._inbound_session_waiters[session_id] @@ -66,7 +66,7 @@ async def wait_for_session( try: return await asyncio.wait_for(asyncio.shield(fut), timeout) except asyncio.TimeoutError: - return await self.crypto_store.has_group_session(room_id, sender_key, session_id) + return await self.crypto_store.has_group_session(room_id, session_id) def _mark_session_received(self, session_id: SessionID) -> None: try: diff --git a/mautrix/crypto/decrypt_megolm.py b/mautrix/crypto/decrypt_megolm.py index d3ccead3..8edf5aaf 100644 --- a/mautrix/crypto/decrypt_megolm.py +++ b/mautrix/crypto/decrypt_megolm.py @@ -45,9 +45,7 @@ async def decrypt_megolm_event(self, evt: EncryptedEvent) -> Event: raise DecryptionError("Unsupported event content class") elif evt.content.algorithm != EncryptionAlgorithm.MEGOLM_V1: raise DecryptionError("Unsupported event encryption algorithm") - session = await self.crypto_store.get_group_session( - evt.room_id, evt.content.sender_key, evt.content.session_id - ) + session = await self.crypto_store.get_group_session(evt.room_id, evt.content.session_id) if session is None: # TODO check if olm session is wedged raise SessionNotFound(evt.content.session_id, evt.content.sender_key) diff --git a/mautrix/crypto/key_request.py b/mautrix/crypto/key_request.py index b70cc947..9bbd2d67 100644 --- a/mautrix/crypto/key_request.py +++ b/mautrix/crypto/key_request.py @@ -114,7 +114,7 @@ async def request_room_key( async def _receive_forwarded_room_key(self, evt: DecryptedOlmEvent) -> None: key: ForwardedRoomKeyEventContent = evt.content - if await self.crypto_store.has_group_session(key.room_id, key.sender_key, key.session_id): + if await self.crypto_store.has_group_session(key.room_id, key.session_id): self.log.debug( f"Ignoring received session {key.session_id} from {evt.sender}/" f"{evt.sender_device}, as crypto store says we have it already" diff --git a/mautrix/crypto/key_share.py b/mautrix/crypto/key_share.py index 0221e5fb..ad4ccc74 100644 --- a/mautrix/crypto/key_share.py +++ b/mautrix/crypto/key_share.py @@ -16,6 +16,7 @@ RoomKeyRequestEventContent, RoomKeyWithheldCode, RoomKeyWithheldEventContent, + SessionID, ToDeviceEvent, TrustState, ) @@ -167,9 +168,7 @@ async def _handle_room_key_request( if not await self.allow_key_share(device, request): return - sess = await self.crypto_store.get_group_session( - request.room_id, request.sender_key, request.session_id - ) + sess = await self.crypto_store.get_group_session(request.room_id, request.session_id) if sess is None: raise RejectKeyShare( f"Didn't find group session {request.session_id} to forward to " @@ -182,7 +181,7 @@ async def _handle_room_key_request( forward_content = ForwardedRoomKeyEventContent( algorithm=EncryptionAlgorithm.MEGOLM_V1, room_id=sess.room_id, - session_id=sess.id, + session_id=SessionID(sess.id), session_key=exported_key, sender_key=sess.sender_key, forwarding_key_chain=sess.forwarding_chain, diff --git a/mautrix/crypto/store/abstract.py b/mautrix/crypto/store/abstract.py index 78691989..7828524d 100644 --- a/mautrix/crypto/store/abstract.py +++ b/mautrix/crypto/store/abstract.py @@ -183,7 +183,7 @@ async def put_group_session( @abstractmethod async def get_group_session( - self, room_id: RoomID, sender_key: IdentityKey, session_id: SessionID + self, room_id: RoomID, session_id: SessionID ) -> InboundGroupSession | None: """ Get an inbound Megolm group session that was previously inserted with @@ -191,7 +191,6 @@ async def get_group_session( Args: room_id: The room ID for which the session was made. - sender_key: The curve25519 identity key of the user who made the session. session_id: The unique identifier of the session. Returns: @@ -199,16 +198,13 @@ async def get_group_session( """ @abstractmethod - async def has_group_session( - self, room_id: RoomID, sender_key: IdentityKey, session_id: SessionID - ) -> bool: + async def has_group_session(self, room_id: RoomID, session_id: SessionID) -> bool: """ Check whether or not a specific inbound Megolm session is in the store. This is used before importing forwarded keys. Args: room_id: The room ID for which the session was made. - sender_key: The curve25519 identity key of the user who made the session. session_id: The unique identifier of the session. Returns: diff --git a/mautrix/crypto/store/asyncpg/store.py b/mautrix/crypto/store/asyncpg/store.py index 0f81fc61..2138b4c8 100644 --- a/mautrix/crypto/store/asyncpg/store.py +++ b/mautrix/crypto/store/asyncpg/store.py @@ -247,16 +247,14 @@ async def put_group_session( ) async def get_group_session( - self, room_id: RoomID, sender_key: IdentityKey, session_id: SessionID + self, room_id: RoomID, session_id: SessionID ) -> InboundGroupSession | None: - row = await self.db.fetchrow( - "SELECT signing_key, session, forwarding_chains FROM crypto_megolm_inbound_session " - "WHERE room_id=$1 AND sender_key=$2 AND session_id=$3 AND account_id=$4", - room_id, - sender_key, - session_id, - self.account_id, - ) + q = """ + SELECT sender_key, signing_key, session, forwarding_chains + FROM crypto_megolm_inbound_session + WHERE room_id=$1 AND session_id=$2 AND account_id=$3 + """ + row = await self.db.fetchrow(q, room_id, session_id, self.account_id) if row is None: return None forwarding_chain = row["forwarding_chains"].split(",") if row["forwarding_chains"] else [] @@ -264,19 +262,16 @@ async def get_group_session( row["session"], passphrase=self.pickle_key, signing_key=row["signing_key"], - sender_key=sender_key, + sender_key=row["sender_key"], room_id=room_id, forwarding_chain=forwarding_chain, ) - async def has_group_session( - self, room_id: RoomID, sender_key: IdentityKey, session_id: SessionID - ) -> bool: + async def has_group_session(self, room_id: RoomID, session_id: SessionID) -> bool: count = await self.db.fetchval( "SELECT COUNT(session) FROM crypto_megolm_inbound_session " - "WHERE room_id=$1 AND sender_key=$2 AND session_id=$3 AND account_id=$4", + "WHERE room_id=$1 AND session_id=$2 AND account_id=$3", room_id, - sender_key, session_id, self.account_id, ) diff --git a/mautrix/crypto/store/memory.py b/mautrix/crypto/store/memory.py index ab7c492e..35dc26b5 100644 --- a/mautrix/crypto/store/memory.py +++ b/mautrix/crypto/store/memory.py @@ -33,7 +33,7 @@ class MemoryCryptoStore(CryptoStore, SyncStore): _message_indices: dict[tuple[IdentityKey, SessionID, int], tuple[EventID, int]] _devices: dict[UserID, dict[DeviceID, DeviceIdentity]] _olm_sessions: dict[IdentityKey, list[Session]] - _inbound_sessions: dict[tuple[RoomID, IdentityKey, SessionID], InboundGroupSession] + _inbound_sessions: dict[tuple[RoomID, SessionID], InboundGroupSession] _outbound_sessions: dict[RoomID, OutboundGroupSession] _signatures: dict[CrossSigner, dict[CrossSigner, str]] _cross_signing_keys: dict[UserID, dict[CrossSigningUsage, TOFUSigningKey]] @@ -103,17 +103,15 @@ async def put_group_session( session_id: SessionID, session: InboundGroupSession, ) -> None: - self._inbound_sessions[(room_id, sender_key, session_id)] = session + self._inbound_sessions[(room_id, session_id)] = session async def get_group_session( - self, room_id: RoomID, sender_key: IdentityKey, session_id: SessionID + self, room_id: RoomID, session_id: SessionID ) -> InboundGroupSession: - return self._inbound_sessions.get((room_id, sender_key, session_id)) + return self._inbound_sessions.get((room_id, session_id)) - async def has_group_session( - self, room_id: RoomID, sender_key: IdentityKey, session_id: SessionID - ) -> bool: - return (room_id, sender_key, session_id) in self._inbound_sessions + async def has_group_session(self, room_id: RoomID, session_id: SessionID) -> bool: + return (room_id, session_id) in self._inbound_sessions async def add_outbound_group_session(self, session: OutboundGroupSession) -> None: self._outbound_sessions[session.room_id] = session From d6bb59461a86e67a5917e237158e1e63cfd5c797 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 4 Jul 2022 16:34:41 +0300 Subject: [PATCH 138/456] Clean up crypto store SQL queries --- mautrix/crypto/store/asyncpg/store.py | 258 +++++++++++++------------- 1 file changed, 126 insertions(+), 132 deletions(-) diff --git a/mautrix/crypto/store/asyncpg/store.py b/mautrix/crypto/store/asyncpg/store.py index 2138b4c8..94c1b140 100644 --- a/mautrix/crypto/store/asyncpg/store.py +++ b/mautrix/crypto/store/asyncpg/store.py @@ -33,9 +33,12 @@ from .upgrade import upgrade_table try: + from sqlite3 import sqlite_version_info as sqlite_version + from aiosqlite import Cursor except ImportError: Cursor = None + sqlite_version = (0, 0, 0) class PgCryptoStateStore(PgStateStore, StateStore): @@ -75,43 +78,38 @@ async def delete(self) -> None: await conn.execute(f"DELETE FROM {table} WHERE account_id=$1", self.account_id) async def get_device_id(self) -> DeviceID | None: - device_id = await self.db.fetchval( - "SELECT device_id FROM crypto_account WHERE account_id=$1", self.account_id - ) + q = "SELECT device_id FROM crypto_account WHERE account_id=$1" + device_id = await self.db.fetchval(q, self.account_id) self._device_id = device_id or self._device_id return self._device_id async def put_device_id(self, device_id: DeviceID) -> None: - await self.db.fetchval( - "UPDATE crypto_account SET device_id=$1 WHERE account_id=$2", - device_id, - self.account_id, - ) + q = "UPDATE crypto_account SET device_id=$1 WHERE account_id=$2" + await self.db.fetchval(q, device_id, self.account_id) self._device_id = device_id async def put_next_batch(self, next_batch: SyncToken) -> None: self._sync_token = next_batch - await self.db.execute( - "UPDATE crypto_account SET sync_token=$1 WHERE account_id=$2", - self._sync_token, - self.account_id, - ) + q = "UPDATE crypto_account SET sync_token=$1 WHERE account_id=$2" + await self.db.execute(q, self._sync_token, self.account_id) async def get_next_batch(self) -> SyncToken: if self._sync_token is None: - self._sync_token = await self.db.fetchval( - "SELECT sync_token FROM crypto_account WHERE account_id=$1", self.account_id - ) + q = "SELECT sync_token FROM crypto_account WHERE account_id=$1" + self._sync_token = await self.db.fetchval(q, self.account_id) return self._sync_token async def put_account(self, account: OlmAccount) -> None: self._account = account pickle = account.pickle(self.pickle_key) + q = """ + INSERT INTO crypto_account (account_id, device_id, shared, sync_token, account) + VALUES ($1, $2, $3, $4, $5) + ON CONFLICT (account_id) DO UPDATE + SET shared=excluded.shared, sync_token=excluded.sync_token, account=excluded.account + """ await self.db.execute( - "INSERT INTO crypto_account (account_id, device_id, shared, " - "sync_token, account) VALUES($1, $2, $3, $4, $5) " - "ON CONFLICT (account_id) DO UPDATE SET shared=$3, sync_token=$4," - " account=$5", + q, self.account_id, self._device_id, account.shared, @@ -121,10 +119,8 @@ async def put_account(self, account: OlmAccount) -> None: async def get_account(self) -> OlmAccount: if self._account is None: - row = await self.db.fetchrow( - "SELECT shared, account, device_id FROM crypto_account WHERE account_id=$1", - self.account_id, - ) + q = "SELECT shared, account, device_id FROM crypto_account WHERE account_id=$1" + row = await self.db.fetchrow(q, self.account_id) if row is not None: self._account = OlmAccount.from_pickle( row["account"], passphrase=self.pickle_key, shared=row["shared"] @@ -134,19 +130,16 @@ async def get_account(self) -> OlmAccount: async def has_session(self, key: IdentityKey) -> bool: if len(self._olm_cache[key]) > 0: return True - val = await self.db.fetchval( - "SELECT session_id FROM crypto_olm_session WHERE sender_key=$1 AND account_id=$2", - key, - self.account_id, - ) + q = "SELECT session_id FROM crypto_olm_session WHERE sender_key=$1 AND account_id=$2" + val = await self.db.fetchval(q, key, self.account_id) return val is not None async def get_sessions(self, key: IdentityKey) -> list[Session]: - q = ( - "SELECT session_id, session, created_at, last_encrypted, last_decrypted " - "FROM crypto_olm_session WHERE sender_key=$1 AND account_id=$2 " - "ORDER BY last_decrypted DESC" - ) + q = """ + SELECT session_id, session, created_at, last_encrypted, last_decrypted + FROM crypto_olm_session WHERE sender_key=$1 AND account_id=$2 + ORDER BY last_decrypted DESC + """ rows = await self.db.fetch(q, key, self.account_id) sessions = [] for row in rows: @@ -165,11 +158,11 @@ async def get_sessions(self, key: IdentityKey) -> list[Session]: return sessions async def get_latest_session(self, key: IdentityKey) -> Session | None: - q = ( - "SELECT session_id, session, created_at, last_encrypted, last_decrypted " - "FROM crypto_olm_session WHERE sender_key=$1 AND account_id=$2 " - "ORDER BY last_decrypted DESC LIMIT 1" - ) + q = """ + SELECT session_id, session, created_at, last_encrypted, last_decrypted + FROM crypto_olm_session WHERE sender_key=$1 AND account_id=$2 + ORDER BY last_decrypted DESC LIMIT 1 + """ row = await self.db.fetchrow(q, key, self.account_id) if row is None: return None @@ -192,9 +185,9 @@ async def add_session(self, key: IdentityKey, session: Session) -> None: self._olm_cache[key][SessionID(session.id)] = session pickle = session.pickle(self.pickle_key) q = """ - INSERT INTO crypto_olm_session (session_id, sender_key, session, created_at, - last_encrypted, last_decrypted, account_id) - VALUES ($1, $2, $3, $4, $5, $6, $7) + INSERT INTO crypto_olm_session ( + session_id, sender_key, session, created_at, last_encrypted, last_decrypted, account_id + ) VALUES ($1, $2, $3, $4, $5, $6, $7) """ await self.db.execute( q, @@ -216,10 +209,10 @@ async def update_session(self, key: IdentityKey, session: Session) -> None: f"isn't equal to the one being saved to the database ({e})" ) pickle = session.pickle(self.pickle_key) - q = ( - "UPDATE crypto_olm_session SET session=$1, last_encrypted=$2, last_decrypted=$3 " - "WHERE session_id=$4 AND account_id=$5" - ) + q = """ + UPDATE crypto_olm_session SET session=$1, last_encrypted=$2, last_decrypted=$3 + WHERE session_id=$4 AND account_id=$5 + """ await self.db.execute( q, pickle, session.last_encrypted, session.last_decrypted, session.id, self.account_id ) @@ -233,10 +226,13 @@ async def put_group_session( ) -> None: pickle = session.pickle(self.pickle_key) forwarding_chains = ",".join(session.forwarding_chain) + q = """ + INSERT INTO crypto_megolm_inbound_session ( + session_id, sender_key, signing_key, room_id, session, forwarding_chains, account_id + ) VALUES ($1, $2, $3, $4, $5, $6, $7) + """ await self.db.execute( - "INSERT INTO crypto_megolm_inbound_session (session_id, sender_key, " - "signing_key, room_id, session, forwarding_chains, account_id) " - "VALUES ($1, $2, $3, $4, $5, $6, $7)", + q, session_id, sender_key, session.signing_key, @@ -268,13 +264,11 @@ async def get_group_session( ) async def has_group_session(self, room_id: RoomID, session_id: SessionID) -> bool: - count = await self.db.fetchval( - "SELECT COUNT(session) FROM crypto_megolm_inbound_session " - "WHERE room_id=$1 AND session_id=$2 AND account_id=$3", - room_id, - session_id, - self.account_id, - ) + q = """ + SELECT COUNT(session) FROM crypto_megolm_inbound_session + WHERE room_id=$1 AND session_id=$2 AND account_id=$3 + """ + count = await self.db.fetchval(q, room_id, session_id, self.account_id) return count > 0 async def add_outbound_group_session(self, session: OutboundGroupSession) -> None: @@ -282,12 +276,18 @@ async def add_outbound_group_session(self, session: OutboundGroupSession) -> Non max_age = session.max_age if self.db.scheme == Scheme.SQLITE: max_age = max_age.total_seconds() + q = """ + INSERT INTO crypto_megolm_outbound_session ( + room_id, session_id, session, shared, max_messages, message_count, + max_age, created_at, last_used, account_id + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) + ON CONFLICT (account_id, room_id) DO UPDATE + SET session_id=excluded.session_id, session=excluded.session, shared=excluded.shared, + max_messages=excluded.max_messages, message_count=excluded.message_count, + max_age=excluded.max_age, created_at=excluded.created_at, last_used=excluded.last_used + """ await self.db.execute( - "INSERT INTO crypto_megolm_outbound_session (room_id, session_id, session, shared, " - "max_messages, message_count, max_age, created_at, last_used, account_id) " - "VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)" - "ON CONFLICT (account_id, room_id) DO UPDATE SET session_id=$2, session=$3, shared=$4," - " max_messages=$5, message_count=$6, max_age=$7, created_at=$8, last_used=$9", + q, session.room_id, session.id, pickle, @@ -302,9 +302,12 @@ async def add_outbound_group_session(self, session: OutboundGroupSession) -> Non async def update_outbound_group_session(self, session: OutboundGroupSession) -> None: pickle = session.pickle(self.pickle_key) + q = """ + UPDATE crypto_megolm_outbound_session SET session=$1, message_count=$2, last_used=$3 + WHERE room_id=$4 AND session_id=$5 AND account_id=$6 + """ await self.db.execute( - "UPDATE crypto_megolm_outbound_session SET session=$1, message_count=$2, last_used=$3 " - "WHERE room_id=$4 AND session_id=$5 AND account_id=$6", + q, pickle, session.message_count, session.use_time, @@ -314,13 +317,12 @@ async def update_outbound_group_session(self, session: OutboundGroupSession) -> ) async def get_outbound_group_session(self, room_id: RoomID) -> OutboundGroupSession | None: - row = await self.db.fetchrow( - "SELECT room_id, session_id, session, shared, max_messages, message_count, max_age, " - " created_at, last_used " - "FROM crypto_megolm_outbound_session WHERE room_id=$1 AND account_id=$2", - room_id, - self.account_id, - ) + q = """ + SELECT room_id, session_id, session, shared, max_messages, message_count, max_age, + created_at, last_used + FROM crypto_megolm_outbound_session WHERE room_id=$1 AND account_id=$2 + """ + row = await self.db.fetchrow(q, room_id, self.account_id) if row is None: return None max_age = row["max_age"] @@ -339,39 +341,29 @@ async def get_outbound_group_session(self, room_id: RoomID) -> OutboundGroupSess ) async def remove_outbound_group_session(self, room_id: RoomID) -> None: - await self.db.execute( - "DELETE FROM crypto_megolm_outbound_session WHERE room_id=$1 AND account_id=$2", - room_id, - self.account_id, - ) + q = "DELETE FROM crypto_megolm_outbound_session WHERE room_id=$1 AND account_id=$2" + await self.db.execute(q, room_id, self.account_id) async def remove_outbound_group_sessions(self, rooms: list[RoomID]) -> None: if self.db.scheme in (Scheme.POSTGRES, Scheme.COCKROACH): - await self.db.execute( - "DELETE FROM crypto_megolm_outbound_session " - "WHERE account_id=$1 AND room_id=ANY($2)", - self.account_id, - rooms, - ) + q = """ + DELETE FROM crypto_megolm_outbound_session WHERE account_id=$1 AND room_id=ANY($2) + """ + await self.db.execute(q, self.account_id, rooms) else: params = ",".join(["?"] * len(rooms)) - await self.db.execute( - "DELETE FROM crypto_megolm_outbound_session " - f"WHERE account_id=? AND room_id IN ({params})", - self.account_id, - *rooms, - ) - - _validate_message_index_query = ( - "WITH existing AS (" - " INSERT INTO crypto_message_index(sender_key, session_id, index, event_id, timestamp)" - " VALUES ($1, $2, $3, $4, $5)" - # have to update something so that RETURNING * always returns the row - " ON CONFLICT (sender_key, session_id, index) DO UPDATE SET sender_key=$1" - " RETURNING *" - ")" - "SELECT * FROM existing" - ) + q = f""" + DELETE FROM crypto_megolm_outbound_session WHERE account_id=? AND room_id IN ({params}) + """ + await self.db.execute(q, self.account_id, *rooms) + + _validate_message_index_query = """ + INSERT INTO crypto_message_index (sender_key, session_id, index, event_id, timestamp) + VALUES ($1, $2, $3, $4, $5) + -- have to update something so that RETURNING * always returns the row + ON CONFLICT (sender_key, session_id, index) DO UPDATE SET sender_key=excluded.sender_key + RETURNING * + """ async def validate_message_index( self, @@ -381,7 +373,11 @@ async def validate_message_index( index: int, timestamp: int, ) -> bool: - if self.db.scheme in (Scheme.POSTGRES, Scheme.COCKROACH): + if self.db.scheme in (Scheme.POSTGRES, Scheme.COCKROACH) or ( + # RETURNING was added in SQLite 3.35.0 https://www.sqlite.org/lang_returning.html + self.db.scheme == Scheme.SQLITE + and sqlite_version >= (3, 35) + ): row = await self.db.fetchrow( self._validate_message_index_query, sender_key, @@ -414,16 +410,15 @@ async def validate_message_index( return True async def get_devices(self, user_id: UserID) -> dict[DeviceID, DeviceIdentity] | None: - tracked_user_id = await self.db.fetchval( - "SELECT user_id FROM crypto_tracked_user WHERE user_id=$1", user_id - ) + q = "SELECT user_id FROM crypto_tracked_user WHERE user_id=$1" + tracked_user_id = await self.db.fetchval(q, user_id) if tracked_user_id is None: return None - rows = await self.db.fetch( - "SELECT device_id, identity_key, signing_key, trust, deleted, " - "name FROM crypto_device WHERE user_id=$1", - user_id, - ) + q = """ + SELECT device_id, identity_key, signing_key, trust, deleted, name + FROM crypto_device WHERE user_id=$1 + """ + rows = await self.db.fetch(q, user_id) result = {} for row in rows: result[row["device_id"]] = DeviceIdentity( @@ -438,12 +433,11 @@ async def get_devices(self, user_id: UserID) -> dict[DeviceID, DeviceIdentity] | return result async def get_device(self, user_id: UserID, device_id: DeviceID) -> DeviceIdentity | None: - row = await self.db.fetchrow( - "SELECT identity_key, signing_key, trust, deleted, name " - "FROM crypto_device WHERE user_id=$1 AND device_id=$2", - user_id, - device_id, - ) + q = """ + SELECT identity_key, signing_key, trust, deleted, name FROM crypto_device + WHERE user_id=$1 AND device_id=$2 + """ + row = await self.db.fetchrow(q, user_id, device_id) if row is None: return None return DeviceIdentity( @@ -459,9 +453,12 @@ async def get_device(self, user_id: UserID, device_id: DeviceID) -> DeviceIdenti async def find_device_by_key( self, user_id: UserID, identity_key: IdentityKey ) -> DeviceIdentity | None: + q = """ + SELECT device_id, signing_key, trust, deleted, name FROM crypto_device + WHERE user_id=$1 AND identity_key=$2 + """ row = await self.db.fetchrow( - "SELECT device_id, signing_key, trust, deleted, name " - "FROM crypto_device WHERE user_id=$1 AND identity_key=$2", + q, user_id, identity_key, ) @@ -500,32 +497,29 @@ async def put_devices(self, user_id: UserID, devices: dict[DeviceID, DeviceIdent "name", ] async with self.db.acquire() as conn, conn.transaction(): - await conn.execute( - "INSERT INTO crypto_tracked_user (user_id) VALUES ($1) " - "ON CONFLICT (user_id) DO NOTHING", - user_id, - ) + q = """ + INSERT INTO crypto_tracked_user (user_id) VALUES ($1) ON CONFLICT (user_id) DO NOTHING + """ + await conn.execute(q, user_id) await conn.execute("DELETE FROM crypto_device WHERE user_id=$1", user_id) if self.db.scheme == Scheme.POSTGRES: await conn.copy_records_to_table("crypto_device", records=data, columns=columns) else: - await conn.executemany( - "INSERT INTO crypto_device (user_id, device_id, " - "identity_key, signing_key, trust, deleted, name) " - "VALUES ($1, $2, $3, $4, $5, $6, $7)", - data, - ) + q = """ + INSERT INTO crypto_device ( + user_id, device_id, identity_key, signing_key, trust, deleted, name + ) VALUES ($1, $2, $3, $4, $5, $6, $7) + """ + await conn.executemany(q, data) async def filter_tracked_users(self, users: list[UserID]) -> list[UserID]: if self.db.scheme in (Scheme.POSTGRES, Scheme.COCKROACH): - rows = await self.db.fetch( - "SELECT user_id FROM crypto_tracked_user WHERE user_id = ANY($1)", users - ) + q = "SELECT user_id FROM crypto_tracked_user WHERE user_id = ANY($1)" + rows = await self.db.fetch(q, users) else: params = ",".join(["?"] * len(users)) - rows = await self.db.fetch( - f"SELECT user_id FROM crypto_tracked_user WHERE user_id IN ({params})", *users - ) + q = f"SELECT user_id FROM crypto_tracked_user WHERE user_id IN ({params})" + rows = await self.db.fetch(q, *users) return [row["user_id"] for row in rows] async def put_cross_signing_key( From e25c7837a3089cd82115959f00b851a53f8b8e11 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 4 Jul 2022 16:37:06 +0300 Subject: [PATCH 139/456] Remove accidental test import --- mautrix/types/event/encrypted.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/types/event/encrypted.py b/mautrix/types/event/encrypted.py index 71eb2954..cd08fc94 100644 --- a/mautrix/types/event/encrypted.py +++ b/mautrix/types/event/encrypted.py @@ -3,7 +3,7 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from typing import Annotated, Dict, NewType, Optional, Union +from typing import Dict, NewType, Optional, Union from enum import IntEnum import warnings From 6c170aae033e85717c3b258df576958d5ac7b48b Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 4 Jul 2022 17:05:30 +0300 Subject: [PATCH 140/456] Add quotes around index for SQLite --- mautrix/crypto/store/asyncpg/store.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mautrix/crypto/store/asyncpg/store.py b/mautrix/crypto/store/asyncpg/store.py index 94c1b140..9af2d075 100644 --- a/mautrix/crypto/store/asyncpg/store.py +++ b/mautrix/crypto/store/asyncpg/store.py @@ -358,10 +358,10 @@ async def remove_outbound_group_sessions(self, rooms: list[RoomID]) -> None: await self.db.execute(q, self.account_id, *rooms) _validate_message_index_query = """ - INSERT INTO crypto_message_index (sender_key, session_id, index, event_id, timestamp) + INSERT INTO crypto_message_index (sender_key, session_id, "index", event_id, timestamp) VALUES ($1, $2, $3, $4, $5) -- have to update something so that RETURNING * always returns the row - ON CONFLICT (sender_key, session_id, index) DO UPDATE SET sender_key=excluded.sender_key + ON CONFLICT (sender_key, session_id, "index") DO UPDATE SET sender_key=excluded.sender_key RETURNING * """ From 6f29e18245e6404264824bbdc8a56e9a538f2900 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 5 Jul 2022 13:40:43 +0300 Subject: [PATCH 141/456] Bump request log level to debug --- mautrix/api.py | 49 +++++++++++++++++++++------- mautrix/client/api/authentication.py | 1 + 2 files changed, 39 insertions(+), 11 deletions(-) diff --git a/mautrix/api.py b/mautrix/api.py index 27255ee6..bff8698c 100644 --- a/mautrix/api.py +++ b/mautrix/api.py @@ -5,18 +5,18 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import annotations -from typing import AsyncGenerator, AsyncIterable, ClassVar, Literal, Mapping, Union +from typing import AsyncGenerator, ClassVar, Literal, Mapping, Union from enum import Enum from json.decoder import JSONDecodeError -from time import time from urllib.parse import quote as urllib_quote, urljoin as urllib_join import asyncio import inspect import json import logging import platform +import time -from aiohttp import ClientSession, __version__ as aiohttp_version +from aiohttp import ClientResponse, ClientSession, __version__ as aiohttp_version from aiohttp.client_exceptions import ClientError, ContentTypeError from yarl import URL @@ -239,7 +239,7 @@ async def _send( content: bytes | bytearray | str | AsyncBody, query_params: dict[str, str], headers: dict[str, str], - ) -> JSON: + ) -> tuple[JSON, ClientResponse]: request = self.session.request( str(method), url, data=content, params=query_params, headers=headers ) @@ -258,7 +258,7 @@ async def _send( errcode=errcode, message=message, ) - return await response.json() + return await response.json(), response def _log_request( self, @@ -269,6 +269,7 @@ def _log_request( query_params: dict[str, str], headers: dict[str, str], req_id: int, + sensitive: bool, ) -> None: if not self.log: return @@ -277,26 +278,42 @@ def _log_request( elif inspect.isasyncgen(content): size = headers.get("Content-Length", None) log_content = f"<{size} async bytes>" if size else f"" + elif sensitive: + log_content = f"<{len(content)} sensitive bytes>" else: log_content = content as_user = query_params.get("user_id", None) - level = 1 if path == Path.v3.sync else 5 + level = 5 if path == Path.v3.sync else 10 self.log.log( level, - f"{method}#{req_id} /{path} {log_content}".strip(" "), + f"req #{req_id}: {method} /{path} {log_content}".strip(" "), extra={ "matrix_http_request": { "req_id": req_id, "method": str(method), "path": str(path), "content": ( - orig_content if isinstance(orig_content, (dict, list)) else log_content + orig_content + if isinstance(orig_content, (dict, list)) and not sensitive + else log_content ), "user": as_user, } }, ) + def _log_request_done( + self, path: PathBuilder, req_id: int, duration: float, status: int + ) -> None: + level = 5 if path == Path.v3.sync else 10 + duration_str = f"{duration * 1000:.1f}ms" if duration < 1 else f"{duration:.3f}s" + path_without_prefix = f"/{path}".replace("/_matrix/client", "") + self.log.log( + level, + f"req #{req_id} ({path_without_prefix}) completed in {duration_str} " + f"with status {status}", + ) + def _full_path(self, path: PathBuilder | str) -> str: path = str(path) if path and path[0] == "/": @@ -316,6 +333,7 @@ async def request( retry_count: int | None = None, metrics_method: str = "", min_iter_size: int = 25 * 1024 * 1024, + sensitive: bool = False, ) -> JSON: """ Make a raw Matrix API request. @@ -335,6 +353,7 @@ async def request( min_iter_size: If the request body is larger than this value, it will be passed to aiohttp as an async iterable to stop it from copying the whole thing in memory. + sensitive: If True, the request content will not be logged. Returns: The parsed response JSON. @@ -369,11 +388,18 @@ async def request( headers["Content-Length"] = str(len(content)) backoff = 4 while True: - self._log_request(method, path, content, orig_content, query_params, headers, req_id) + self._log_request( + method, path, content, orig_content, query_params, headers, req_id, sensitive + ) API_CALLS.labels(method=metrics_method).inc() req_content = _async_iter_bytes(content) if do_fake_iter else content + start = time.monotonic() try: - return await self._send(method, full_url, req_content, query_params, headers or {}) + resp_data, resp = await self._send( + method, full_url, req_content, query_params, headers or {} + ) + self._log_request_done(path, req_id, time.monotonic() - start, resp.status) + return resp_data except MatrixRequestError as e: API_CALLS_FAILED.labels(method=metrics_method).inc() if retry_count > 0 and e.http_status in (502, 503, 504): @@ -382,6 +408,7 @@ async def request( f"retrying in {backoff} seconds" ) else: + self._log_request_done(path, req_id, time.monotonic() - start, e.http_status) raise except ClientError as e: API_CALLS_FAILED.labels(method=metrics_method).inc() @@ -401,7 +428,7 @@ async def request( def get_txn_id(self) -> str: """Get a new unique transaction ID.""" self.txn_id += 1 - return f"mautrix-python_R{self.txn_id}@T{int(time() * 1000)}" + return f"mautrix-python_R{self.txn_id}@T{int(time.time() * 1000)}" def get_download_url( self, diff --git a/mautrix/client/api/authentication.py b/mautrix/client/api/authentication.py index ec97ea5e..cd77272d 100644 --- a/mautrix/client/api/authentication.py +++ b/mautrix/client/api/authentication.py @@ -100,6 +100,7 @@ async def login( "identifier": identifier.serialize(), **kwargs, }, + sensitive="password" in kwargs or "token" in kwargs, ) resp_data = LoginResponse.deserialize(resp) if store_access_token: From 214399c0c09920f53e848f2d686d4dad5efa706a Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 5 Jul 2022 13:40:54 +0300 Subject: [PATCH 142/456] Update transaction ID format to match mautrix-go --- mautrix/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/api.py b/mautrix/api.py index bff8698c..88a75921 100644 --- a/mautrix/api.py +++ b/mautrix/api.py @@ -428,7 +428,7 @@ async def request( def get_txn_id(self) -> str: """Get a new unique transaction ID.""" self.txn_id += 1 - return f"mautrix-python_R{self.txn_id}@T{int(time.time() * 1000)}" + return f"mautrix-python_{time.time_ns()}_{self.txn_id}" def get_download_url( self, From 57963f5a5d04124762294947d730e5d3196803bf Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 5 Jul 2022 14:59:57 +0300 Subject: [PATCH 143/456] Flip argument order in admin commands Fixes mautrix/go#76 --- mautrix/bridge/commands/admin.py | 131 +++++++++++++++---------------- 1 file changed, 63 insertions(+), 68 deletions(-) diff --git a/mautrix/bridge/commands/admin.py b/mautrix/bridge/commands/admin.py index a7f0aed2..b3712135 100644 --- a/mautrix/bridge/commands/admin.py +++ b/mautrix/bridge/commands/admin.py @@ -3,11 +3,12 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from typing import Optional +from __future__ import annotations from mautrix.errors import IntentError, MatrixRequestError, MForbidden from mautrix.types import ContentURI, EventID, UserID +from ... import bridge as br from .handler import SECTION_ADMIN, CommandEvent, command_handler @@ -16,19 +17,28 @@ needs_auth=False, name="set-pl", help_section=SECTION_ADMIN, - help_args="<_level_> [_mxid_]", + help_args="[_mxid_] <_level_>", help_text="Set a temporary power level without affecting the remote platform.", ) async def set_power_level(evt: CommandEvent) -> EventID: + try: + user_id = UserID(evt.args[0]) + except IndexError: + return await evt.reply(f"**Usage:** `$cmdprefix+sp set-pl [mxid] `") + + if user_id.startswith("@"): + evt.args.pop(0) + else: + user_id = evt.sender.mxid + try: level = int(evt.args[0]) except (KeyError, IndexError): - return await evt.reply("**Usage:** `$cmdprefix+sp set-pl [mxid]`") + return await evt.reply("**Usage:** `$cmdprefix+sp set-pl [mxid] `") except ValueError: return await evt.reply("The level must be an integer.") - mxid = evt.args[1] if len(evt.args) > 1 else evt.sender.mxid levels = await evt.main_intent.get_power_levels(evt.room_id, ignore_cache=True) - levels.users[mxid] = level + levels.users[user_id] = level try: return await evt.main_intent.set_power_levels(evt.room_id, levels) except MForbidden as e: @@ -38,6 +48,28 @@ async def set_power_level(evt: CommandEvent) -> EventID: return await evt.reply("Failed to update power levels (see logs for more details)") +async def _get_mxid_param( + evt: CommandEvent, args: str +) -> tuple[br.BasePuppet | None, EventID | None]: + try: + user_id = UserID(evt.args[0]) + except IndexError: + return None, await evt.reply(f"**Usage:** `$cmdprefix+sp {evt.command} {args}`") + + if user_id.startswith("@") and ":" in user_id: + # TODO support parsing mention pills instead of requiring a plaintext mxid + puppet = await evt.bridge.get_puppet(user_id) + if not puppet: + return None, await evt.reply("The given user ID is not a valid ghost user.") + evt.args.pop(0) + return puppet, None + elif evt.is_portal and (puppet := await evt.portal.get_dm_puppet()): + return puppet, None + return None, await evt.reply( + "This is not a private chat portal, you must pass a user ID explicitly." + ) + + @command_handler( needs_admin=True, needs_auth=False, @@ -46,27 +78,20 @@ async def set_power_level(evt: CommandEvent) -> EventID: help_args="<_mxc:// uri_> [_mxid_]", help_text="Set an avatar for a ghost user.", ) -async def set_ghost_avatar(evt: CommandEvent) -> Optional[EventID]: +async def set_ghost_avatar(evt: CommandEvent) -> EventID | None: + puppet, err = await _get_mxid_param(evt, "[mxid] ") + if err: + return err + try: mxc_uri = ContentURI(evt.args[0]) - except (KeyError, IndexError): - return await evt.reply("**Usage:** `$cmdprefix+sp set-avatar [mxid]`") + except IndexError: + return await evt.reply("**Usage:** `$cmdprefix+sp set-avatar [mxid] `") if not mxc_uri.startswith("mxc://"): - return await evt.reply("The URI has to start with mxc://.") - if len(evt.args) > 1: - # TODO support parsing mention pills instead of requiring a plaintext mxid - puppet = await evt.processor.bridge.get_puppet(UserID(evt.args[1])) - if puppet is None: - return await evt.reply("The given mxid was not a valid ghost user.") - intent = puppet.intent - elif evt.is_portal: - intent = evt.portal.main_intent - if intent == evt.az.intent: - return await evt.reply("No mxid given and the main intent is not a ghost user.") - else: - return await evt.reply("No mxid given and not in a portal.") + return await evt.reply("The avatar URL must start with `mxc://`") + try: - return await intent.set_avatar_url(mxc_uri) + return await puppet.default_mxid_intent.set_avatar_url(mxc_uri) except (MatrixRequestError, IntentError): evt.log.exception("Failed to set avatar.") return await evt.reply("Failed to set avatar (see logs for more details).") @@ -80,20 +105,12 @@ async def set_ghost_avatar(evt: CommandEvent) -> Optional[EventID]: help_args="[_mxid_]", help_text="Remove the avatar for a ghost user.", ) -async def remove_ghost_avatar(evt: CommandEvent) -> Optional[EventID]: - if len(evt.args) > 0: - puppet = await evt.processor.bridge.get_puppet(UserID(evt.args[0])) - if puppet is None: - return await evt.reply("The given mxid was not a valid ghost user.") - intent = puppet.intent - elif evt.is_portal: - intent = evt.portal.main_intent - if intent == evt.az.intent: - return await evt.reply("No mxid given and the main intent is not a ghost user.") - else: - return await evt.reply("No mxid given and not in a portal.") +async def remove_ghost_avatar(evt: CommandEvent) -> EventID | None: + puppet, err = await _get_mxid_param(evt, "[mxid]") + if err: + return err try: - return await intent.set_avatar_url(ContentURI("")) + return await puppet.default_mxid_intent.set_avatar_url(ContentURI("")) except (MatrixRequestError, IntentError): evt.log.exception("Failed to remove avatar.") return await evt.reply("Failed to remove avatar (see logs for more details).") @@ -104,29 +121,15 @@ async def remove_ghost_avatar(evt: CommandEvent) -> Optional[EventID]: needs_auth=False, name="set-displayname", help_section=SECTION_ADMIN, - help_args="<_displayname_> [_mxid_]", + help_args="[_mxid_] <_displayname_>", help_text="Set the display name for a ghost user.", ) -async def set_ghost_display_name(evt: CommandEvent) -> Optional[EventID]: - if len(evt.args) > 1: - # This allows whitespaces in the name - puppet = await evt.processor.bridge.get_puppet(UserID(evt.args[len(evt.args) - 1])) - if puppet is None: - return await evt.reply( - "The given mxid was not a valid ghost user. " - "If the display name has whitespaces mxid is required" - ) - intent = puppet.intent - displayname = " ".join(evt.args[:-1]) - elif evt.is_portal: - intent = evt.portal.main_intent - if intent == evt.az.intent: - return await evt.reply("No mxid given and the main intent is not a ghost user.") - displayname = evt.args[0] - else: - return await evt.reply("No mxid given and not in a portal.") +async def set_ghost_display_name(evt: CommandEvent) -> EventID | None: + puppet, err = await _get_mxid_param(evt, "[mxid] ") + if err: + return err try: - return await intent.set_displayname(displayname) + return await puppet.default_mxid_intent.set_displayname(" ".join(evt.args)) except (MatrixRequestError, IntentError): evt.log.exception("Failed to set display name.") return await evt.reply("Failed to set display name (see logs for more details).") @@ -140,20 +143,12 @@ async def set_ghost_display_name(evt: CommandEvent) -> Optional[EventID]: help_args="[_mxid_]", help_text="Remove the display name for a ghost user.", ) -async def set_ghost_display_name(evt: CommandEvent) -> Optional[EventID]: - if len(evt.args) > 0: - puppet = await evt.processor.bridge.get_puppet(UserID(evt.args[0])) - if puppet is None: - return await evt.reply("The given mxid was not a valid ghost user.") - intent = puppet.intent - elif evt.is_portal: - intent = evt.portal.main_intent - if intent == evt.az.intent: - return await evt.reply("No mxid given and the main intent is not a ghost user.") - else: - return await evt.reply("No mxid given and not in a portal (see logs for more details).") +async def remove_ghost_display_name(evt: CommandEvent) -> EventID | None: + puppet, err = await _get_mxid_param(evt, "[mxid]") + if err: + return err try: - return await intent.set_displayname("") + return await puppet.default_mxid_intent.set_displayname("") except (MatrixRequestError, IntentError): evt.log.exception("Failed to remove display name.") return await evt.reply("Failed to remove display name (see logs for more details).") From ec4732ca3b074f776df2a078c2702c8317236a53 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 5 Jul 2022 15:01:03 +0300 Subject: [PATCH 144/456] Fix one help string that was missed --- mautrix/bridge/commands/admin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/bridge/commands/admin.py b/mautrix/bridge/commands/admin.py index b3712135..80ff15e8 100644 --- a/mautrix/bridge/commands/admin.py +++ b/mautrix/bridge/commands/admin.py @@ -75,7 +75,7 @@ async def _get_mxid_param( needs_auth=False, name="set-avatar", help_section=SECTION_ADMIN, - help_args="<_mxc:// uri_> [_mxid_]", + help_args="[_mxid_] <_mxc:// uri_>", help_text="Set an avatar for a ghost user.", ) async def set_ghost_avatar(evt: CommandEvent) -> EventID | None: From 1e51a78c71c46769fbdbaebb5b18126d4a5bfb37 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 5 Jul 2022 15:15:15 +0300 Subject: [PATCH 145/456] Replace legacy encryption unsupported notice method --- mautrix/bridge/matrix.py | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/mautrix/bridge/matrix.py b/mautrix/bridge/matrix.py index 1726b4a9..90f9d585 100644 --- a/mautrix/bridge/matrix.py +++ b/mautrix/bridge/matrix.py @@ -521,12 +521,13 @@ async def _send_crypto_status_error( if wait_for: msg += f". The bridge will retry for {wait_for} seconds" + full_msg = f"\u26a0\ufe0f Your message was not bridged: {msg}." + if msg == "encryption is not supported": + full_msg = "🔒️ This bridge has not been configured to support encryption" event_id = None if self.config.get("bridge.delivery_error_reports", True): try: - content = TextMessageEventContent( - msgtype=MessageType.NOTICE, body=f"\u26a0 Your message was not bridged: {msg}." - ) + content = TextMessageEventContent(msgtype=MessageType.NOTICE, body=full_msg) if edit: content.set_edit(edit) event_id = await self.az.intent.send_message(evt.room_id, content) @@ -752,9 +753,12 @@ async def _post_decrypt( async def handle_encrypted(self, evt: EncryptedEvent) -> None: if not self.e2ee: + self.log.debug( + "Got encrypted message %s from %s, but encryption is not enabled", + evt.event_id, + evt.sender, + ) await self._send_crypto_status_error(evt, "encryption is not supported") - # TODO replace this with code in _send_crypto_status_error? - await self.handle_encrypted_unsupported(evt) return try: decrypted = await self.e2ee.decrypt(evt, wait_session_timeout=3) @@ -767,16 +771,6 @@ async def handle_encrypted(self, evt: EncryptedEvent) -> None: else: await self._post_decrypt(decrypted) - async def handle_encrypted_unsupported(self, evt: EncryptedEvent) -> None: - self.log.debug( - "Got encrypted message %s from %s, but encryption is not enabled", - evt.event_id, - evt.sender, - ) - await self.az.intent.send_notice( - evt.room_id, "🔒️ This bridge has not been configured to support encryption" - ) - async def _handle_encrypted_wait( self, evt: EncryptedEvent, err: SessionNotFound, wait: int ) -> None: From 4a36fc50fe51025e0fb54ed6ffd6e473eca8b2df Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 5 Jul 2022 15:41:33 +0300 Subject: [PATCH 146/456] Bump version to 0.17.0 --- CHANGELOG.md | 22 ++++++++++++++++++++++ mautrix/__init__.py | 2 +- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a2db57d..a9337e8d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,25 @@ +## v0.17.0 (2022-07-05) + +* **Breaking change *(bridge)*** Added options to check cross-signing status + for bridge users. This requires changes to the base config. + * New options include requiring cross-signed devices (with TOFU) for sending + and/or receiving messages, and an option to drop any unencrypted messages. +* **Breaking change *(crypto)*** Removed `sender_key` parameter from + CryptoStore's `has_group_session` and `put_group_session`, and also + OlmMachine's `wait_for_session`. +* **Breaking change *(crypto.store.memory)*** Updated the key of the + `_inbound_sessions` dict to be (room_id, session_id), removing the identity + key in the middle. This only affects custom stores based on the memory store. +* *(crypto)* Added basic cross-signing validation code. +* *(crypto)* Marked device_id and sender_key as deprecated in Megolm events + as per Matrix 1.3. +* *(api)* Bumped request logs to `DEBUG` level. + * Also added new `sensitive` parameter to the `request` method to prevent + logging content in sensitive requests. The `login` method was updated to + mark the content as sensitive if a password or token is provided. +* *(bridge.commands)* Switched the order of the user ID parameter in `set-pl`, + `set-avatar` and `set-displayname`. + ## v0.16.11 (2022-06-28) * *(appservice)* Fixed the `extra_content` parameter in membership methods diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 20dd361e..b80b70da 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.16.11" +__version__ = "0.17.0" __author__ = "Tulir Asokan " __all__ = [ "api", From 21ffe8b00df9002bbfa7675a552cdd91826cdc0b Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 5 Jul 2022 18:22:28 +0300 Subject: [PATCH 147/456] Fix old Python versions and improve device validation logs --- mautrix/crypto/device_lists.py | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/mautrix/crypto/device_lists.py b/mautrix/crypto/device_lists.py index 0cb67e5f..42c17074 100644 --- a/mautrix/crypto/device_lists.py +++ b/mautrix/crypto/device_lists.py @@ -3,7 +3,7 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from typing import Dict, List, Optional +from __future__ import annotations from mautrix.errors import DeviceValidationError from mautrix.types import ( @@ -28,8 +28,8 @@ class DeviceListMachine(BaseOlmMachine): async def _fetch_keys( - self, users: List[UserID], since: SyncToken = "", include_untracked: bool = False - ) -> Dict[UserID, Dict[DeviceID, DeviceIdentity]]: + self, users: list[UserID], since: SyncToken = "", include_untracked: bool = False + ) -> dict[UserID, dict[DeviceID, DeviceIdentity]]: if not include_untracked: users = await self.crypto_store.filter_tracked_users(users) if len(users) == 0: @@ -211,7 +211,7 @@ async def _store_cross_signing_keys(self, resp: QueryKeysResponse, user_id: User async def get_or_fetch_device( self, user_id: UserID, device_id: DeviceID - ) -> Optional[DeviceIdentity]: + ) -> DeviceIdentity | None: device = await self.crypto_store.get_device(user_id, device_id) if device is not None: return device @@ -223,7 +223,7 @@ async def get_or_fetch_device( async def get_or_fetch_device_by_key( self, user_id: UserID, identity_key: IdentityKey - ) -> Optional[DeviceIdentity]: + ) -> DeviceIdentity | None: device = await self.crypto_store.find_device_by_key(user_id, identity_key) if device is not None: return device @@ -245,12 +245,16 @@ async def _validate_device( user_id: UserID, device_id: DeviceID, device_keys: DeviceKeys, - existing: Optional[DeviceIdentity] = None, + existing: DeviceIdentity | None = None, ) -> DeviceIdentity: if user_id != device_keys.user_id: - raise DeviceValidationError("mismatching user ID in parameter and keys object") + raise DeviceValidationError( + f"mismatching user ID (expected {user_id}, got {device_keys.user_id})" + ) elif device_id != device_keys.device_id: - raise DeviceValidationError("mismatching device ID in parameter and keys object") + raise DeviceValidationError( + f"mismatching device ID (expected {device_id}, got {device_keys.device_id})" + ) signing_key = device_keys.ed25519 if not signing_key: @@ -260,7 +264,10 @@ async def _validate_device( raise DeviceValidationError("didn't find curve25519 identity key") if existing and existing.signing_key != signing_key: - raise DeviceValidationError("received update for device with different signing key") + raise DeviceValidationError( + f"received update for device with different signing key " + f"(expected {existing.signing_key}, got {signing_key})" + ) if not verify_signature_json(device_keys.serialize(), user_id, device_id, signing_key): raise DeviceValidationError("invalid signature on device keys") From 3f8d60516bc057a8fe3e431d0e1c847f81f8e4de Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 5 Jul 2022 19:10:38 +0300 Subject: [PATCH 148/456] Add and update tests --- .../client/state_store/tests/store_test.py | 9 +- .../attachments/async_attachments_test.py | 44 ++++++ mautrix/crypto/attachments/attachments.py | 24 ++-- .../crypto/attachments/attachments_test.py | 111 +++++++++++++++ mautrix/crypto/store/tests/__init__.py | 0 mautrix/crypto/store/tests/store_test.py | 130 ++++++++++++++++++ mautrix/util/formatter/parser_test.py | 2 - 7 files changed, 305 insertions(+), 15 deletions(-) create mode 100644 mautrix/crypto/attachments/async_attachments_test.py create mode 100644 mautrix/crypto/attachments/attachments_test.py create mode 100644 mautrix/crypto/store/tests/__init__.py create mode 100644 mautrix/crypto/store/tests/store_test.py diff --git a/mautrix/client/state_store/tests/store_test.py b/mautrix/client/state_store/tests/store_test.py index 6bceb673..dbbd376a 100644 --- a/mautrix/client/state_store/tests/store_test.py +++ b/mautrix/client/state_store/tests/store_test.py @@ -3,7 +3,9 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from typing import AsyncContextManager, AsyncIterator, Callable, Dict, List +from __future__ import annotations + +from typing import AsyncContextManager, AsyncIterator, Callable from contextlib import asynccontextmanager import json import os @@ -83,7 +85,7 @@ async def store(request) -> AsyncIterator[StateStore]: yield state_store -def read_state_file(request, file) -> Dict[RoomID, List[StateEvent]]: +def read_state_file(request, file) -> dict[RoomID, list[StateEvent]]: path = pathlib.Path(request.node.fspath).with_name(file) with path.open() as fp: content = json.load(fp) @@ -122,7 +124,6 @@ async def get_joined_members(request, store: StateStore) -> None: await store.set_members(room_id, parsed_members, only_membership=Membership.JOIN) -@pytest.mark.asyncio async def test_basic(store: StateStore) -> None: room_id = RoomID("!foo:example.com") user_id = UserID("@tulir:example.com") @@ -136,7 +137,6 @@ async def test_basic(store: StateStore) -> None: assert await store.is_encrypted(RoomID("!unknown-room:example.com")) is None -@pytest.mark.asyncio async def test_basic_updated(request, store: StateStore) -> None: await store_room_state(request, store) test_group = RoomID("!telegram-group:example.com") @@ -145,7 +145,6 @@ async def test_basic_updated(request, store: StateStore) -> None: assert not await store.is_encrypted(RoomID("!unencrypted-room:example.com")) -@pytest.mark.asyncio async def test_updates(request, store: StateStore) -> None: await store_room_state(request, store) room_id = RoomID("!telegram-group:example.com") diff --git a/mautrix/crypto/attachments/async_attachments_test.py b/mautrix/crypto/attachments/async_attachments_test.py new file mode 100644 index 00000000..931ee894 --- /dev/null +++ b/mautrix/crypto/attachments/async_attachments_test.py @@ -0,0 +1,44 @@ +# Copyright © 2019 Damir Jelić (under the Apache 2.0 license) +# Copyright © 2019 miruka (under the Apache 2.0 license) +# Copyright (c) 2022 Tulir Asokan +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +from mautrix.types import EncryptedFile + +from .async_attachments import async_encrypt_attachment, async_inplace_encrypt_attachment +from .attachments import decrypt_attachment + +try: + from Crypto import Random +except ImportError: + from Cryptodome import Random + + +async def _get_data_cypher_keys(data: bytes) -> tuple[bytes, EncryptedFile]: + *chunks, keys = [i async for i in async_encrypt_attachment(data)] + return b"".join(chunks), keys + + +async def test_async_encrypt(): + data = b"Test bytes" + + cyphertext, keys = await _get_data_cypher_keys(data) + + plaintext = decrypt_attachment(cyphertext, keys.key.key, keys.hashes["sha256"], keys.iv) + + assert data == plaintext + + +async def test_async_inplace_encrypt(): + orig_data = b"Test bytes" + data = bytearray(orig_data) + + keys = await async_inplace_encrypt_attachment(data) + + assert data != orig_data + + decrypt_attachment(data, keys.key.key, keys.hashes["sha256"], keys.iv, inplace=True) + + assert data == orig_data diff --git a/mautrix/crypto/attachments/attachments.py b/mautrix/crypto/attachments/attachments.py index 7d91c745..80b62212 100644 --- a/mautrix/crypto/attachments/attachments.py +++ b/mautrix/crypto/attachments/attachments.py @@ -29,7 +29,9 @@ from Cryptodome.Util import Counter -def decrypt_attachment(ciphertext: bytes, key: str, hash: str, iv: str) -> bytes: +def decrypt_attachment( + ciphertext: bytes | bytearray | memoryview, key: str, hash: str, iv: str, inplace: bool = False +) -> bytes: """Decrypt an encrypted attachment. Args: @@ -37,12 +39,12 @@ def decrypt_attachment(ciphertext: bytes, key: str, hash: str, iv: str) -> bytes key: AES_CTR JWK key object. hash: Base64 encoded SHA-256 hash of the ciphertext. iv: Base64 encoded 16 byte AES-CTR IV. + inplace: Should the decryption be performed in-place? + The input must be a bytearray or writable memoryview to use this. Returns: The plaintext bytes. Raises: EncryptionError: if the integrity check fails. - - """ expected_hash = unpaddedbase64.decode_base64(hash) @@ -50,21 +52,23 @@ def decrypt_attachment(ciphertext: bytes, key: str, hash: str, iv: str) -> bytes h.update(ciphertext) if h.digest() != expected_hash: - raise DecryptionError("Mismatched SHA-256 digest.") + raise DecryptionError("Mismatched SHA-256 digest") try: byte_key: bytes = unpaddedbase64.decode_base64(key) except (binascii.Error, TypeError): - raise DecryptionError("Error decoding key.") + raise DecryptionError("Error decoding key") try: byte_iv: bytes = unpaddedbase64.decode_base64(iv) + if len(byte_iv) != 16: + raise DecryptionError("Invalid IV length") prefix = byte_iv[:8] # A non-zero IV counter is not spec-compliant, but some clients still do it, # so decode the counter part too. initial_value = struct.unpack(">Q", byte_iv[8:])[0] except (binascii.Error, TypeError, IndexError, struct.error): - raise DecryptionError("Error decoding initial values.") + raise DecryptionError("Error decoding IV") ctr = Counter.new(64, prefix=prefix, initial_value=initial_value) @@ -73,7 +77,11 @@ def decrypt_attachment(ciphertext: bytes, key: str, hash: str, iv: str) -> bytes except ValueError as e: raise DecryptionError("Failed to create AES cipher") from e - return cipher.decrypt(ciphertext) + if inplace: + cipher.decrypt(ciphertext, ciphertext) + return ciphertext + else: + return cipher.decrypt(ciphertext) def encrypt_attachment(plaintext: bytes) -> tuple[bytes, EncryptedFile]: @@ -103,7 +111,7 @@ def _prepare_encryption() -> tuple[bytes, bytes, AES, SHA256.SHA256Hash]: return key, iv, cipher, sha256 -def inplace_encrypt_attachment(data: bytearray) -> EncryptedFile: +def inplace_encrypt_attachment(data: bytearray | memoryview) -> EncryptedFile: key, iv, cipher, sha256 = _prepare_encryption() cipher.encrypt(plaintext=data, output=data) diff --git a/mautrix/crypto/attachments/attachments_test.py b/mautrix/crypto/attachments/attachments_test.py new file mode 100644 index 00000000..a8cb83b6 --- /dev/null +++ b/mautrix/crypto/attachments/attachments_test.py @@ -0,0 +1,111 @@ +# Copyright © 2019 Damir Jelić (under the Apache 2.0 license) +# Copyright (c) 2022 Tulir Asokan +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +import pytest +import unpaddedbase64 + +from mautrix.errors import DecryptionError + +from .attachments import decrypt_attachment, encrypt_attachment, inplace_encrypt_attachment + +try: + from Crypto import Random +except ImportError: + from Cryptodome import Random + + +def test_encrypt(): + data = b"Test bytes" + + cyphertext, keys = encrypt_attachment(data) + + plaintext = decrypt_attachment(cyphertext, keys.key.key, keys.hashes["sha256"], keys.iv) + + assert data == plaintext + + +def test_inplace_encrypt(): + orig_data = b"Test bytes" + data = bytearray(orig_data) + + keys = inplace_encrypt_attachment(data) + + assert data != orig_data + + decrypt_attachment(data, keys.key.key, keys.hashes["sha256"], keys.iv, inplace=True) + + assert data == orig_data + + +def test_hash_verification(): + data = b"Test bytes" + + cyphertext, keys = encrypt_attachment(data) + + with pytest.raises(DecryptionError): + decrypt_attachment(cyphertext, keys.key.key, "Fake hash", keys.iv) + + +def test_invalid_key(): + data = b"Test bytes" + + cyphertext, keys = encrypt_attachment(data) + + with pytest.raises(DecryptionError): + decrypt_attachment(cyphertext, "Fake key", keys.hashes["sha256"], keys.iv) + + +def test_invalid_iv(): + data = b"Test bytes" + + cyphertext, keys = encrypt_attachment(data) + + with pytest.raises(DecryptionError): + decrypt_attachment(cyphertext, keys.key.key, keys.hashes["sha256"], "Fake iv") + + +def test_short_key(): + data = b"Test bytes" + + cyphertext, keys = encrypt_attachment(data) + + with pytest.raises(DecryptionError): + decrypt_attachment( + cyphertext, + unpaddedbase64.encode_base64(b"Fake key", urlsafe=True), + keys["hashes"]["sha256"], + keys["iv"], + ) + + +def test_short_iv(): + data = b"Test bytes" + + cyphertext, keys = encrypt_attachment(data) + + with pytest.raises(DecryptionError): + decrypt_attachment( + cyphertext, + keys.key.key, + keys.hashes["sha256"], + unpaddedbase64.encode_base64(b"F" + b"\x00" * 8), + ) + + +def test_fake_key(): + data = b"Test bytes" + + cyphertext, keys = encrypt_attachment(data) + + fake_key = Random.new().read(32) + + plaintext = decrypt_attachment( + cyphertext, + unpaddedbase64.encode_base64(fake_key, urlsafe=True), + keys["hashes"]["sha256"], + keys["iv"], + ) + assert plaintext != data diff --git a/mautrix/crypto/store/tests/__init__.py b/mautrix/crypto/store/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/mautrix/crypto/store/tests/store_test.py b/mautrix/crypto/store/tests/store_test.py new file mode 100644 index 00000000..3c54eaa1 --- /dev/null +++ b/mautrix/crypto/store/tests/store_test.py @@ -0,0 +1,130 @@ +# Copyright (c) 2022 Tulir Asokan +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +from typing import AsyncContextManager, AsyncIterator, Callable +from contextlib import asynccontextmanager +import os +import random +import string +import time + +import asyncpg +import pytest + +from mautrix.client.state_store import SyncStore +from mautrix.crypto import InboundGroupSession, OlmAccount, OutboundGroupSession +from mautrix.types import DeviceID, EventID, RoomID, SessionID, SyncToken +from mautrix.util.async_db import Database + +from .. import CryptoStore, MemoryCryptoStore, PgCryptoStore + + +@asynccontextmanager +async def async_postgres_store() -> AsyncIterator[PgCryptoStore]: + try: + pg_url = os.environ["MEOW_TEST_PG_URL"] + except KeyError: + pytest.skip("Skipped Postgres tests (MEOW_TEST_PG_URL not specified)") + return + conn: asyncpg.Connection = await asyncpg.connect(pg_url) + schema_name = "".join(random.choices(string.ascii_lowercase, k=8)) + schema_name = f"test_schema_{schema_name}_{int(time.time())}" + await conn.execute(f"CREATE SCHEMA {schema_name}") + db = Database.create( + pg_url, + upgrade_table=PgCryptoStore.upgrade_table, + db_args={"min_size": 1, "max_size": 3, "server_settings": {"search_path": schema_name}}, + ) + store = PgCryptoStore("", "test", db) + await db.start() + yield store + await db.stop() + await conn.execute(f"DROP SCHEMA {schema_name} CASCADE") + await conn.close() + + +@asynccontextmanager +async def async_sqlite_store() -> AsyncIterator[PgCryptoStore]: + db = Database.create( + "sqlite:///:memory:", upgrade_table=PgCryptoStore.upgrade_table, db_args={"min_size": 1} + ) + store = PgCryptoStore("", "test", db) + await db.start() + yield store + await db.stop() + + +@asynccontextmanager +async def memory_store() -> AsyncIterator[MemoryCryptoStore]: + yield MemoryCryptoStore("", "test") + + +@pytest.fixture(params=[async_postgres_store, async_sqlite_store, memory_store]) +async def crypto_store(request) -> AsyncIterator[CryptoStore]: + param: Callable[[], AsyncContextManager[CryptoStore]] = request.param + async with param() as state_store: + yield state_store + + +async def test_basic(crypto_store: CryptoStore) -> None: + acc = OlmAccount() + keys = acc.identity_keys + await crypto_store.put_account(acc) + await crypto_store.put_device_id(DeviceID("TEST")) + if isinstance(crypto_store, SyncStore): + await crypto_store.put_next_batch(SyncToken("TEST")) + + assert await crypto_store.get_device_id() == "TEST" + assert (await crypto_store.get_account()).identity_keys == keys + if isinstance(crypto_store, SyncStore): + assert await crypto_store.get_next_batch() == "TEST" + + +def _make_group_sess( + acc: OlmAccount, room_id: RoomID +) -> tuple[InboundGroupSession, OutboundGroupSession]: + outbound = OutboundGroupSession(room_id) + inbound = InboundGroupSession( + session_key=outbound.session_key, + signing_key=acc.signing_key, + sender_key=acc.identity_key, + room_id=room_id, + ) + return inbound, outbound + + +async def test_validate_message_index(crypto_store: CryptoStore) -> None: + acc = OlmAccount() + + inbound, outbound = _make_group_sess(acc, RoomID("!foo:bar.com")) + outbound.shared = True + orig_plaintext = "hello world" + ciphertext = outbound.encrypt(orig_plaintext) + ts = int(time.time() * 1000) + plaintext, index = inbound.decrypt(ciphertext) + assert plaintext == orig_plaintext + + assert await crypto_store.validate_message_index( + acc.identity_key, SessionID(inbound.id), EventID("$foo"), index, ts + ), "Initial validation returns True" + assert await crypto_store.validate_message_index( + acc.identity_key, SessionID(inbound.id), EventID("$foo"), index, ts + ), "Validating the same details again returns True" + assert not await crypto_store.validate_message_index( + acc.identity_key, SessionID(inbound.id), EventID("$bar"), index, ts + ), "Different event ID causes validation to fail" + assert not await crypto_store.validate_message_index( + acc.identity_key, SessionID(inbound.id), EventID("$foo"), index, ts + 1 + ), "Different timestamp causes validation to fail" + assert not await crypto_store.validate_message_index( + acc.identity_key, SessionID(inbound.id), EventID("$foo"), index, ts + 1 + ), "Validating incorrect details twice fails" + assert await crypto_store.validate_message_index( + acc.identity_key, SessionID(inbound.id), EventID("$foo"), index, ts + ), "Validating the same details after fails still returns True" + + +# TODO tests for device identity storage, group session storage +# and cross-signing key/signature storage diff --git a/mautrix/util/formatter/parser_test.py b/mautrix/util/formatter/parser_test.py index 730b62b7..ccfc2724 100644 --- a/mautrix/util/formatter/parser_test.py +++ b/mautrix/util/formatter/parser_test.py @@ -8,7 +8,6 @@ from . import parse_html -@pytest.mark.asyncio async def test_basic_markdown() -> None: tests = { "test": "**test**", @@ -27,7 +26,6 @@ async def test_basic_markdown() -> None: assert await parse_html(html) == markdown_ish -@pytest.mark.asyncio async def test_nested_markdown() -> None: input_html = """

Hello, World!

From 8bb54afdad4c8de0bb3e674c52f3b30779aa6b05 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 5 Jul 2022 19:14:35 +0300 Subject: [PATCH 149/456] Add encryption dependencies to tests --- setup.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 08a10cbb..f7596154 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,8 @@ from mautrix import __version__ -test_dependencies = ["aiosqlite", "sqlalchemy", "asyncpg"] +encryption_dependencies = ["python-olm", "unpaddedbase64", "pycryptodome"] +test_dependencies = ["aiosqlite", "sqlalchemy", "asyncpg", *encryption_dependencies] setuptools.setup( name="mautrix", @@ -29,6 +30,7 @@ "detect_mimetype": ["python-magic>=0.4.15,<0.5"], "lint": ["black==22.1.0", "isort"], "test": ["pytest", "pytest-asyncio", *test_dependencies], + "encryption": encryption_dependencies, }, tests_require=test_dependencies, python_requires="~=3.8", From 6a4b2df7d33ab3ea89077da58caa028558d36c52 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 5 Jul 2022 19:19:00 +0300 Subject: [PATCH 150/456] Install libolm in actions --- .github/workflows/python-package.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 70e8cf65..81c07712 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -17,9 +17,12 @@ jobs: uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} + - name: Install libolm + run: sudo apt-get install libolm3 - name: Install dependencies run: | python -m pip install --upgrade pip + python -m pip install python-olm --extra-index-url https://gitlab.matrix.org/api/v4/projects/27/packages/pypi/simple python -m pip install .[test] - name: Test with pytest run: | From 2b49a503f2b99b01d0acea1c8c4ad172913816d8 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 5 Jul 2022 19:21:50 +0300 Subject: [PATCH 151/456] Fix new test on Python 3.8 --- mautrix/crypto/store/tests/store_test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mautrix/crypto/store/tests/store_test.py b/mautrix/crypto/store/tests/store_test.py index 3c54eaa1..8d3fc851 100644 --- a/mautrix/crypto/store/tests/store_test.py +++ b/mautrix/crypto/store/tests/store_test.py @@ -3,6 +3,8 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. +from __future__ import annotations + from typing import AsyncContextManager, AsyncIterator, Callable from contextlib import asynccontextmanager import os From 79f8969683729482416a6fb4d91a35a67f273484 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 5 Jul 2022 19:24:32 +0300 Subject: [PATCH 152/456] Fix another new test on Python 3.8 --- mautrix/crypto/attachments/async_attachments_test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mautrix/crypto/attachments/async_attachments_test.py b/mautrix/crypto/attachments/async_attachments_test.py index 931ee894..571f8ead 100644 --- a/mautrix/crypto/attachments/async_attachments_test.py +++ b/mautrix/crypto/attachments/async_attachments_test.py @@ -5,6 +5,8 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. +from __future__ import annotations + from mautrix.types import EncryptedFile from .async_attachments import async_encrypt_attachment, async_inplace_encrypt_attachment From c19e11a452e066f412c17a84592ce147b1ba6fe6 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 5 Jul 2022 20:00:36 +0300 Subject: [PATCH 153/456] Bump version to 0.17.1 --- CHANGELOG.md | 6 ++++++ mautrix/__init__.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a9337e8d..1e876dfa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## v0.17.1 (2022-07-05) + +* *(crypto)* Fixed Python 3.8/9 compatibility broken in v0.17.0. +* *(crypto)* Added some tests for attachments and store code. +* *(crypto)* Improved logging when device change validation fails. + ## v0.17.0 (2022-07-05) * **Breaking change *(bridge)*** Added options to check cross-signing status diff --git a/mautrix/__init__.py b/mautrix/__init__.py index b80b70da..92e3a0b2 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.17.0" +__version__ = "0.17.1" __author__ = "Tulir Asokan " __all__ = [ "api", From 0ac4135a2d2c928ceba6ee0f55e697f294069ed2 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 6 Jul 2022 14:38:06 +0300 Subject: [PATCH 154/456] Add beeper_new_messages flag for batch sending --- mautrix/appservice/api/intent.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/mautrix/appservice/api/intent.py b/mautrix/appservice/api/intent.py index 3fd1600b..d2d06f36 100644 --- a/mautrix/appservice/api/intent.py +++ b/mautrix/appservice/api/intent.py @@ -496,6 +496,7 @@ async def batch_send( batch_id: BatchID | None = None, events: Iterable[BatchSendEvent], state_events_at_start: Iterable[BatchSendStateEvent] = (), + beeper_new_messages: bool = False, ) -> BatchSendResponse: """ Send a batch of historical events into a room. See `MSC2716`_ for more info. @@ -513,6 +514,8 @@ async def batch_send( state_events_at_start: The state events to send at the start of the batch. These will be sent as outlier events, which means they won't be a part of the actual room state. + beeper_new_messages: Custom flag to tell the server that the messages can be sent to + the end of the room as normal messages instead of history. Returns: All the event IDs generated, plus a batch ID that can be passed back to this method. @@ -521,6 +524,8 @@ async def batch_send( query: JSON = {"prev_event_id": prev_event_id} if batch_id: query["batch_id"] = batch_id + if beeper_new_messages: + query["com.beeper.new_messages"] = "true" resp = await self.api.request( Method.POST, path, From 635d4039d52e9ef122e5032c8a6ed4a0b2b3e202 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 6 Jul 2022 14:38:14 +0300 Subject: [PATCH 155/456] Log full URL when making requests --- mautrix/api.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/mautrix/api.py b/mautrix/api.py index 88a75921..2311ff88 100644 --- a/mautrix/api.py +++ b/mautrix/api.py @@ -263,7 +263,7 @@ async def _send( def _log_request( self, method: Method, - path: PathBuilder, + url: URL, content: str | bytes | bytearray | AsyncBody, orig_content, query_params: dict[str, str], @@ -283,15 +283,15 @@ def _log_request( else: log_content = content as_user = query_params.get("user_id", None) - level = 5 if path == Path.v3.sync else 10 + level = 5 if url.path.endswith("/v3/sync") else 10 self.log.log( level, - f"req #{req_id}: {method} /{path} {log_content}".strip(" "), + f"req #{req_id}: {method} {url} {log_content}".strip(" "), extra={ "matrix_http_request": { "req_id": req_id, "method": str(method), - "path": str(path), + "url": str(url), "content": ( orig_content if isinstance(orig_content, (dict, list)) and not sensitive @@ -387,9 +387,10 @@ async def request( if do_fake_iter: headers["Content-Length"] = str(len(content)) backoff = 4 + log_url = full_url.with_query(query_params) while True: self._log_request( - method, path, content, orig_content, query_params, headers, req_id, sensitive + method, log_url, content, orig_content, query_params, headers, req_id, sensitive ) API_CALLS.labels(method=metrics_method).inc() req_content = _async_iter_bytes(content) if do_fake_iter else content From c15699106ba708580cc5c1453e49d46c05cc0c4f Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 6 Jul 2022 19:06:50 +0300 Subject: [PATCH 156/456] Fix typo in config upgrade --- mautrix/bridge/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/bridge/config.py b/mautrix/bridge/config.py index d4441d94..5f55ca91 100644 --- a/mautrix/bridge/config.py +++ b/mautrix/bridge/config.py @@ -109,7 +109,7 @@ def do_update(self, helper: ConfigUpdateHelper) -> None: copy("bridge.encryption.verification_levels.send") copy("bridge.encryption.verification_levels.share") copy("bridge.encryption.allow_key_sharing") - if self.get("bridge.encryption.key_sharing_allow", False): + if self.get("bridge.encryption.key_sharing.allow", False): helper.base["bridge.encryption.allow_key_sharing"] = True require_verif = self.get("bridge.encryption.key_sharing.require_verification", True) require_cs = self.get("bridge.encryption.key_sharing.require_cross_signing", False) From baa97a46f0e00d7f04b045b7f16350cf6dc4fa35 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 6 Jul 2022 19:11:42 +0300 Subject: [PATCH 157/456] Bump version to 0.17.2 --- CHANGELOG.md | 6 ++++++ mautrix/__init__.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e876dfa..ec24c691 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## v0.17.2 (2022-07-06) + +* *(api)* Updated request logging to log full URL instead of only path. +* *(bridge)* Fixed migrating key sharing allow flag to new config format. +* *(appservice)* Added `beeper_new_messages` flag for `batch_send` method. + ## v0.17.1 (2022-07-05) * *(crypto)* Fixed Python 3.8/9 compatibility broken in v0.17.0. diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 92e3a0b2..c2d62cda 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.17.1" +__version__ = "0.17.2" __author__ = "Tulir Asokan " __all__ = [ "api", From a63c04314f59979f6181468a1a0d743ea03247ab Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 12 Jul 2022 14:58:05 +0300 Subject: [PATCH 158/456] Update decryption error status events --- mautrix/bridge/matrix.py | 74 ++++++++++++++++++++++----------- mautrix/errors/crypto.py | 8 +++- mautrix/types/__init__.py | 2 + mautrix/types/event/__init__.py | 7 +++- mautrix/types/event/beeper.py | 23 ++++++++-- 5 files changed, 84 insertions(+), 30 deletions(-) diff --git a/mautrix/bridge/matrix.py b/mautrix/bridge/matrix.py index 90f9d585..e4729424 100644 --- a/mautrix/bridge/matrix.py +++ b/mautrix/bridge/matrix.py @@ -33,6 +33,7 @@ MemberStateEventContent, MessageEvent, MessageEventContent, + MessageStatus, MessageStatusReason, MessageType, PresenceEvent, @@ -93,6 +94,44 @@ ) +class UnencryptedMessageError(DecryptionError): + def __init__(self) -> None: + super().__init__("unencrypted message") + + @property + def human_message(self) -> str: + return "the message is not encrypted" + + +class EncryptionUnsupportedError(DecryptionError): + def __init__(self) -> None: + super().__init__("encryption is not supported") + + @property + def human_message(self) -> str: + return "the bridge is not configured to support encryption" + + +class DeviceUntrustedError(DecryptionError): + def __init__(self, trust: TrustState) -> None: + explanation = { + TrustState.BLACKLISTED: "device is blacklisted", + TrustState.UNVERIFIED: "unverified", + TrustState.UNKNOWN_DEVICE: "device info not found", + TrustState.FORWARDED: "keys were forwarded from an unknown device", + TrustState.CROSS_SIGNED_UNTRUSTED: ( + "cross-signing keys changed after setting up the bridge" + ), + }.get(trust) + base = "your device is not trusted" + self.message = f"{base} ({explanation})" if explanation else base + super().__init__(self.message) + + @property + def human_message(self) -> str: + return self.message + + class BaseMatrixHandler: log: TraceLogger = logging.getLogger("mau.mx") az: AppService @@ -506,23 +545,23 @@ def is_command(self, message: MessageEventContent) -> tuple[bool, str]: async def _send_crypto_status_error( self, evt: Event, - err: Exception | str | None = None, + err: DecryptionError | None = None, retry_num: int = 0, is_final: bool = True, edit: EventID | None = None, wait_for: int | None = None, ) -> EventID | None: msg = str(err) - if isinstance(err, SessionNotFound): - msg = "the bridge hasn't received the decryption keys" + if isinstance(err, (SessionNotFound, UnencryptedMessageError)): + msg = err.human_message self._send_message_checkpoint( evt, MessageSendCheckpointStep.DECRYPTED, msg, permanent=is_final, retry_num=retry_num ) if wait_for: msg += f". The bridge will retry for {wait_for} seconds" - full_msg = f"\u26a0\ufe0f Your message was not bridged: {msg}." - if msg == "encryption is not supported": + full_msg = f"\u26a0 Your message was not bridged: {msg}." + if isinstance(err, EncryptionUnsupportedError): full_msg = "🔒️ This bridge has not been configured to support encryption" event_id = None if self.config.get("bridge.delivery_error_reports", True): @@ -544,12 +583,12 @@ async def _send_crypto_status_error( status_content = BeeperMessageStatusEventContent( network="", # TODO set network properly relates_to=RelatesTo(rel_type=RelationType.REFERENCE, event_id=evt.event_id), - success=False, - is_certain=True, - can_retry=True, + status=MessageStatus.RETRIABLE if is_final else MessageStatus.PENDING, reason=MessageStatusReason.UNDECRYPTABLE, error=msg, + message=err.human_message if err else None, ) + status_content.fill_legacy_booleans() await self.az.intent.send_message_event( evt.room_id, EventType.BEEPER_MESSAGE_STATUS, status_content ) @@ -564,7 +603,7 @@ async def handle_message(self, evt: MessageEvent, was_encrypted: bool = False) - if not was_encrypted and self.require_e2ee: self.log.warning(f"Dropping {event_id} from {user_id} as it's not encrypted!") - await self._send_crypto_status_error(evt, "unencrypted message", 0) + await self._send_crypto_status_error(evt, UnencryptedMessageError(), 0) return sender = await self.bridge.get_user(user_id) @@ -714,19 +753,6 @@ async def try_handle_sync_event(self, evt: Event) -> None: except Exception: self.log.exception("Error handling manually received Matrix event") - @staticmethod - def _device_unverified_explanation(trust: TrustState) -> str: - explanation = { - TrustState.BLACKLISTED: "device is blacklisted", - TrustState.UNKNOWN_DEVICE: "device info not found", - TrustState.FORWARDED: "keys were forwarded from an unknown device", - TrustState.CROSS_SIGNED_UNTRUSTED: ( - "cross-signing keys changed after setting up the bridge" - ), - }.get(trust) - base = "your device is not trusted" - return f"{base} ({explanation})" if explanation else base - async def _post_decrypt( self, evt: Event, retry_num: int = 0, error_event_id: EventID | None = None ) -> None: @@ -739,7 +765,7 @@ async def _post_decrypt( await self._send_crypto_status_error( evt, retry_num=retry_num, - err=self._device_unverified_explanation(trust_state), + err=DeviceUntrustedError(trust_state), edit=error_event_id, ) return @@ -758,7 +784,7 @@ async def handle_encrypted(self, evt: EncryptedEvent) -> None: evt.event_id, evt.sender, ) - await self._send_crypto_status_error(evt, "encryption is not supported") + await self._send_crypto_status_error(evt, EncryptionUnsupportedError()) return try: decrypted = await self.e2ee.decrypt(evt, wait_session_timeout=3) diff --git a/mautrix/errors/crypto.py b/mautrix/errors/crypto.py index 3e4cf5dc..97592b05 100644 --- a/mautrix/errors/crypto.py +++ b/mautrix/errors/crypto.py @@ -27,7 +27,9 @@ class SessionShareError(CryptoError): class DecryptionError(CryptoError): - pass + @property + def human_message(self) -> str: + return "the bridge failed to decrypt the message" class MatchingSessionDecryptionError(DecryptionError): @@ -42,6 +44,10 @@ def __init__(self, session_id: SessionID, sender_key: IdentityKey | None = None) self.session_id = session_id self._sender_key = sender_key + @property + def human_message(self) -> str: + return "the bridge hasn't received the decryption keys" + @property def sender_key(self) -> IdentityKey | None: """ diff --git a/mautrix/types/__init__.py b/mautrix/types/__init__.py index 7e134626..b8bf5d5d 100644 --- a/mautrix/types/__init__.py +++ b/mautrix/types/__init__.py @@ -86,6 +86,7 @@ MemberStateEventContent, MessageEvent, MessageEventContent, + MessageStatus, MessageStatusReason, MessageType, MessageUnsigned, @@ -277,6 +278,7 @@ "MemberStateEventContent", "MessageEvent", "MessageEventContent", + "MessageStatus", "MessageStatusReason", "MessageType", "MessageUnsigned", diff --git a/mautrix/types/event/__init__.py b/mautrix/types/event/__init__.py index 1515bc9a..d9a1d681 100644 --- a/mautrix/types/event/__init__.py +++ b/mautrix/types/event/__init__.py @@ -11,7 +11,12 @@ ) from .base import BaseEvent, BaseRoomEvent, BaseUnsigned, GenericEvent from .batch import BatchSendEvent, BatchSendStateEvent -from .beeper import BeeperMessageStatusEvent, BeeperMessageStatusEventContent, MessageStatusReason +from .beeper import ( + BeeperMessageStatusEvent, + BeeperMessageStatusEventContent, + MessageStatus, + MessageStatusReason, +) from .encrypted import ( EncryptedEvent, EncryptedEventContent, diff --git a/mautrix/types/event/beeper.py b/mautrix/types/event/beeper.py index 6c3c6010..5ceb2373 100644 --- a/mautrix/types/event/beeper.py +++ b/mautrix/types/event/beeper.py @@ -32,21 +32,36 @@ def checkpoint_status(self): return MessageSendCheckpointStatus.PERM_FAILURE -@dataclass +class MessageStatus(SerializableEnum): + SUCCESS = "SUCCESS" + PENDING = "PENDNIG" + RETRIABLE = "FAIL_RETRIABLE" + FAIL = "FAIL_PERMANENT" + + +@dataclass(kw_only=True) class BeeperMessageStatusEventContent(SerializableAttrs): - network: str - success: bool relates_to: RelatesTo = field(json="m.relates_to") + network: str = "" + status: Optional[MessageStatus] = None reason: Optional[MessageStatusReason] = None error: Optional[str] = None message: Optional[str] = None + + success: Optional[bool] = None + still_working: Optional[bool] = None can_retry: Optional[bool] = None is_certain: Optional[bool] = None - still_working: Optional[bool] = None last_retry: Optional[EventID] = None + def fill_legacy_booleans(self) -> None: + self.success = self.status == MessageStatus.SUCCESS + if not self.success: + self.still_working = self.status == MessageStatus.PENDING + self.can_retry = self.status in (MessageStatus.PENDING, MessageStatus.RETRIABLE) + @dataclass class BeeperMessageStatusEvent(BaseRoomEvent, SerializableAttrs): From 16d6ec09e8bc858efbcaa722c30de2cae39b7487 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 12 Jul 2022 15:05:31 +0300 Subject: [PATCH 159/456] Bump version to 0.17.3 --- CHANGELOG.md | 4 ++++ mautrix/__init__.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ec24c691..ef0ad036 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## v0.17.3 (2022-07-12) + +* *(types)* Updated `BeeperMessageStatusEventContent` fields. + ## v0.17.2 (2022-07-06) * *(api)* Updated request logging to log full URL instead of only path. diff --git a/mautrix/__init__.py b/mautrix/__init__.py index c2d62cda..e4bb162c 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.17.2" +__version__ = "0.17.3" __author__ = "Tulir Asokan " __all__ = [ "api", From 3d291272016797263aa64504e9c50127e560e715 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 14 Jul 2022 17:26:32 +0300 Subject: [PATCH 160/456] Add wrapper for /context --- mautrix/client/api/events.py | 43 ++++++++++++++++++++++++++++++++++++ mautrix/types/__init__.py | 2 ++ mautrix/types/misc.py | 12 +++++++++- 3 files changed, 56 insertions(+), 1 deletion(-) diff --git a/mautrix/client/api/events.py b/mautrix/client/api/events.py index 7065146c..99e1c178 100644 --- a/mautrix/client/api/events.py +++ b/mautrix/client/api/events.py @@ -16,6 +16,7 @@ ContentURI, Event, EventContent, + EventContext, EventID, EventType, FilterID, @@ -125,6 +126,48 @@ async def get_event(self, room_id: RoomID, event_id: EventID) -> Event: except SerializerError as e: raise MatrixResponseError("Invalid event in response") from e + async def get_event_context( + self, + room_id: RoomID, + event_id: EventID, + limit: int | None = 10, + filter: RoomEventFilter | None = None, + ) -> EventContext: + """ + Get a number of events that happened just before and after the specified event. + This allows clients to get the context surrounding an event, as well as get the state at + an event and paginate in either direction. + + Args: + room_id: The room to get events from. + event_id: The event to get context around. + limit: The maximum number of events to return. The limit applies to the total number of + events before and after the requested event. A limit of 0 means no other events + are returned, while 2 means one event before and one after are returned. + filter: A JSON RoomEventFilter_ to filter returned events with. + + Returns: + The event itself, up to ``limit/2`` events before and after the event, the room state + at the event, and pagination tokens to scroll up and down. + + .. _RoomEventFilter: + https://spec.matrix.org/v1.1/client-server-api/#filtering + """ + query_params = {} + if limit is not None: + query_params["limit"] = str(limit) + if filter is not None: + query_params["filter"] = ( + filter.serialize() if isinstance(filter, Serializable) else filter + ) + resp = await self.api.request( + Method.GET, + Path.v3.rooms[room_id].context[event_id], + query_params=query_params, + metrics_method="get_event_context", + ) + return EventContext.deserialize(resp) + async def get_state_event( self, room_id: RoomID, diff --git a/mautrix/types/__init__.py b/mautrix/types/__init__.py index b8bf5d5d..fec96028 100644 --- a/mautrix/types/__init__.py +++ b/mautrix/types/__init__.py @@ -144,6 +144,7 @@ DeviceLists, DeviceOTKCount, DirectoryPaginationToken, + EventContext, PaginatedMessages, PaginationDirection, RoomAliasInfo, @@ -345,6 +346,7 @@ "DeviceLists", "DeviceOTKCount", "DirectoryPaginationToken", + "EventContext", "PaginatedMessages", "PaginationDirection", "RoomAliasInfo", diff --git a/mautrix/types/misc.py b/mautrix/types/misc.py index 87f5aa9a..bd81c6d8 100644 --- a/mautrix/types/misc.py +++ b/mautrix/types/misc.py @@ -9,7 +9,7 @@ from attr import dataclass import attr -from .event import Event +from .event import Event, StateEvent from .primitive import BatchID, ContentURI, EventID, RoomAlias, RoomID, SyncToken, UserID from .util import SerializableAttrs @@ -107,6 +107,16 @@ class RoomDirectoryResponse(SerializableAttrs): ) +@dataclass +class EventContext(SerializableAttrs): + end: SyncToken + start: SyncToken + event: Event + events_after: List[Event] + events_before: List[Event] + state: List[StateEvent] + + @dataclass class BatchSendResponse(SerializableAttrs): state_event_ids: List[EventID] From a209c3b56cd7f076e2ee3e680636d020d7e472c0 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 15 Jul 2022 16:25:37 +0300 Subject: [PATCH 161/456] Include info in bridge state deduplication --- mautrix/util/bridge_state.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mautrix/util/bridge_state.py b/mautrix/util/bridge_state.py index 22830320..b7c346f0 100644 --- a/mautrix/util/bridge_state.py +++ b/mautrix/util/bridge_state.py @@ -102,6 +102,7 @@ def should_deduplicate(self, prev_state: Optional["BridgeState"]) -> bool: not prev_state or prev_state.state_event != self.state_event or prev_state.error != self.error + or prev_state.info != self.info ): # If there's no previous state or the state was different, send this one. return False From d1e3fc5f4af3d9dd0354e0d782111e49720bacfa Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 17 Jul 2022 11:43:57 +0300 Subject: [PATCH 162/456] Allow raw dicts in StateStore.set_encryption_info --- mautrix/client/state_store/abstract.py | 4 ++-- mautrix/client/state_store/asyncpg/store.py | 9 ++++++--- mautrix/client/state_store/file.py | 4 ++-- mautrix/client/state_store/memory.py | 4 +++- mautrix/client/state_store/sqlalchemy/mx_room_state.py | 2 +- mautrix/client/state_store/sqlalchemy/sqlstatestore.py | 4 +++- 6 files changed, 17 insertions(+), 10 deletions(-) diff --git a/mautrix/client/state_store/abstract.py b/mautrix/client/state_store/abstract.py index dc3e7071..e241d8f1 100644 --- a/mautrix/client/state_store/abstract.py +++ b/mautrix/client/state_store/abstract.py @@ -5,7 +5,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import annotations -from typing import Awaitable +from typing import Any, Awaitable from abc import ABC, abstractmethod from mautrix.types import ( @@ -135,7 +135,7 @@ async def get_encryption_info(self, room_id: RoomID) -> RoomEncryptionStateEvent @abstractmethod async def set_encryption_info( - self, room_id: RoomID, content: RoomEncryptionStateEventContent + self, room_id: RoomID, content: RoomEncryptionStateEventContent | dict[str, any] ) -> None: pass diff --git a/mautrix/client/state_store/asyncpg/store.py b/mautrix/client/state_store/asyncpg/store.py index 265e607b..060561ea 100644 --- a/mautrix/client/state_store/asyncpg/store.py +++ b/mautrix/client/state_store/asyncpg/store.py @@ -5,7 +5,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import annotations -from typing import NamedTuple +from typing import Any, NamedTuple from mautrix.types import ( Member, @@ -14,6 +14,7 @@ PowerLevelStateEventContent, RoomEncryptionStateEventContent, RoomID, + Serializable, UserID, ) from mautrix.util.async_db import Database, Scheme @@ -242,10 +243,12 @@ async def get_encryption_info(self, room_id: RoomID) -> RoomEncryptionStateEvent return RoomEncryptionStateEventContent.parse_json(row["encryption"]) async def set_encryption_info( - self, room_id: RoomID, content: RoomEncryptionStateEventContent + self, room_id: RoomID, content: RoomEncryptionStateEventContent | dict[str, Any] ) -> None: q = ( "INSERT INTO mx_room_state (room_id, is_encrypted, encryption) VALUES ($1, true, $2) " "ON CONFLICT (room_id) DO UPDATE SET is_encrypted=true, encryption=$2" ) - await self.db.execute(q, room_id, content.json()) + await self.db.execute( + q, room_id, content.json() if isinstance(content, Serializable) else content + ) diff --git a/mautrix/client/state_store/file.py b/mautrix/client/state_store/file.py index 18644b08..d567c853 100644 --- a/mautrix/client/state_store/file.py +++ b/mautrix/client/state_store/file.py @@ -5,7 +5,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import annotations -from typing import IO +from typing import IO, Any from pathlib import Path from mautrix.types import ( @@ -55,7 +55,7 @@ async def set_members( self._time_limited_flush() async def set_encryption_info( - self, room_id: RoomID, content: RoomEncryptionStateEventContent + self, room_id: RoomID, content: RoomEncryptionStateEventContent | dict[str, Any] ) -> None: await super().set_encryption_info(room_id, content) self._time_limited_flush() diff --git a/mautrix/client/state_store/memory.py b/mautrix/client/state_store/memory.py index 5db7fa8d..5d2d5b48 100644 --- a/mautrix/client/state_store/memory.py +++ b/mautrix/client/state_store/memory.py @@ -187,6 +187,8 @@ async def get_encryption_info(self, room_id: RoomID) -> RoomEncryptionStateEvent return self.encryption.get(room_id) async def set_encryption_info( - self, room_id: RoomID, content: RoomEncryptionStateEventContent + self, room_id: RoomID, content: RoomEncryptionStateEventContent | dict[str, Any] ) -> None: + if not isinstance(content, RoomEncryptionStateEventContent): + content = RoomEncryptionStateEventContent.deserialize(content) self.encryption[room_id] = content diff --git a/mautrix/client/state_store/sqlalchemy/mx_room_state.py b/mautrix/client/state_store/sqlalchemy/mx_room_state.py index 6124cc91..9f97056f 100644 --- a/mautrix/client/state_store/sqlalchemy/mx_room_state.py +++ b/mautrix/client/state_store/sqlalchemy/mx_room_state.py @@ -32,7 +32,7 @@ def python_type(self) -> Type[Serializable]: def process_bind_param(self, value: Serializable, dialect) -> str | None: if value is not None: - return json.dumps(value.serialize()) + return json.dumps(value.serialize() if isinstance(value, Serializable) else value) return None def process_result_value(self, value: str, dialect) -> Serializable | None: diff --git a/mautrix/client/state_store/sqlalchemy/sqlstatestore.py b/mautrix/client/state_store/sqlalchemy/sqlstatestore.py index 201e6d76..1145dec7 100644 --- a/mautrix/client/state_store/sqlalchemy/sqlstatestore.py +++ b/mautrix/client/state_store/sqlalchemy/sqlstatestore.py @@ -5,6 +5,8 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import annotations +from typing import Any + from mautrix.types import ( Member, Membership, @@ -174,7 +176,7 @@ async def get_encryption_info(self, room_id: RoomID) -> RoomEncryptionStateEvent return room.encryption async def set_encryption_info( - self, room_id: RoomID, content: RoomEncryptionStateEventContent + self, room_id: RoomID, content: RoomEncryptionStateEventContent | dict[str, Any] ) -> None: if not content: raise ValueError("content is empty") From aacc831e8cc99b72e5a3f444ba562691ac9f5f85 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 18 Jul 2022 16:00:58 +0300 Subject: [PATCH 163/456] Actually allow raw dicts in PgStateStore --- mautrix/client/state_store/asyncpg/store.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/mautrix/client/state_store/asyncpg/store.py b/mautrix/client/state_store/asyncpg/store.py index 060561ea..d78c04d6 100644 --- a/mautrix/client/state_store/asyncpg/store.py +++ b/mautrix/client/state_store/asyncpg/store.py @@ -6,6 +6,7 @@ from __future__ import annotations from typing import Any, NamedTuple +import json from mautrix.types import ( Member, @@ -213,13 +214,13 @@ async def get_power_levels(self, room_id: RoomID) -> PowerLevelStateEventContent return PowerLevelStateEventContent.parse_json(power_levels_json) async def set_power_levels( - self, room_id: RoomID, content: PowerLevelStateEventContent + self, room_id: RoomID, content: PowerLevelStateEventContent | dict[str, Any] ) -> None: await self.db.execute( "INSERT INTO mx_room_state (room_id, power_levels) VALUES ($1, $2) " "ON CONFLICT (room_id) DO UPDATE SET power_levels=$2", room_id, - content.json(), + json.dumps(content.serialize() if isinstance(content, Serializable) else content), ) async def has_encryption_info_cached(self, room_id: RoomID) -> bool: @@ -250,5 +251,7 @@ async def set_encryption_info( "ON CONFLICT (room_id) DO UPDATE SET is_encrypted=true, encryption=$2" ) await self.db.execute( - q, room_id, content.json() if isinstance(content, Serializable) else content + q, + room_id, + json.dumps(content.serialize() if isinstance(content, Serializable) else content), ) From d71b1497ad348083480ee332cbc3d489f050b254 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 21 Jul 2022 16:20:31 +0300 Subject: [PATCH 164/456] Implement new error codes from MSC3848 Also includes `M_NOT_JOINED` which is not yet written in the MSC https://github.com/matrix-org/matrix-spec-proposals/pull/3848 --- mautrix/api.py | 4 +++- mautrix/appservice/api/intent.py | 4 ++++ mautrix/client/api/rooms.py | 12 +++++++++++- mautrix/errors/__init__.py | 6 ++++++ mautrix/errors/request.py | 31 ++++++++++++++++++++++++++++--- 5 files changed, 52 insertions(+), 5 deletions(-) diff --git a/mautrix/api.py b/mautrix/api.py index 2311ff88..092e67f8 100644 --- a/mautrix/api.py +++ b/mautrix/api.py @@ -245,11 +245,12 @@ async def _send( ) async with request as response: if response.status < 200 or response.status >= 300: - errcode = message = None + errcode = unstable_errcode = message = None try: response_data = await response.json() errcode = response_data["errcode"] message = response_data["error"] + unstable_errcode = response_data.get("org.matrix.unstable.errcode") except (JSONDecodeError, ContentTypeError, KeyError): pass raise make_request_error( @@ -257,6 +258,7 @@ async def _send( text=await response.text(), errcode=errcode, message=message, + unstable_errcode=unstable_errcode, ) return await response.json(), response diff --git a/mautrix/appservice/api/intent.py b/mautrix/appservice/api/intent.py index d2d06f36..dbc26ae3 100644 --- a/mautrix/appservice/api/intent.py +++ b/mautrix/appservice/api/intent.py @@ -12,6 +12,7 @@ from mautrix.client import ClientAPI, StoreUpdatingAPI from mautrix.errors import ( IntentError, + MAlreadyJoined, MatrixRequestError, MBadState, MForbidden, @@ -243,7 +244,10 @@ async def invite_user( room_id, user_id, reason=reason, extra_content=extra_content ) await self.state_store.invited(room_id, user_id) + except MAlreadyJoined as e: + await self.state_store.joined(room_id, user_id) except MatrixRequestError as e: + # TODO remove this once MSC3848 is released and minimum spec version is bumped if e.errcode == "M_FORBIDDEN" and "is already in the room" in e.message: await self.state_store.joined(room_id, user_id) else: diff --git a/mautrix/client/api/rooms.py b/mautrix/client/api/rooms.py index 726d1153..eb6d352d 100644 --- a/mautrix/client/api/rooms.py +++ b/mautrix/client/api/rooms.py @@ -11,7 +11,13 @@ from multidict import CIMultiDict from mautrix.api import Method, Path -from mautrix.errors import MatrixRequestError, MatrixResponseError, MNotFound, MRoomInUse +from mautrix.errors import ( + MatrixRequestError, + MatrixResponseError, + MNotFound, + MNotJoined, + MRoomInUse, +) from mautrix.types import ( JSON, DirectoryPaginationToken, @@ -478,7 +484,11 @@ async def leave_room( if reason: data["reason"] = reason await self.api.request(Method.POST, Path.v3.rooms[room_id].leave, content=data) + except MNotJoined: + if raise_not_in_room: + raise except MatrixRequestError as e: + # TODO remove this once MSC3848 is released and minimum spec version is bumped if "not in room" not in e.message or raise_not_in_room: raise diff --git a/mautrix/errors/__init__.py b/mautrix/errors/__init__.py index 1bb90885..04acfa9c 100644 --- a/mautrix/errors/__init__.py +++ b/mautrix/errors/__init__.py @@ -13,6 +13,7 @@ VerificationError, ) from .request import ( + MAlreadyJoined, MatrixBadContent, MatrixBadRequest, MatrixInvalidToken, @@ -27,6 +28,7 @@ MForbidden, MGuestAccessForbidden, MIncompatibleRoomVersion, + MInsufficientPower, MInvalidParam, MInvalidRoomState, MInvalidUsername, @@ -34,6 +36,7 @@ MMissingParam, MMissingToken, MNotFound, + MNotJoined, MNotJSON, MRoomInUse, MTooLarge, @@ -73,6 +76,7 @@ "SessionNotFound", "SessionShareError", "VerificationError", + "MAlreadyJoined", "MatrixBadContent", "MatrixBadRequest", "MatrixInvalidToken", @@ -87,6 +91,7 @@ "MForbidden", "MGuestAccessForbidden", "MIncompatibleRoomVersion", + "MInsufficientPower", "MInvalidParam", "MInvalidRoomState", "MInvalidUsername", @@ -94,6 +99,7 @@ "MMissingParam", "MMissingToken", "MNotFound", + "MNotJoined", "MNotJSON", "MRoomInUse", "MTooLarge", diff --git a/mautrix/errors/request.py b/mautrix/errors/request.py index 6bef7311..16c3f233 100644 --- a/mautrix/errors/request.py +++ b/mautrix/errors/request.py @@ -46,19 +46,23 @@ def __init__(self, http_status: int, message: str = "") -> None: MxSRE = Type[MatrixStandardRequestError] ec_map: Dict[str, MxSRE] = {} +uec_map: Dict[str, MxSRE] = {} -def standard_error(code: str) -> Callable[[MxSRE], MxSRE]: +def standard_error(code: str, unstable: Optional[str] = None) -> Callable[[MxSRE], MxSRE]: def decorator(cls: MxSRE) -> MxSRE: cls.errcode = code ec_map[code] = cls + if unstable: + cls.unstable_errcode = unstable + uec_map[unstable] = cls return cls return decorator def make_request_error( - http_status: int, text: str, errcode: str, message: str + http_status: int, text: str, errcode: str, message: str, unstable_errcode: Optional[str] = None ) -> MatrixRequestError: """ Determine the correct exception class for the error code and create an instance of that class @@ -70,6 +74,12 @@ def make_request_error( errcode: The errcode field in the response JSON. message: The error field in the response JSON. """ + if unstable_errcode: + try: + ec_class = uec_map[unstable_errcode] + return ec_class(http_status, message) + except KeyError: + pass try: ec_class = ec_map[errcode] return ec_class(http_status, message) @@ -77,7 +87,7 @@ def make_request_error( return MatrixUnknownRequestError(http_status, text, errcode, message) -# Standard error codes from https://matrix.org/docs/spec/client_server/r0.4.0.html#api-standards +# Standard error codes from https://spec.matrix.org/v1.3/client-server-api/#api-standards # Additionally some combining superclasses for some of the error codes @@ -86,6 +96,21 @@ class MForbidden(MatrixStandardRequestError): pass +@standard_error("M_ALREADY_JOINED", unstable="ORG.MATRIX.MSC3848.ALREADY_JOINED") +class MAlreadyJoined(MForbidden): + pass + + +@standard_error("M_NOT_JOINED", unstable="ORG.MATRIX.MSC3848.NOT_JOINED") +class MNotJoined(MForbidden): + pass + + +@standard_error("M_INSUFFICIENT_POWER", unstable="ORG.MATRIX.MSC3848.INSUFFICIENT_POWER") +class MInsufficientPower(MForbidden): + pass + + @standard_error("M_USER_DEACTIVATED") class MUserDeactivated(MForbidden): pass From 8fa1c7e2efa46daa30784e23bb9c90efc9ea801d Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 21 Jul 2022 16:30:54 +0300 Subject: [PATCH 165/456] Disable deserializing m.direct. Fixes #108 --- mautrix/types/event/account_data.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mautrix/types/event/account_data.py b/mautrix/types/event/account_data.py index ea7aacca..2bbe1ac9 100644 --- a/mautrix/types/event/account_data.py +++ b/mautrix/types/event/account_data.py @@ -28,7 +28,8 @@ class RoomTagAccountDataEventContent(SerializableAttrs): AccountDataEventContent = Union[RoomTagAccountDataEventContent, DirectAccountDataEventContent, Obj] account_data_event_content_map = { EventType.TAG: RoomTagAccountDataEventContent, - EventType.DIRECT: DirectAccountDataEventContent, + # m.direct doesn't really need deserializing + # EventType.DIRECT: DirectAccountDataEventContent, } From 5a441eea6a6572ce52cd53a274b23737380a47ce Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 21 Jul 2022 16:39:50 +0300 Subject: [PATCH 166/456] Reject reusing access tokens when enabling double puppeting --- mautrix/bridge/__init__.py | 1 + mautrix/bridge/commands/login_matrix.py | 22 ++-------- mautrix/bridge/custom_puppet.py | 55 ++++++++++++++++++++----- 3 files changed, 48 insertions(+), 30 deletions(-) diff --git a/mautrix/bridge/__init__.py b/mautrix/bridge/__init__.py index 4e31784a..e628eeff 100644 --- a/mautrix/bridge/__init__.py +++ b/mautrix/bridge/__init__.py @@ -5,6 +5,7 @@ AutologinError, CustomPuppetError, CustomPuppetMixin, + EncryptionKeysFound, HomeserverURLNotFound, InvalidAccessToken, OnlyLoginSelf, diff --git a/mautrix/bridge/commands/login_matrix.py b/mautrix/bridge/commands/login_matrix.py index 7ad91c72..2fd89d2d 100644 --- a/mautrix/bridge/commands/login_matrix.py +++ b/mautrix/bridge/commands/login_matrix.py @@ -6,13 +6,7 @@ from mautrix.client import Client from mautrix.types import EventID -from ..custom_puppet import ( - AutologinError, - HomeserverURLNotFound, - InvalidAccessToken, - OnlyLoginSelf, - OnlyLoginTrustedDomain, -) +from ..custom_puppet import AutologinError, CustomPuppetError, InvalidAccessToken from .handler import SECTION_AUTH, CommandEvent, command_handler @@ -36,20 +30,10 @@ async def login_matrix(evt: CommandEvent) -> None: try: await puppet.switch_mxid(evt.args[0], evt.sender.mxid) await evt.reply("Successfully enabled double puppeting.") - except OnlyLoginSelf: - await evt.reply("You may only enable double puppeting with your own Matrix account.") - except OnlyLoginTrustedDomain: - await evt.reply(f"This bridge does not allow double puppeting from {homeserver}.") - except HomeserverURLNotFound: - await evt.reply( - f"Unable to find the base URL for {homeserver}. Please ensure a client" - " .well-known file is set up, or ask the bridge administrator to add the" - " homeserver URL to the bridge config." - ) except AutologinError as e: await evt.reply(f"Failed to create an access token: {e}") - except InvalidAccessToken: - await evt.reply("Invalid access token.") + except CustomPuppetError as e: + await evt.reply(str(e)) @command_handler( diff --git a/mautrix/bridge/custom_puppet.py b/mautrix/bridge/custom_puppet.py index 77e1f1d8..402f6f4a 100644 --- a/mautrix/bridge/custom_puppet.py +++ b/mautrix/bridge/custom_puppet.py @@ -56,12 +56,24 @@ def __init__(self): class OnlyLoginSelf(CustomPuppetError): def __init__(self): - super().__init__("You may only replace your puppet with your own Matrix account.") + super().__init__("You may only enable double puppeting with your own Matrix account.") + + +class EncryptionKeysFound(CustomPuppetError): + def __init__(self): + super().__init__( + "The given access token is for a device that has encryption keys set up. " + "Please provide a fresh token, don't reuse one from another client." + ) class HomeserverURLNotFound(CustomPuppetError): def __init__(self, domain: str): - super().__init__(f"Could not discover a valid homeserver URL for {domain}") + super().__init__( + f"Could not discover a valid homeserver URL for {domain}." + " Please ensure a client .well-known file is set up, or ask the bridge administrator " + "to add the homeserver URL to the bridge config." + ) class OnlyLoginTrustedDomain(CustomPuppetError): @@ -235,7 +247,7 @@ async def switch_mxid( self.base_url = base_url self.intent = self._fresh_intent() - await self.start(start_sync_task=start_sync_task) + await self.start(start_sync_task=start_sync_task, check_e2ee_keys=True) try: del self.by_custom_mxid[prev_mxid] @@ -255,7 +267,21 @@ async def try_start(self, retry_auto_login: bool = True) -> None: except Exception: self.log.exception("Failed to initialize custom mxid") - async def start(self, retry_auto_login: bool = False, start_sync_task: bool = True) -> None: + async def _invalidate_double_puppet(self) -> None: + if self.custom_mxid and self.by_custom_mxid.get(self.custom_mxid) == self: + del self.by_custom_mxid[self.custom_mxid] + self.custom_mxid = None + self.access_token = None + self.next_batch = None + await self.save() + self.intent = self._fresh_intent() + + async def start( + self, + retry_auto_login: bool = False, + start_sync_task: bool = True, + check_e2ee_keys: bool = False, + ) -> None: """Initialize the custom account this puppet uses. Should be called at startup to start the /sync task. Is called by :meth:`switch_mxid` automatically.""" if not self.is_real_user: @@ -271,17 +297,24 @@ async def start(self, retry_auto_login: bool = False, start_sync_task: bool = Tr self.log.warning(f"Got {e.errcode} while trying to initialize custom mxid") whoami = None if not whoami or whoami.user_id != self.custom_mxid: - if self.custom_mxid and self.by_custom_mxid.get(self.custom_mxid) == self: - del self.by_custom_mxid[self.custom_mxid] prev_custom_mxid = self.custom_mxid - self.custom_mxid = None - self.access_token = None - self.next_batch = None - await self.save() - self.intent = self._fresh_intent() + await self._invalidate_double_puppet() if whoami and whoami.user_id != prev_custom_mxid: raise OnlyLoginSelf() raise InvalidAccessToken() + if check_e2ee_keys: + try: + devices = await self.intent.query_keys({whoami.user_id: [whoami.device_id]}) + device_keys = devices.device_keys.get(whoami.user_id, {}).get(whoami.device_id) + except Exception: + self.log.warning( + "Failed to query keys to check if double puppeting token was reused", + exc_info=True, + ) + else: + if device_keys and len(device_keys.keys) > 0: + await self._invalidate_double_puppet() + raise EncryptionKeysFound() if self.sync_with_custom_puppets and start_sync_task: if self._sync_task: self._sync_task.cancel() From 0398b3f4756b3a17a4ac086383fa9a69b289bbc4 Mon Sep 17 00:00:00 2001 From: Scott Weber Date: Wed, 27 Jul 2022 11:11:50 -0400 Subject: [PATCH 167/456] PENDNIG -> PENDING (#110) Co-authored-by: Scott Weber --- mautrix/types/event/beeper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/types/event/beeper.py b/mautrix/types/event/beeper.py index 5ceb2373..dc588800 100644 --- a/mautrix/types/event/beeper.py +++ b/mautrix/types/event/beeper.py @@ -34,7 +34,7 @@ def checkpoint_status(self): class MessageStatus(SerializableEnum): SUCCESS = "SUCCESS" - PENDING = "PENDNIG" + PENDING = "PENDING" RETRIABLE = "FAIL_RETRIABLE" FAIL = "FAIL_PERMANENT" From 92b10b8346cc346a63fb0825f14a9e335a939480 Mon Sep 17 00:00:00 2001 From: Malte E <97891689+maltee1@users.noreply.github.com> Date: Wed, 27 Jul 2022 20:35:18 +0200 Subject: [PATCH 168/456] Fix error when fetching e2ee keys for user with no cross-signing keys (#109) --- mautrix/crypto/device_lists.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/crypto/device_lists.py b/mautrix/crypto/device_lists.py index 42c17074..c0cea43f 100644 --- a/mautrix/crypto/device_lists.py +++ b/mautrix/crypto/device_lists.py @@ -56,7 +56,7 @@ async def _fetch_keys( ) changed = False ssks = resp.self_signing_keys.get(user_id) - ssk = ssks.first_ed25519_key + ssk = ssks.first_ed25519_key if ssks else None for device_id, device_keys in devices.items(): try: existing = existing_devices[device_id] From 325a2d138d97828bf8634504895d1b652e15feef Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 28 Jul 2022 18:00:34 +0300 Subject: [PATCH 169/456] Update MSC3848 unstable errcode field name --- mautrix/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/api.py b/mautrix/api.py index 092e67f8..a0561d08 100644 --- a/mautrix/api.py +++ b/mautrix/api.py @@ -250,7 +250,7 @@ async def _send( response_data = await response.json() errcode = response_data["errcode"] message = response_data["error"] - unstable_errcode = response_data.get("org.matrix.unstable.errcode") + unstable_errcode = response_data.get("org.matrix.msc3848.unstable.errcode") except (JSONDecodeError, ContentTypeError, KeyError): pass raise make_request_error( From 20ec165b8c5a4b6619c8ab2b6faf96a136e77c43 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 28 Jul 2022 18:09:42 +0300 Subject: [PATCH 170/456] Update changelog. Closes #111 --- CHANGELOG.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ef0ad036..88c44cb2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,20 @@ +## v0.17.4 (unreleased) + +* *(bridge)* Started rejecting reusing access tokens when enabling double + puppeting. Reuse is detected by presence of encryption keys on the device. +* *(client.api)* Added wrapper method for the `/context` API. +* *(api, errors)* Implemented new error codes from [MSC3848]. +* *(types)* Disabled deserializing `m.direct` content (it didn't work and it + wasn't really necessary). +* *(client.state_store)* Updated `set_encryption_info` to allow raw dicts. + This fixes the bug where sending a `m.room.encryption` event with a raw dict + as the content would throw an error from the state store. +* *(crypto)* Fixed error when fetching keys for user with no cross-signing keys + (thanks to [@maltee1] in [#109]). + +[MSC3848]: https://github.com/matrix-org/matrix-spec-proposals/pull/3848 +[#109]: https://github.com/mautrix/python/pull/109 + ## v0.17.3 (2022-07-12) * *(types)* Updated `BeeperMessageStatusEventContent` fields. From 2a59fea77faede9d7628c9a346f6356a10f24fb0 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 28 Jul 2022 18:13:56 +0300 Subject: [PATCH 171/456] Bump version to 0.17.4 --- CHANGELOG.md | 2 +- mautrix/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 88c44cb2..19c0d7df 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -## v0.17.4 (unreleased) +## v0.17.4 (2022-07-28) * *(bridge)* Started rejecting reusing access tokens when enabling double puppeting. Reuse is detected by presence of encryption keys on the device. diff --git a/mautrix/__init__.py b/mautrix/__init__.py index e4bb162c..0bead8f1 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.17.3" +__version__ = "0.17.4" __author__ = "Tulir Asokan " __all__ = [ "api", From 18a4bcef7828037131ff71154f88b2a7ab1cfbad Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 1 Aug 2022 19:57:02 +0300 Subject: [PATCH 172/456] Log checkpoint step when sending fails --- mautrix/util/message_send_checkpoint.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/mautrix/util/message_send_checkpoint.py b/mautrix/util/message_send_checkpoint.py index 8d9a3333..61eb691d 100644 --- a/mautrix/util/message_send_checkpoint.py +++ b/mautrix/util/message_send_checkpoint.py @@ -66,16 +66,18 @@ async def send(self, endpoint: str, as_token: str, log: logging.Logger) -> None: text = await resp.text() text = text.replace("\n", "\\n") log.warning( - f"Unexpected status code {resp.status} sending checkpoints " - f"for {self.event_id}: {text}" + f"Unexpected status code {resp.status} sending checkpoint " + f"for {self.event_id} ({self.step}/{self.status}): {text}" ) else: log.info( - f"Successfully sent checkpoint for {self.event_id} (step: {self.step})" + f"Successfully sent checkpoint for {self.event_id} " + f"({self.step}/{self.status})" ) except Exception as e: log.warning( - f"Failed to send checkpoint for {self.event_id}: " f"{type(e).__name__}: {e}" + f"Failed to send checkpoint for {self.event_id} ({self.step}/{self.status}): " + f"{type(e).__name__}: {e}" ) From 2820c01e51c399a0a1510db18cf225b74f59fd4f Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 10 Aug 2022 15:12:54 +0300 Subject: [PATCH 173/456] Don't raise IntentError from ensure_registered I don't think anyone catches that, but some things do want to catch MExclusive --- mautrix/appservice/api/intent.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/mautrix/appservice/api/intent.py b/mautrix/appservice/api/intent.py index dbc26ae3..702e5508 100644 --- a/mautrix/appservice/api/intent.py +++ b/mautrix/appservice/api/intent.py @@ -622,8 +622,6 @@ async def ensure_registered(self) -> None: await self._register() except MUserInUse: pass - except MatrixRequestError as e: - raise IntentError(f"Failed to register {self.mxid}", e) await self.state_store.registered(self.mxid) async def _ensure_has_power_level_for( From 1c959ac16a037e9f7f103acba691df1c53998cce Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 14 Aug 2022 12:57:03 +0300 Subject: [PATCH 174/456] Add m.read.private ReceiptType --- mautrix/types/event/ephemeral.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mautrix/types/event/ephemeral.py b/mautrix/types/event/ephemeral.py index 930a7de3..7252f5c5 100644 --- a/mautrix/types/event/ephemeral.py +++ b/mautrix/types/event/ephemeral.py @@ -8,7 +8,7 @@ from attr import dataclass from ..primitive import JSON, EventID, RoomID, UserID -from ..util import SerializableAttrs, SerializableEnum, deserializer +from ..util import ExtensibleEnum, SerializableAttrs, SerializableEnum, deserializer from .base import BaseEvent, GenericEvent from .type import EventType @@ -49,8 +49,9 @@ class SingleReceiptEventContent(SerializableAttrs): ts: int -class ReceiptType(SerializableEnum): +class ReceiptType(ExtensibleEnum): READ = "m.read" + READ_PRIVATE = "m.read.private" ReceiptEventContent = Dict[EventID, Dict[ReceiptType, Dict[UserID, SingleReceiptEventContent]]] From 0b0568e581c9c3f699181b950dc7675db4c23544 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 14 Aug 2022 12:57:46 +0300 Subject: [PATCH 175/456] Catch all errors when redacting commands --- mautrix/bridge/commands/handler.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mautrix/bridge/commands/handler.py b/mautrix/bridge/commands/handler.py index fe461937..4e6caa3d 100644 --- a/mautrix/bridge/commands/handler.py +++ b/mautrix/bridge/commands/handler.py @@ -158,6 +158,8 @@ async def redact(self, reason: str | None = None) -> None: await self.main_intent.redact(self.room_id, self.event_id, reason=reason) except MForbidden as e: self.log.warning(f"Failed to redact command {self.command}: {e}") + except Exception: + self.log.warning(f"Failed to redact command {self.command}", exc_info=True) def reply( self, message: str, allow_html: bool = False, render_markdown: bool = True From c83a8431b0e1b2d92e9a1df52564e9ff441a9daf Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 14 Aug 2022 12:57:57 +0300 Subject: [PATCH 176/456] Don't require failures in QueryKeysResponse --- mautrix/types/crypto.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/types/crypto.py b/mautrix/types/crypto.py index 08c9dd22..9a7bcfe5 100644 --- a/mautrix/types/crypto.py +++ b/mautrix/types/crypto.py @@ -79,11 +79,11 @@ def first_key_with_algorithm(self, alg: EncryptionKeyAlgorithm) -> Optional[Sign @dataclass class QueryKeysResponse(SerializableAttrs): - failures: Dict[str, Any] device_keys: Dict[UserID, Dict[DeviceID, DeviceKeys]] master_keys: Dict[UserID, CrossSigningKeys] self_signing_keys: Dict[UserID, CrossSigningKeys] user_signing_keys: Dict[UserID, CrossSigningKeys] + failures: Dict[str, Any] = field(factory=lambda: {}) @dataclass From 949594bdb7240b37a40031d266a679f36109c59f Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 14 Aug 2022 12:58:55 +0300 Subject: [PATCH 177/456] Don't throw error if no m.read receipt is returned --- mautrix/bridge/matrix.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/bridge/matrix.py b/mautrix/bridge/matrix.py index e4729424..e1487cc2 100644 --- a/mautrix/bridge/matrix.py +++ b/mautrix/bridge/matrix.py @@ -716,7 +716,7 @@ async def _is_direct_chat(self, room_id: RoomID) -> tuple[bool, bool]: async def handle_receipt(self, evt: ReceiptEvent) -> None: for event_id, receipts in evt.content.items(): - for user_id, data in receipts[ReceiptType.READ].items(): + for user_id, data in receipts.get(ReceiptType.READ, {}).items(): user = await self.bridge.get_user(user_id, create=False) if not user or not await user.is_logged_in(): continue From e3dfb3cec89cf5d5b65694cfc493f34502e0d682 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 14 Aug 2022 14:22:12 +0300 Subject: [PATCH 178/456] Don't raise IntentError from invite_user --- mautrix/appservice/api/intent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/appservice/api/intent.py b/mautrix/appservice/api/intent.py index 702e5508..2d91f8db 100644 --- a/mautrix/appservice/api/intent.py +++ b/mautrix/appservice/api/intent.py @@ -251,7 +251,7 @@ async def invite_user( if e.errcode == "M_FORBIDDEN" and "is already in the room" in e.message: await self.state_store.joined(room_id, user_id) else: - raise IntentError(f"Failed to invite {user_id} to {room_id}", e) + raise async def kick_user( self, From 3ac5ee2107fc2b816e15e407392f4883fc382cd3 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 15 Aug 2022 11:32:54 +0300 Subject: [PATCH 179/456] Bump version to 0.17.5 --- CHANGELOG.md | 6 ++++++ mautrix/__init__.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 19c0d7df..ce3622c9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## v0.17.5 (2022-08-15) + +* *(types)* Added `m.read.private` to receipt types. +* *(appservice)* Stopped `ensure_registered` and `invite_user` raising + `IntentError`s (now they raise the original Matrix error instead). + ## v0.17.4 (2022-07-28) * *(bridge)* Started rejecting reusing access tokens when enabling double diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 0bead8f1..6b6d4eb0 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.17.4" +__version__ = "0.17.5" __author__ = "Tulir Asokan " __all__ = [ "api", From b2fe6decf109eb9f688fb5772d7957f970f60e3e Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 15 Aug 2022 14:09:40 +0300 Subject: [PATCH 180/456] Update MSC3202 support --- mautrix/appservice/as_handler.py | 9 ++++++--- mautrix/types/misc.py | 4 ++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/mautrix/appservice/as_handler.py b/mautrix/appservice/as_handler.py index 42a29b3b..fa8c196d 100644 --- a/mautrix/appservice/as_handler.py +++ b/mautrix/appservice/as_handler.py @@ -169,8 +169,11 @@ async def _http_handle_transaction(self, request: web.Request) -> web.Response: self._get_with_fallback(data, "device_lists", "org.matrix.msc3202") ) otk_counts = { - user_id: DeviceOTKCount.deserialize(count) - for user_id, count in self._get_with_fallback( + user_id: { + device_id: DeviceOTKCount.deserialize(count) + for device_id, count in devices.items() + } + for user_id, devices in self._get_with_fallback( data, "device_one_time_keys_count", "org.matrix.msc3202", default={} ).items() } @@ -209,7 +212,7 @@ async def handle_transaction( events: list[JSON], extra_data: JSON, ephemeral: list[JSON] | None = None, - device_otk_count: dict[UserID, DeviceOTKCount] | None = None, + device_otk_count: dict[UserID, dict[DeviceID, DeviceOTKCount]] | None = None, device_lists: DeviceLists | None = None, ) -> JSON: for raw_edu in ephemeral or []: diff --git a/mautrix/types/misc.py b/mautrix/types/misc.py index bd81c6d8..6be085c5 100644 --- a/mautrix/types/misc.py +++ b/mautrix/types/misc.py @@ -22,8 +22,8 @@ class DeviceLists(SerializableAttrs): @dataclass class DeviceOTKCount(SerializableAttrs): - curve25519: int - signed_curve25519: int + signed_curve25519: int = 0 + curve25519: int = 0 class RoomCreatePreset(Enum): From 1f5db5494172022477076995db5b0061dc9ea14a Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 15 Aug 2022 14:25:56 +0300 Subject: [PATCH 181/456] Don't fail sync handling if event parsing fails --- mautrix/client/syncer.py | 40 +++++++++++++++++++++++++++++----------- 1 file changed, 29 insertions(+), 11 deletions(-) diff --git a/mautrix/client/syncer.py b/mautrix/client/syncer.py index 3ade8492..f3392a4a 100644 --- a/mautrix/client/syncer.py +++ b/mautrix/client/syncer.py @@ -5,7 +5,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import annotations -from typing import Any, Awaitable, Callable, Type +from typing import Any, Awaitable, Callable, Type, TypeVar from abc import ABC, abstractmethod from contextlib import suppress from enum import Enum, Flag, auto @@ -24,8 +24,10 @@ EventType, Filter, FilterID, + GenericEvent, MessageEvent, PresenceState, + SerializerError, StateEvent, StrippedStateEvent, SyncToken, @@ -39,6 +41,8 @@ EventHandler = Callable[[Event], Awaitable[None]] +T = TypeVar("T", bound=Event) + class SyncStream(Flag): INTERNAL = auto() @@ -201,7 +205,7 @@ def remove_event_handler( if len(handler_list) == 0 and event_type != EventType.ALL: del self.event_handlers[event_type] - def dispatch_event(self, event: Event, source: SyncStream) -> list[asyncio.Task]: + def dispatch_event(self, event: Event | None, source: SyncStream) -> list[asyncio.Task]: """ Send the given event to all applicable event handlers. @@ -209,6 +213,8 @@ def dispatch_event(self, event: Event, source: SyncStream) -> list[asyncio.Task] event: The event to send. source: The sync stream the event was received in. """ + if event is None: + return [] if isinstance(event.content, BaseMessageEventContentFuncs): event.content.trim_reply_fallback() if getattr(event, "state_key", None) is not None: @@ -264,6 +270,17 @@ def dispatch_internal_event( event_type, custom_type or kwargs, include_global_handlers=False ) + def _try_deserialize(self, type: Type[T], data: JSON) -> T | GenericEvent: + try: + return type.deserialize(data) + except SerializerError as e: + self.log.trace("Deserialization error traceback", exc_info=True) + self.log.warning(f"Failed to deserialize {data} into {type.__name__}: {e}") + try: + return GenericEvent.deserialize(data) + except SerializerError: + return None + def handle_sync(self, data: JSON) -> list[asyncio.Task]: """ Handle a /sync object. @@ -286,21 +303,22 @@ def handle_sync(self, data: JSON) -> list[asyncio.Task]: tasks += self.dispatch_internal_event( InternalEventType.DEVICE_LISTS, custom_type=DeviceLists( - changed=device_lists.get("changed", []), left=device_lists.get("left", []) + changed=device_lists.get("changed", []), + left=device_lists.get("left", []), ), ) for raw_event in data.get("account_data", {}).get("events", []): tasks += self.dispatch_event( - AccountDataEvent.deserialize(raw_event), source=SyncStream.ACCOUNT_DATA + self._try_deserialize(AccountDataEvent, raw_event), source=SyncStream.ACCOUNT_DATA ) for raw_event in data.get("ephemeral", {}).get("events", []): tasks += self.dispatch_event( - EphemeralEvent.deserialize(raw_event), source=SyncStream.EPHEMERAL + self._try_deserialize(EphemeralEvent, raw_event), source=SyncStream.EPHEMERAL ) for raw_event in data.get("to_device", {}).get("events", []): tasks += self.dispatch_event( - ToDeviceEvent.deserialize(raw_event), source=SyncStream.TO_DEVICE + self._try_deserialize(ToDeviceEvent, raw_event), source=SyncStream.TO_DEVICE ) rooms = data.get("rooms", {}) @@ -308,14 +326,14 @@ def handle_sync(self, data: JSON) -> list[asyncio.Task]: for raw_event in room_data.get("state", {}).get("events", []): raw_event["room_id"] = room_id tasks += self.dispatch_event( - StateEvent.deserialize(raw_event), + self._try_deserialize(StateEvent, raw_event), source=SyncStream.JOINED_ROOM | SyncStream.STATE, ) for raw_event in room_data.get("timeline", {}).get("events", []): raw_event["room_id"] = room_id tasks += self.dispatch_event( - Event.deserialize(raw_event), + self._try_deserialize(Event, raw_event), source=SyncStream.JOINED_ROOM | SyncStream.TIMELINE, ) for room_id, room_data in rooms.get("invite", {}).items(): @@ -332,9 +350,9 @@ def handle_sync(self, data: JSON) -> list[asyncio.Task]: raw_invite.setdefault("event_id", None) raw_invite.setdefault("origin_server_ts", int(time.time() * 1000)) - invite = StateEvent.deserialize(raw_invite) + invite = self._try_deserialize(StateEvent, raw_invite) invite.unsigned.invite_room_state = [ - StrippedStateEvent.deserialize(raw_event) + self._try_deserialize(StrippedStateEvent, raw_event) for raw_event in events if raw_event != raw_invite ] @@ -344,7 +362,7 @@ def handle_sync(self, data: JSON) -> list[asyncio.Task]: if "state_key" in raw_event: raw_event["room_id"] = room_id tasks += self.dispatch_event( - StateEvent.deserialize(raw_event), + self._try_deserialize(StateEvent, raw_event), source=SyncStream.LEFT_ROOM | SyncStream.TIMELINE, ) return tasks From b5998769c0435549774ccd95e2ed037d475dd3f1 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 15 Aug 2022 15:06:51 +0300 Subject: [PATCH 182/456] Add M_UNKNOWN_ENDPOINT error code from MSC3743 --- mautrix/errors/__init__.py | 2 ++ mautrix/errors/request.py | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/mautrix/errors/__init__.py b/mautrix/errors/__init__.py index 04acfa9c..fdec6e3a 100644 --- a/mautrix/errors/__init__.py +++ b/mautrix/errors/__init__.py @@ -42,6 +42,7 @@ MTooLarge, MUnauthorized, MUnknown, + MUnknownEndpoint, MUnknownToken, MUnrecognized, MUnsupportedRoomVersion, @@ -105,6 +106,7 @@ "MTooLarge", "MUnauthorized", "MUnknown", + "MUnknownEndpoint", "MUnknownToken", "MUnrecognized", "MUnsupportedRoomVersion", diff --git a/mautrix/errors/request.py b/mautrix/errors/request.py index 16c3f233..73824c3f 100644 --- a/mautrix/errors/request.py +++ b/mautrix/errors/request.py @@ -111,6 +111,11 @@ class MInsufficientPower(MForbidden): pass +@standard_error("M_UNKNOWN_ENDPOINT") +class MUnknownEndpoint(MatrixStandardRequestError): + pass + + @standard_error("M_USER_DEACTIVATED") class MUserDeactivated(MForbidden): pass From 6823471a89a0940611235a1f6f481710c5788d45 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 16 Aug 2022 17:05:09 +0300 Subject: [PATCH 183/456] Add hidden option to use appservice login for double puppeting --- mautrix/bridge/custom_puppet.py | 35 +++++++++++++++++---------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/mautrix/bridge/custom_puppet.py b/mautrix/bridge/custom_puppet.py index 402f6f4a..3d5c7f9e 100644 --- a/mautrix/bridge/custom_puppet.py +++ b/mautrix/bridge/custom_puppet.py @@ -181,24 +181,25 @@ async def _login_with_shared_secret(cls, mxid: UserID) -> str: base_url = cls.az.intent.api.base_url else: raise AutologinError(f"No homeserver URL configured for {server}") - password = hmac.new(secret, mxid.encode("utf-8"), hashlib.sha512).hexdigest() url = base_url / str(Path.v3.login) - resp = await cls.az.http_session.post( - url, - data=json.dumps( - { - "type": str(LoginType.PASSWORD), - "initial_device_display_name": cls.login_device_name, - "device_id": cls.login_device_name, - "identifier": { - "type": "m.id.user", - "user": mxid, - }, - "password": password, - } - ), - headers={"Content-Type": "application/json"}, - ) + headers = {"Content-Type": "application/json"} + login_req = { + "initial_device_display_name": cls.login_device_name, + "device_id": cls.login_device_name, + "identifier": { + "type": "m.id.user", + "user": mxid, + }, + } + if secret == b"appservice": + login_req["type"] = str(LoginType.APPSERVICE) + headers["Authorization"] = f"Bearer {cls.az.as_token}" + else: + login_req["type"] = str(LoginType.PASSWORD) + login_req["password"] = hmac.new( + secret, mxid.encode("utf-8"), hashlib.sha512 + ).hexdigest() + resp = await cls.az.http_session.post(url, data=json.dumps(login_req), headers=headers) data = await resp.json() try: return data["access_token"] From deb1ab22bf068cc02ffecb4d1838c8bf12b9812e Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 17 Aug 2022 15:17:31 +0300 Subject: [PATCH 184/456] Bump version to 0.17.6 --- CHANGELOG.md | 9 +++++++++ mautrix/__init__.py | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ce3622c9..fffc3c28 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,12 @@ +## v0.17.6 (2022-08-17) + +* *(bridge)* Added hidden option to use appservice login for double puppeting. +* *(client)* Fixed sync handling throwing an error if event parsing failed. +* *(errors)* Added `M_UNKNOWN_ENDPOINT` error code from [MSC3743] +* *(appservice)* Updated [MSC3202] support to handle one time keys correctly. + +[MSC3743]: https://github.com/matrix-org/matrix-spec-proposals/pull/3743 + ## v0.17.5 (2022-08-15) * *(types)* Added `m.read.private` to receipt types. diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 6b6d4eb0..75b1a19c 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.17.5" +__version__ = "0.17.6" __author__ = "Tulir Asokan " __all__ = [ "api", From dfaec5b80afd5ff2387e6f0d3563db4310237897 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 19 Aug 2022 15:04:17 +0300 Subject: [PATCH 185/456] Reset encryption if keys are missing from server --- mautrix/bridge/e2ee.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/mautrix/bridge/e2ee.py b/mautrix/bridge/e2ee.py index fc6db00a..facaf456 100644 --- a/mautrix/bridge/e2ee.py +++ b/mautrix/bridge/e2ee.py @@ -260,9 +260,31 @@ async def start(self) -> None: if not device_id: await self.crypto_store.put_device_id(self.client.device_id) self.log.debug(f"Logged in with new device ID {self.client.device_id}") + elif self.crypto.account.shared: + await self._verify_keys_are_on_server() _ = self.client.start(self._filter) self.log.info("End-to-bridge encryption support is enabled") + async def _verify_keys_are_on_server(self) -> None: + self.log.debug("Making sure keys are still on server") + try: + resp = await self.client.query_keys([self.client.mxid]) + except Exception: + self.log.critical( + "Failed to query own keys to make sure device still exists", exc_info=True + ) + sys.exit(33) + try: + own_keys = resp.device_keys[self.client.mxid][self.client.device_id] + if len(own_keys.keys) > 0: + return + except KeyError: + pass + self.log.critical("Existing device doesn't have keys on server, resetting crypto") + await self.crypto.crypto_store.delete() + await self.client.logout_all() + sys.exit(34) + async def stop(self) -> None: self.client.stop() await self.crypto_store.close() From 57dd61f648616048a517355797d15b8a358b6bf9 Mon Sep 17 00:00:00 2001 From: Toni Spets Date: Mon, 22 Aug 2022 10:10:43 +0300 Subject: [PATCH 186/456] Add init_commands list for aiosqlite database_opts --- mautrix/util/async_db/aiosqlite.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/mautrix/util/async_db/aiosqlite.py b/mautrix/util/async_db/aiosqlite.py index fa902b7c..8d0b51fd 100644 --- a/mautrix/util/async_db/aiosqlite.py +++ b/mautrix/util/async_db/aiosqlite.py @@ -84,6 +84,7 @@ class SQLiteDatabase(Database): _pool: asyncio.Queue[TxnConnection] _stopped: bool _conns: int + _init_commands: List[str] def __init__( self, @@ -109,11 +110,18 @@ def __init__( self._db_args.pop("max_size", None) self._stopped = False self._conns = 0 + self._init_commands = self._db_args.pop("init_commands", []) async def start(self) -> None: self.log.debug(f"Connecting to {self.url}") for _ in range(self._pool.maxsize): conn = await TxnConnection(self._path, **self._db_args) + if self._init_commands: + cur = await conn.cursor() + for command in self._init_commands: + self.log.debug("Executing command: %s", command) + await cur.execute(command) + await conn.commit() conn.row_factory = sqlite3.Row self._pool.put_nowait(conn) self._conns += 1 From 05cfcf84feef161568ca0b8d22968a83fd35e5d6 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 21 Aug 2022 00:20:43 +0300 Subject: [PATCH 187/456] Change word --- mautrix/util/async_db/database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/util/async_db/database.py b/mautrix/util/async_db/database.py index 5ed0c7f1..0f23b02d 100644 --- a/mautrix/util/async_db/database.py +++ b/mautrix/util/async_db/database.py @@ -108,7 +108,7 @@ async def _check_foreign_tables(self) -> None: if await self.table_exists("state_groups_state"): raise ForeignTablesFound("found state_groups_state likely belonging to Synapse") elif await self.table_exists("roomserver_rooms"): - raise ForeignTablesFound("found roomserver_rooms possibly belonging to Dendrite") + raise ForeignTablesFound("found roomserver_rooms likely belonging to Dendrite") async def _check_owner(self) -> None: await self.execute( From fef6a6bdcc8f3f7c86e9a9de51762e19a7365989 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 22 Aug 2022 12:39:52 +0300 Subject: [PATCH 188/456] Remove init_commands from db_args before passing to asyncpg --- mautrix/util/async_db/asyncpg.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mautrix/util/async_db/asyncpg.py b/mautrix/util/async_db/asyncpg.py index e7f2102f..cf3f684e 100644 --- a/mautrix/util/async_db/asyncpg.py +++ b/mautrix/util/async_db/asyncpg.py @@ -41,6 +41,7 @@ def __init__( # Send postgres scheme to asyncpg url = url.with_scheme("postgres") self._exit_on_ice = (db_args or {}).pop("meow_exit_on_ice", True) + db_args.pop("init_commands", None) super().__init__( url, db_args=db_args, From 2ae6f78ca17bf600a3462d94e711e25e5de7ef77 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 22 Aug 2022 12:46:53 +0300 Subject: [PATCH 189/456] Fix removing parameter --- mautrix/util/async_db/asyncpg.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mautrix/util/async_db/asyncpg.py b/mautrix/util/async_db/asyncpg.py index cf3f684e..03a33646 100644 --- a/mautrix/util/async_db/asyncpg.py +++ b/mautrix/util/async_db/asyncpg.py @@ -40,8 +40,10 @@ def __init__( self.scheme = Scheme.COCKROACH # Send postgres scheme to asyncpg url = url.with_scheme("postgres") - self._exit_on_ice = (db_args or {}).pop("meow_exit_on_ice", True) - db_args.pop("init_commands", None) + self._exit_on_ice = True + if db_args: + self._exit_on_ice = db_args.pop("meow_exit_on_ice", True) + db_args.pop("init_commands", None) super().__init__( url, db_args=db_args, From 670b99ba4c2804eb6084a8c243037f4500a3ab41 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 22 Aug 2022 12:47:21 +0300 Subject: [PATCH 190/456] Bump version to 0.17.7 --- CHANGELOG.md | 7 +++++++ mautrix/__init__.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fffc3c28..cd7eb400 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## v0.17.7 (2022-08-22) + +* *(util.async_db)* Added `init_commands` to run commands on each SQLite + connection (e.g. to enable `PRAGMA`s). No-op on Postgres. +* *(bridge)* Added check to make sure e2ee keys are intact on server. + If they aren't, the crypto database will be wiped and the bridge will stop. + ## v0.17.6 (2022-08-17) * *(bridge)* Added hidden option to use appservice login for double puppeting. diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 75b1a19c..2cce4b5b 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.17.6" +__version__ = "0.17.7" __author__ = "Tulir Asokan " __all__ = [ "api", From 04be73abe4eb27a93c9990560584a1e8b59f6749 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 22 Aug 2022 15:42:41 +0300 Subject: [PATCH 191/456] Don't require failures in claim keys response either --- mautrix/types/crypto.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/types/crypto.py b/mautrix/types/crypto.py index 9a7bcfe5..56fd3fb8 100644 --- a/mautrix/types/crypto.py +++ b/mautrix/types/crypto.py @@ -88,8 +88,8 @@ class QueryKeysResponse(SerializableAttrs): @dataclass class ClaimKeysResponse(SerializableAttrs): - failures: Dict[str, Any] one_time_keys: Dict[UserID, Dict[DeviceID, Dict[KeyID, Any]]] + failures: Dict[str, Any] = field(factory=lambda: {}) class TrustState(IntEnum): From 9c5b890f05dfdab440fd98a1239bedb02ce1e1d0 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 22 Aug 2022 15:54:25 +0300 Subject: [PATCH 192/456] Fix parsing bridge e2ee key sharing config --- CHANGELOG.md | 5 +++++ mautrix/bridge/e2ee.py | 10 ++++------ mautrix/bridge/matrix.py | 1 - mautrix/crypto/key_share.py | 12 +++--------- 4 files changed, 12 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cd7eb400..14b74816 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +## v0.17.8 (unreleased) + +* *(crypto)* Fixed parsing `/keys/claim` responses with no `failures` field. +* *(bridge)* Fixed parsing e2ee key sharing allow/minimum level config. + ## v0.17.7 (2022-08-22) * *(util.async_db)* Added `init_commands` to run commands on each SQLite diff --git a/mautrix/bridge/e2ee.py b/mautrix/bridge/e2ee.py index facaf456..a10ba543 100644 --- a/mautrix/bridge/e2ee.py +++ b/mautrix/bridge/e2ee.py @@ -61,6 +61,7 @@ class EncryptionManager: min_send_trust: TrustState min_share_trust: TrustState min_receive_trust: TrustState + key_sharing_enabled: bool bridge: br.Bridge az: AppService @@ -76,7 +77,6 @@ def __init__( user_id_prefix: str, user_id_suffix: str, db_url: str, - key_sharing_config: dict[str, bool] = None, ) -> None: self.loop = bridge.loop or asyncio.get_event_loop() self.bridge = bridge @@ -85,7 +85,6 @@ def __init__( self._id_prefix = user_id_prefix self._id_suffix = user_id_suffix self._share_session_events = {} - self.key_sharing_config = key_sharing_config or {} pickle_key = "mautrix.bridge.e2ee" self.crypto_db = Database.create( url=db_url, @@ -112,6 +111,7 @@ def __init__( self.min_receive_trust = TrustState.parse(verification_levels["receive"]) self.crypto.share_keys_min_trust = self.min_share_trust self.crypto.send_keys_min_trust = self.min_receive_trust + self.key_sharing_enabled = bridge.config["bridge.encryption.allow_key_sharing"] async def _exit_on_sync_fail(self, data) -> None: if data["error"]: @@ -119,9 +119,7 @@ async def _exit_on_sync_fail(self, data) -> None: sys.exit(32) async def allow_key_share(self, device: DeviceIdentity, request: RequestedKeyInfo) -> bool: - require_verification = self.key_sharing_config.get("require_verification", True) - allow = self.key_sharing_config.get("allow", False) - if not allow: + if not self.key_sharing_enabled: self.log.debug( f"Key sharing not enabled, ignoring key request from " f"{device.user_id}/{device.device_id}" @@ -134,7 +132,7 @@ async def allow_key_share(self, device: DeviceIdentity, request: RequestedKeyInf code=RoomKeyWithheldCode.BLACKLISTED, reason="You have been blacklisted by this device", ) - elif device.trust == TrustState.VERIFIED or not require_verification: + elif device.trust >= self.crypto.share_keys_min_trust: portal = await self.bridge.get_portal(request.room_id) if portal is None: raise RejectKeyShare( diff --git a/mautrix/bridge/matrix.py b/mautrix/bridge/matrix.py index e1487cc2..cf7b962a 100644 --- a/mautrix/bridge/matrix.py +++ b/mautrix/bridge/matrix.py @@ -182,7 +182,6 @@ def __init__( user_id_suffix=self.user_id_suffix, homeserver_address=self.config["homeserver.address"], db_url=self.config["appservice.database"], - key_sharing_config=self.config["bridge.encryption.key_sharing"], ) self.require_e2ee = self.config["bridge.encryption.require"] diff --git a/mautrix/crypto/key_share.py b/mautrix/crypto/key_share.py index ad4ccc74..58525bbf 100644 --- a/mautrix/crypto/key_share.py +++ b/mautrix/crypto/key_share.py @@ -76,18 +76,12 @@ async def default_allow_key_share( code=RoomKeyWithheldCode.BLACKLISTED, reason="You have been blacklisted by this device", ) - elif device.trust == TrustState.VERIFIED: - self.log.debug(f"Accepting key request from verified device {device.device_id}") - return True - elif self.share_to_unverified_devices: - self.log.debug( - f"Accepting key request from unverified device {device.device_id}, " - f"as share_to_unverified_devices is True" - ) + elif device.trust >= self.share_keys_min_trust: + self.log.debug(f"Accepting key request from trusted device {device.device_id}") return True else: raise RejectKeyShare( - f"Rejecting key request from unverified device {device.device_id}", + f"Rejecting key request from untrusted device {device.device_id}", code=RoomKeyWithheldCode.UNVERIFIED, reason="You have not been verified by this device", ) From b35801d7818fa872f1d6146771a05245a4396cbe Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 22 Aug 2022 16:00:40 +0300 Subject: [PATCH 193/456] Remove some unnecessary variables --- mautrix/bridge/e2ee.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/mautrix/bridge/e2ee.py b/mautrix/bridge/e2ee.py index a10ba543..24fc18eb 100644 --- a/mautrix/bridge/e2ee.py +++ b/mautrix/bridge/e2ee.py @@ -59,8 +59,6 @@ class EncryptionManager: state_store: StateStore min_send_trust: TrustState - min_share_trust: TrustState - min_receive_trust: TrustState key_sharing_enabled: bool bridge: br.Bridge @@ -106,11 +104,9 @@ def __init__( self.client.add_event_handler(InternalEventType.SYNC_STOPPED, self._exit_on_sync_fail) self.crypto.allow_key_share = self.allow_key_share verification_levels = bridge.config["bridge.encryption.verification_levels"] - self.min_share_trust = TrustState.parse(verification_levels["share"]) self.min_send_trust = TrustState.parse(verification_levels["send"]) - self.min_receive_trust = TrustState.parse(verification_levels["receive"]) - self.crypto.share_keys_min_trust = self.min_share_trust - self.crypto.send_keys_min_trust = self.min_receive_trust + self.crypto.share_keys_min_trust = TrustState.parse(verification_levels["share"]) + self.crypto.send_keys_min_trust = TrustState.parse(verification_levels["receive"]) self.key_sharing_enabled = bridge.config["bridge.encryption.allow_key_sharing"] async def _exit_on_sync_fail(self, data) -> None: From beab8df966ccafd04093c3644728da94bb30154c Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 22 Aug 2022 16:05:59 +0300 Subject: [PATCH 194/456] Bump version to 0.17.8 --- CHANGELOG.md | 2 +- mautrix/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 14b74816..baafedcc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -## v0.17.8 (unreleased) +## v0.17.8 (2022-08-22) * *(crypto)* Fixed parsing `/keys/claim` responses with no `failures` field. * *(bridge)* Fixed parsing e2ee key sharing allow/minimum level config. diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 2cce4b5b..50569673 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.17.7" +__version__ = "0.17.8" __author__ = "Tulir Asokan " __all__ = [ "api", From 34ca7248b42a3bf93ce3818ec24e4c73f11dfa33 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 22 Aug 2022 16:57:08 +0300 Subject: [PATCH 195/456] Log warnings if key claim fails --- mautrix/crypto/encrypt_olm.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/mautrix/crypto/encrypt_olm.py b/mautrix/crypto/encrypt_olm.py index e76fc9ef..620cbbc8 100644 --- a/mautrix/crypto/encrypt_olm.py +++ b/mautrix/crypto/encrypt_olm.py @@ -60,6 +60,7 @@ async def _create_outbound_sessions_locked( self, users: ClaimKeysList, _force_recreate_session: bool = False ) -> None: request: Dict[UserID, Dict[DeviceID, EncryptionKeyAlgorithm]] = {} + expected_devices = set() for user_id, devices in users.items(): request[user_id] = {} for device_id, identity in devices.items(): @@ -67,13 +68,18 @@ async def _create_outbound_sessions_locked( identity.identity_key ): request[user_id][device_id] = EncryptionKeyAlgorithm.SIGNED_CURVE25519 + expected_devices.add((user_id, device_id)) if not request[user_id]: del request[user_id] if not request: return + request_device_count = len(expected_devices) keys = await self.client.claim_keys(request) + for server, info in (keys.failures or {}).items(): + self.log.warning(f"Key claim failure for {server}: {info}") for user_id, devices in keys.one_time_keys.items(): for device_id, one_time_keys in devices.items(): + expected_devices.discard((user_id, device_id)) key_id, one_time_key_data = one_time_keys.popitem() one_time_key = one_time_key_data["key"] identity = users[user_id][device_id] @@ -90,6 +96,19 @@ async def _create_outbound_sessions_locked( f"Created new Olm session with {user_id}/{device_id} " f"(OTK ID: {key_id})" ) + if expected_devices: + if request_device_count == 1: + raise Exception( + "Key claim response didn't contain key " + f"for queried device {expected_devices.pop()}" + ) + else: + self.log.warning( + "Key claim response didn't contain keys for %d out of %d expected devices: %s", + len(expected_devices), + request_device_count, + expected_devices, + ) async def send_encrypted_to_device( self, From c7043793babd85ce6b3144524435cd9f943868f5 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 23 Aug 2022 21:15:49 +0300 Subject: [PATCH 196/456] Add knock_restricted join rule --- mautrix/types/event/state.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mautrix/types/event/state.py b/mautrix/types/event/state.py index acb24f81..d1638fff 100644 --- a/mautrix/types/event/state.py +++ b/mautrix/types/event/state.py @@ -144,6 +144,7 @@ class JoinRule(SerializableEnum): RESTRICTED = "restricted" INVITE = "invite" PRIVATE = "private" + KNOCK_RESTRICTED = "knock_restricted" class JoinRestrictionType(SerializableEnum): From e0b8a47425e21828318330b1de1ef58b4920d9b8 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 23 Aug 2022 21:26:47 +0300 Subject: [PATCH 197/456] Fix type hint --- mautrix/util/async_db/aiosqlite.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/util/async_db/aiosqlite.py b/mautrix/util/async_db/aiosqlite.py index 8d0b51fd..11d1015f 100644 --- a/mautrix/util/async_db/aiosqlite.py +++ b/mautrix/util/async_db/aiosqlite.py @@ -84,7 +84,7 @@ class SQLiteDatabase(Database): _pool: asyncio.Queue[TxnConnection] _stopped: bool _conns: int - _init_commands: List[str] + _init_commands: list[str] def __init__( self, From b58856e6881b86ceeb3caf30f84a41ff07970c7b Mon Sep 17 00:00:00 2001 From: finn Date: Wed, 24 Aug 2022 13:36:29 -0700 Subject: [PATCH 198/456] pass allow_redirect=true query param for downloads --- mautrix/client/api/modules/media_repository.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mautrix/client/api/modules/media_repository.py b/mautrix/client/api/modules/media_repository.py index 8e592679..deefe828 100644 --- a/mautrix/client/api/modules/media_repository.py +++ b/mautrix/client/api/modules/media_repository.py @@ -168,7 +168,7 @@ async def download_media(self, url: ContentURI, max_stall_ms: int | None = None) The raw downloaded data. """ url = self.api.get_download_url(url) - query_params: dict[str, Any] = {} + query_params: dict[str, Any] = {"allow_redirect": "true"} if max_stall_ms is not None: query_params["max_stall_ms"] = max_stall_ms query_params["fi.mau.msc2246.max_stall_ms"] = max_stall_ms @@ -206,7 +206,7 @@ async def download_thumbnail( The raw downloaded data. """ url = self.api.get_download_url(url, download_type="thumbnail") - query_params: dict[str, Any] = {} + query_params: dict[str, Any] = {"allow_redirect": "true"} if width is not None: query_params["width"] = width if height is not None: From cf6da2164b46a686af04a87db7182256f780ccf6 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 28 Aug 2022 22:40:27 +0300 Subject: [PATCH 199/456] Add support for overriding parts of config from env --- mautrix/bridge/bridge.py | 4 ++++ mautrix/bridge/config.py | 29 ++++++++++++++++++++++++++++- 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/mautrix/bridge/bridge.py b/mautrix/bridge/bridge.py index b09b0d11..14bea424 100644 --- a/mautrix/bridge/bridge.py +++ b/mautrix/bridge/bridge.py @@ -106,6 +106,8 @@ def preinit(self) -> None: sys.exit(0) def prepare(self) -> None: + if self.config.env: + self.log.debug("Loaded config overrides from environment: %s", self.config.env.keys()) super().prepare() self.prepare_db() self.prepare_appservice() @@ -115,6 +117,8 @@ def prepare_config(self) -> None: self.config = self.config_class( self.args.config, self.args.registration, self.args.base_config ) + if not self.config.env_prefix: + self.config.env_prefix = self.module.upper() if self.args.generate_registration: self.config._check_tokens = False self.load_and_update_config() diff --git a/mautrix/bridge/config.py b/mautrix/bridge/config.py index 5f55ca91..9a270490 100644 --- a/mautrix/bridge/config.py +++ b/mautrix/bridge/config.py @@ -5,8 +5,10 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import annotations -from typing import Any +from typing import Any, ClassVar from abc import ABC +import json +import os import re import secrets import time @@ -21,15 +23,40 @@ class BaseBridgeConfig(BaseFileConfig, BaseValidatableConfig, ABC): + env_prefix: str | None = None registration_path: str _registration: dict | None _check_tokens: bool + env: dict[str, Any] def __init__(self, path: str, registration_path: str, base_path: str) -> None: super().__init__(path, base_path) self.registration_path = registration_path self._registration = None self._check_tokens = True + self.env = {} + if self.env_prefix: + env_prefix = f"{self.env_prefix}_" + for key, value in os.environ.items(): + if not key.startswith(env_prefix): + continue + key = key.removeprefix(env_prefix) + if value.startswith("json::"): + value = json.loads(value) + self.env[key] = value + + def __getitem__(self, item: str) -> Any: + if self.env: + try: + sanitized_item = item.replace(".", "_").replace("[", "").replace("]", "").upper() + val = self.env[sanitized_item] + except KeyError: + pass + else: + if val.startswith("json::"): + val = json.loads(val.removeprefix("json::")) + return val + return super().__getitem__(item) def save(self) -> None: super().save() From 217c57bbf1f9d89ef4751ee5e3e01b8260e3b525 Mon Sep 17 00:00:00 2001 From: finn Date: Wed, 24 Aug 2022 16:30:07 -0700 Subject: [PATCH 200/456] MSC3870: allow media create endpoint to specify alternate upload URL --- .../client/api/modules/media_repository.py | 50 +++++++++++++------ mautrix/types/__init__.py | 10 +++- mautrix/types/media.py | 13 +++++ 3 files changed, 58 insertions(+), 15 deletions(-) diff --git a/mautrix/client/api/modules/media_repository.py b/mautrix/client/api/modules/media_repository.py index deefe828..5fc53529 100644 --- a/mautrix/client/api/modules/media_repository.py +++ b/mautrix/client/api/modules/media_repository.py @@ -13,7 +13,13 @@ from mautrix import __optional_imports__ from mautrix.api import MediaPath, Method from mautrix.errors import MatrixResponseError -from mautrix.types import ContentURI, MediaRepoConfig, MXOpenGraph, SerializerError +from mautrix.types import ( + ContentURI, + MediaCreateResponse, + MediaRepoConfig, + MXOpenGraph, + SerializerError, +) from mautrix.util.opt_prometheus import Histogram from ..base import BaseClientAPI @@ -45,22 +51,16 @@ class MediaRepositoryMethods(BaseClientAPI): uploads of media. """ - async def unstable_create_mxc(self) -> ContentURI: + async def unstable_create_mxc(self) -> MediaCreateResponse: """ Create a media ID for uploading media to the homeserver. Requires the homeserver to have `MSC2246 `__ support. Returns: - The MXC URI that can be used to upload a file to later. - - Raises: - MatrixResponseError: If the response does not contain a ``content_uri`` field. + MediaCreateResponse Containing the MXC URI that can be used to upload a file to later, as well as an optional upload URL """ resp = await self.api.request(Method.POST, MediaPath.unstable["fi.mau.msc2246"].create) - try: - return resp["content_uri"] - except KeyError: - raise MatrixResponseError("`content_uri` not in response.") + return MediaCreateResponse.deserialize(resp) @contextmanager def _observe_upload_time(self, size: int | None, mxc: ContentURI | None = None) -> None: @@ -121,19 +121,32 @@ async def upload_media( if filename: query["filename"] = filename + upload_url = None + if async_upload: if mxc: raise ValueError("async_upload and mxc can't be provided simultaneously") - mxc = await self.unstable_create_mxc() + create_response = await self.unstable_create_mxc() + mxc = create_response.content_uri + upload_url = create_response.upload_url path = MediaPath.v3.upload method = Method.POST if mxc: server_name, media_id = self.api.parse_mxc_uri(mxc) - path = MediaPath.unstable["fi.mau.msc2246"].upload[server_name][media_id] - method = Method.PUT + if upload_url is None: + path = MediaPath.unstable["fi.mau.msc2246"].upload[server_name][media_id] + method = Method.PUT + else: + path = MediaPath.unstable["fi.mau.msc2246"].upload[server_name][media_id].complete + + if upload_url is not None: + task = self._upload_to_url(upload_url, path, data) + else: + task = self.api.request( + method, path, content=data, headers=headers, query_params=query + ) - task = self.api.request(method, path, content=data, headers=headers, query_params=query) if async_upload: async def _try_upload(): @@ -265,3 +278,12 @@ async def get_media_repo_config(self) -> MediaRepoConfig: return MediaRepoConfig.deserialize(content) except SerializerError as e: raise MatrixResponseError("Invalid MediaRepoConfig in response") from e + + async def _upload_to_url(self, upload_url: str, post_upload_path: str, data: Any): + response = await self.api.session.request(Method.PUT.name, upload_url, data=data) + if not response.ok: + self.log.error( + f"non-ok http response from upload URL: PUT {upload_url} returned {response.status}" + ) + raise Exception("non-ok http response from server") + await self.api.request(Method.POST, post_upload_path) diff --git a/mautrix/types/__init__.py b/mautrix/types/__init__.py index fec96028..83718f57 100644 --- a/mautrix/types/__init__.py +++ b/mautrix/types/__init__.py @@ -138,7 +138,14 @@ ) from .filter import EventFilter, Filter, RoomEventFilter, RoomFilter, StateFilter from .matrixuri import IdentifierType, MatrixURI, MatrixURIError, URIAction -from .media import MediaRepoConfig, MXOpenGraph, OpenGraphAudio, OpenGraphImage, OpenGraphVideo +from .media import ( + MediaCreateResponse, + MediaRepoConfig, + MXOpenGraph, + OpenGraphAudio, + OpenGraphImage, + OpenGraphVideo, +) from .misc import ( BatchSendResponse, DeviceLists, @@ -342,6 +349,7 @@ "OpenGraphAudio", "OpenGraphImage", "OpenGraphVideo", + "MediaCreateResponse", "BatchSendResponse", "DeviceLists", "DeviceOTKCount", diff --git a/mautrix/types/media.py b/mautrix/types/media.py index 38746cbf..72aa6c4f 100644 --- a/mautrix/types/media.py +++ b/mautrix/types/media.py @@ -3,6 +3,8 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. +from typing import Optional + from attr import dataclass from .primitive import ContentURI @@ -59,3 +61,14 @@ class MXOpenGraph(SerializableAttrs): image: OpenGraphImage = field(default=None, flatten=True) video: OpenGraphVideo = field(default=None, flatten=True) audio: OpenGraphAudio = field(default=None, flatten=True) + + +@dataclass +class MediaCreateResponse(SerializableAttrs): + """ + Matrix media create response including MSC3870 + """ + + content_uri: ContentURI + unused_expired_at: Optional[int] = None + upload_url: Optional[str] = None From 93d39bed9297140cbb86c8d203e8cd7aec238424 Mon Sep 17 00:00:00 2001 From: finn Date: Tue, 30 Aug 2022 16:14:44 -0700 Subject: [PATCH 201/456] add retry logic to media upload --- .../client/api/modules/media_repository.py | 28 +++++++++++++++---- 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/mautrix/client/api/modules/media_repository.py b/mautrix/client/api/modules/media_repository.py index 5fc53529..965e0713 100644 --- a/mautrix/client/api/modules/media_repository.py +++ b/mautrix/client/api/modules/media_repository.py @@ -12,7 +12,7 @@ from mautrix import __optional_imports__ from mautrix.api import MediaPath, Method -from mautrix.errors import MatrixResponseError +from mautrix.errors import MatrixResponseError, make_request_error from mautrix.types import ( ContentURI, MediaCreateResponse, @@ -280,10 +280,26 @@ async def get_media_repo_config(self) -> MediaRepoConfig: raise MatrixResponseError("Invalid MediaRepoConfig in response") from e async def _upload_to_url(self, upload_url: str, post_upload_path: str, data: Any): - response = await self.api.session.request(Method.PUT.name, upload_url, data=data) - if not response.ok: - self.log.error( - f"non-ok http response from upload URL: PUT {upload_url} returned {response.status}" + retry_count = self.api.default_retry_count + backoff = 4 + while True: + upload_response = await self.api.session.request( + Method.PUT.name, upload_url, data=data ) - raise Exception("non-ok http response from server") + if not upload_response.ok: + if retry_count == 0: + raise make_request_error( + http_status=upload_response.status, + text=await upload_response.text(), + ) + self.log.warning( + f"non-ok http response from upload URL: PUT {upload_url} returned" + f" {upload_response.status}, retrying in {backoff} seconds" + ) + await asyncio.sleep(backoff) + backoff *= 2 + retry_count = -1 + else: + break + await self.api.request(Method.POST, post_upload_path) From c748aa353fc647bbcdda17590d3fe0e5b4fbd019 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 14 Sep 2022 18:00:02 +0300 Subject: [PATCH 202/456] Prevent restarting async database --- mautrix/util/async_db/aiosqlite.py | 4 ++++ mautrix/util/async_db/asyncpg.py | 2 ++ 2 files changed, 6 insertions(+) diff --git a/mautrix/util/async_db/aiosqlite.py b/mautrix/util/async_db/aiosqlite.py index 11d1015f..444aaef4 100644 --- a/mautrix/util/async_db/aiosqlite.py +++ b/mautrix/util/async_db/aiosqlite.py @@ -113,6 +113,10 @@ def __init__( self._init_commands = self._db_args.pop("init_commands", []) async def start(self) -> None: + if self._conns: + raise RuntimeError("database pool has already been started") + elif self._stopped: + raise RuntimeError("database pool can't be restarted") self.log.debug(f"Connecting to {self.url}") for _ in range(self._pool.maxsize): conn = await TxnConnection(self._path, **self._db_args) diff --git a/mautrix/util/async_db/asyncpg.py b/mautrix/util/async_db/asyncpg.py index 03a33646..07ef7ad7 100644 --- a/mautrix/util/async_db/asyncpg.py +++ b/mautrix/util/async_db/asyncpg.py @@ -61,6 +61,8 @@ def override_pool(self, db: PostgresDatabase) -> None: async def start(self) -> None: if not self._pool_override: + if self._pool: + raise RuntimeError("Database has already been started") self._db_args["loop"] = asyncio.get_running_loop() log_url = self.url if log_url.password: From e9233d754977b41fe2c55f508b720f6208b49955 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 14 Sep 2022 18:11:36 +0300 Subject: [PATCH 203/456] Add missing import --- mautrix/appservice/as_handler.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mautrix/appservice/as_handler.py b/mautrix/appservice/as_handler.py index fa8c196d..206786b4 100644 --- a/mautrix/appservice/as_handler.py +++ b/mautrix/appservice/as_handler.py @@ -16,6 +16,7 @@ from mautrix.types import ( JSON, + DeviceID, DeviceLists, DeviceOTKCount, EphemeralEvent, From d86dc8c9390e36579e5170c7510c984572cc257f Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 14 Sep 2022 20:01:38 +0300 Subject: [PATCH 204/456] Add support for MSC2409/MSC3202 for end-to-bridge encryption --- mautrix/appservice/appservice.py | 3 +- mautrix/appservice/as_handler.py | 56 ++++++++++++++++++++++++-------- mautrix/bridge/bridge.py | 1 + mautrix/bridge/config.py | 1 + mautrix/bridge/e2ee.py | 15 +++++++-- mautrix/crypto/machine.py | 27 +++++++++++++++ mautrix/types/__init__.py | 4 ++- mautrix/types/event/__init__.py | 1 + mautrix/types/event/generic.py | 3 +- mautrix/types/event/to_device.py | 6 ++++ 10 files changed, 97 insertions(+), 20 deletions(-) diff --git a/mautrix/appservice/appservice.py b/mautrix/appservice/appservice.py index 053ddb1b..4c9bf471 100644 --- a/mautrix/appservice/appservice.py +++ b/mautrix/appservice/appservice.py @@ -77,11 +77,12 @@ def __init__( state_store: ASStateStore = None, aiohttp_params: dict = None, ephemeral_events: bool = False, + encryption_events: bool = False, default_ua: str = HTTPAPI.default_ua, default_http_retry_count: int = 0, connection_limit: int | None = None, ) -> None: - super().__init__(ephemeral_events=ephemeral_events) + super().__init__(ephemeral_events=ephemeral_events, encryption_events=encryption_events) self.server = server self.domain = domain self.id = id diff --git a/mautrix/appservice/as_handler.py b/mautrix/appservice/as_handler.py index 206786b4..e3c6980e 100644 --- a/mautrix/appservice/as_handler.py +++ b/mautrix/appservice/as_handler.py @@ -16,6 +16,7 @@ from mautrix.types import ( JSON, + ASToDeviceEvent, DeviceID, DeviceLists, DeviceOTKCount, @@ -35,17 +36,25 @@ class AppServiceServerMixin: hs_token: str ephemeral_events: bool + encryption_events: bool query_user: Callable[[UserID], JSON] query_alias: Callable[[RoomAlias], JSON] transactions: set[str] event_handlers: list[HandlerFunc] + to_device_handler: HandlerFunc | None + otk_handler: Callable[[dict[UserID, dict[DeviceID, DeviceOTKCount]]], Awaitable] | None + device_list_handler: Callable[[DeviceLists], Awaitable] | None - def __init__(self, ephemeral_events: bool = False) -> None: + def __init__(self, ephemeral_events: bool = False, encryption_events: bool = False) -> None: self.transactions = set() self.event_handlers = [] + self.to_device_handler = None + self.otk_handler = None + self.device_list_handler = None self.ephemeral_events = ephemeral_events + self.encryption_events = encryption_events async def default_query_handler(_): return None @@ -166,18 +175,24 @@ async def _http_handle_transaction(self, request: web.Request) -> web.Response: if self.ephemeral_events else None ) - device_lists = DeviceLists.deserialize( - self._get_with_fallback(data, "device_lists", "org.matrix.msc3202") - ) - otk_counts = { - user_id: { - device_id: DeviceOTKCount.deserialize(count) - for device_id, count in devices.items() + if self.encryption_events: + to_device = self._get_with_fallback(data, "to_device", "de.sorunome.msc2409") + device_lists = DeviceLists.deserialize( + self._get_with_fallback(data, "device_lists", "org.matrix.msc3202") + ) + otk_counts = { + user_id: { + device_id: DeviceOTKCount.deserialize(count) + for device_id, count in devices.items() + } + for user_id, devices in self._get_with_fallback( + data, "device_one_time_keys_count", "org.matrix.msc3202", default={} + ).items() } - for user_id, devices in self._get_with_fallback( - data, "device_one_time_keys_count", "org.matrix.msc3202", default={} - ).items() - } + else: + otk_counts = {} + device_lists = None + to_device = None try: output = await self.handle_transaction( @@ -185,8 +200,9 @@ async def _http_handle_transaction(self, request: web.Request) -> web.Response: events=events, extra_data=data, ephemeral=ephemeral, + to_device=to_device, device_lists=device_lists, - device_otk_count=otk_counts, + otk_counts=otk_counts, ) except Exception: self.log.exception("Exception in transaction handler") @@ -213,9 +229,21 @@ async def handle_transaction( events: list[JSON], extra_data: JSON, ephemeral: list[JSON] | None = None, - device_otk_count: dict[UserID, dict[DeviceID, DeviceOTKCount]] | None = None, + to_device: list[JSON] | None = None, + otk_counts: dict[UserID, dict[DeviceID, DeviceOTKCount]] | None = None, device_lists: DeviceLists | None = None, ) -> JSON: + for raw_td in to_device or []: + try: + td = ASToDeviceEvent.deserialize(raw_td) + except SerializerError: + self.log.exception("Failed to deserialize to-device event %s", raw_td) + else: + await self.to_device_handler(td) + if device_lists and self.device_list_handler: + await self.device_list_handler(device_lists) + if otk_counts and self.otk_handler: + await self.otk_handler(otk_counts) for raw_edu in ephemeral or []: try: edu = EphemeralEvent.deserialize(raw_edu) diff --git a/mautrix/bridge/bridge.py b/mautrix/bridge/bridge.py index 14bea424..e7c3cde8 100644 --- a/mautrix/bridge/bridge.py +++ b/mautrix/bridge/bridge.py @@ -156,6 +156,7 @@ def prepare_appservice(self) -> None: tls_key=self.config.get("appservice.tls_key", None), bot_localpart=self.config["appservice.bot_username"], ephemeral_events=self.config["appservice.ephemeral_events"], + encryption_events=self.config["bridge.encryption.appservice"], default_ua=HTTPAPI.default_ua, default_http_retry_count=default_http_retry_count, log="mau.as", diff --git a/mautrix/bridge/config.py b/mautrix/bridge/config.py index 9a270490..0352f88b 100644 --- a/mautrix/bridge/config.py +++ b/mautrix/bridge/config.py @@ -132,6 +132,7 @@ def do_update(self, helper: ConfigUpdateHelper) -> None: copy("bridge.encryption.allow") copy("bridge.encryption.default") copy("bridge.encryption.require") + copy("bridge.encryption.appservice") copy("bridge.encryption.verification_levels.receive") copy("bridge.encryption.verification_levels.send") copy("bridge.encryption.verification_levels.share") diff --git a/mautrix/bridge/e2ee.py b/mautrix/bridge/e2ee.py index 24fc18eb..516e54bd 100644 --- a/mautrix/bridge/e2ee.py +++ b/mautrix/bridge/e2ee.py @@ -34,7 +34,7 @@ StateFilter, TrustState, ) -from mautrix.util.async_db import Database, DatabaseException +from mautrix.util.async_db import Database from mautrix.util.logging import TraceLogger from .. import bridge as br @@ -60,6 +60,7 @@ class EncryptionManager: min_send_trust: TrustState key_sharing_enabled: bool + appservice_mode: bool bridge: br.Bridge az: AppService @@ -108,6 +109,11 @@ def __init__( self.crypto.share_keys_min_trust = TrustState.parse(verification_levels["share"]) self.crypto.send_keys_min_trust = TrustState.parse(verification_levels["receive"]) self.key_sharing_enabled = bridge.config["bridge.encryption.allow_key_sharing"] + self.appservice_mode = bridge.config["bridge.encryption.appservice"] + if self.appservice_mode: + self.az.otk_handler = self.crypto.handle_as_otk_counts + self.az.device_list_handler = self.crypto.handle_as_device_lists + self.az.to_device_handler = self.crypto.handle_as_to_device_event async def _exit_on_sync_fail(self, data) -> None: if data["error"]: @@ -256,8 +262,11 @@ async def start(self) -> None: self.log.debug(f"Logged in with new device ID {self.client.device_id}") elif self.crypto.account.shared: await self._verify_keys_are_on_server() - _ = self.client.start(self._filter) - self.log.info("End-to-bridge encryption support is enabled") + if self.appservice_mode: + self.log.info("End-to-bridge encryption support is enabled (appservice mode)") + else: + _ = self.client.start(self._filter) + self.log.info("End-to-bridge encryption support is enabled (sync mode)") async def _verify_keys_are_on_server(self) -> None: self.log.debug("Making sure keys are still on server") diff --git a/mautrix/crypto/machine.py b/mautrix/crypto/machine.py index 3d0410f0..14e9e95f 100644 --- a/mautrix/crypto/machine.py +++ b/mautrix/crypto/machine.py @@ -11,7 +11,9 @@ from mautrix import client as cli from mautrix.types import ( + ASToDeviceEvent, DecryptedOlmEvent, + DeviceID, DeviceLists, DeviceOTKCount, EncryptionAlgorithm, @@ -20,6 +22,7 @@ StateEvent, ToDeviceEvent, TrustState, + UserID, ) from mautrix.util.logging import TraceLogger @@ -95,6 +98,30 @@ async def load(self) -> None: self.account = OlmAccount() await self.crypto_store.put_account(self.account) + async def handle_as_otk_counts( + self, otk_counts: dict[UserID, dict[DeviceID, DeviceOTKCount]] + ) -> None: + for user_id, devices in otk_counts.items(): + for device_id, count in devices.items(): + if user_id == self.client.mxid and device_id == self.client.device_id: + await self.handle_otk_count(count) + else: + self.log.warning(f"Got OTK count for unknown device {user_id}/{device_id}") + + async def handle_as_device_lists(self, device_lists: DeviceLists) -> None: + asyncio.create_task(self.handle_device_lists(device_lists)) + + async def handle_as_to_device_event(self, evt: ASToDeviceEvent) -> None: + if evt.to_user_id != self.client.mxid or evt.to_device_id != self.client.device_id: + self.log.warning( + f"Got to-device event for unknown device {evt.to_user_id}/{evt.to_device_id}" + ) + return + if evt.type == EventType.TO_DEVICE_ENCRYPTED: + await self.handle_to_device_event(evt) + else: + self.log.debug(f"Got unknown to-device event {evt.type} from {evt.sender}") + async def handle_otk_count(self, otk_count: DeviceOTKCount) -> None: """ Handle the ``device_one_time_keys_count`` data in a sync response. diff --git a/mautrix/types/__init__.py b/mautrix/types/__init__.py index 83718f57..6d2ffa7c 100644 --- a/mautrix/types/__init__.py +++ b/mautrix/types/__init__.py @@ -31,6 +31,7 @@ from .event import ( AccountDataEvent, AccountDataEventContent, + ASToDeviceEvent, AudioInfo, BaseEvent, BaseFileInfo, @@ -231,6 +232,7 @@ "UnsignedDeviceInfo", "AccountDataEvent", "AccountDataEventContent", + "ASToDeviceEvent", "AudioInfo", "BaseEvent", "BaseFileInfo", @@ -344,12 +346,12 @@ "MatrixURI", "MatrixURIError", "URIAction", + "MediaCreateResponse", "MediaRepoConfig", "MXOpenGraph", "OpenGraphAudio", "OpenGraphImage", "OpenGraphVideo", - "MediaCreateResponse", "BatchSendResponse", "DeviceLists", "DeviceOTKCount", diff --git a/mautrix/types/event/__init__.py b/mautrix/types/event/__init__.py index d9a1d681..b391e912 100644 --- a/mautrix/types/event/__init__.py +++ b/mautrix/types/event/__init__.py @@ -92,6 +92,7 @@ StrippedStateEvent, ) from .to_device import ( + ASToDeviceEvent, ForwardedRoomKeyEventContent, KeyRequestAction, RequestedKeyInfo, diff --git a/mautrix/types/event/generic.py b/mautrix/types/event/generic.py index f105f694..155aef90 100644 --- a/mautrix/types/event/generic.py +++ b/mautrix/types/event/generic.py @@ -23,7 +23,7 @@ from .reaction import ReactionEvent, ReactionEventContent from .redaction import RedactionEvent, RedactionEventContent from .state import StateEvent, StateEventContent -from .to_device import ToDeviceEvent, ToDeviceEventContent +from .to_device import ASToDeviceEvent, ToDeviceEvent, ToDeviceEventContent from .voip import CallEvent, CallEventContent, type_to_class as voip_types Event = NewType( @@ -38,6 +38,7 @@ PresenceEvent, EncryptedEvent, ToDeviceEvent, + ASToDeviceEvent, CallEvent, BeeperMessageStatusEvent, GenericEvent, diff --git a/mautrix/types/event/to_device.py b/mautrix/types/event/to_device.py index 32ffe6e3..d1f19c1a 100644 --- a/mautrix/types/event/to_device.py +++ b/mautrix/types/event/to_device.py @@ -110,3 +110,9 @@ def deserialize_content(data: JSON) -> ToDeviceEventContent: if not content_type: return Obj(**data) return content_type.deserialize(data) + + +@dataclass +class ASToDeviceEvent(ToDeviceEvent, SerializableAttrs): + to_user_id: UserID + to_device_id: DeviceID From 398743782bdb926574194139bd954584bcf7821e Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 14 Sep 2022 20:19:14 +0300 Subject: [PATCH 205/456] Ensure event type class is correct in more places --- mautrix/appservice/as_handler.py | 14 +++++++++----- mautrix/types/event/account_data.py | 7 +++++-- mautrix/types/event/ephemeral.py | 10 ++++++---- mautrix/types/event/state.py | 4 +++- mautrix/types/event/to_device.py | 4 +++- 5 files changed, 26 insertions(+), 13 deletions(-) diff --git a/mautrix/appservice/as_handler.py b/mautrix/appservice/as_handler.py index e3c6980e..8fd318a1 100644 --- a/mautrix/appservice/as_handler.py +++ b/mautrix/appservice/as_handler.py @@ -22,6 +22,7 @@ DeviceOTKCount, EphemeralEvent, Event, + EventType, RoomAlias, SerializerError, UserID, @@ -250,7 +251,7 @@ async def handle_transaction( except SerializerError: self.log.exception("Failed to deserialize ephemeral event %s", raw_edu) else: - self.handle_matrix_event(edu) + self.handle_matrix_event(edu, ephemeral=True) for raw_event in events: try: self._fix_prev_content(raw_event) @@ -261,10 +262,13 @@ async def handle_transaction( self.handle_matrix_event(event) return {} - def handle_matrix_event(self, event: Event) -> None: - if event.type.is_state and event.state_key is None: - self.log.debug(f"Not sending {event.event_id} to handlers: expected state_key.") - return + def handle_matrix_event(self, event: Event, ephemeral: bool = False) -> None: + if ephemeral: + event.type = event.type.with_class(EventType.Class.EPHEMERAL) + elif getattr(event, "state_key", None) is not None: + event.type = event.type.with_class(EventType.Class.STATE) + else: + event.type = event.type.with_class(EventType.Class.MESSAGE) async def try_handle(handler_func: HandlerFunc): try: diff --git a/mautrix/types/event/account_data.py b/mautrix/types/event/account_data.py index 2bbe1ac9..3c144a36 100644 --- a/mautrix/types/event/account_data.py +++ b/mautrix/types/event/account_data.py @@ -43,10 +43,13 @@ class AccountDataEvent(BaseEvent, SerializableAttrs): @classmethod def deserialize(cls, data: JSON) -> "AccountDataEvent": try: - data.get("content", {})["__mautrix_event_type"] = EventType.find(data.get("type")) + evt_type = EventType.find(data.get("type")) + data.get("content", {})["__mautrix_event_type"] = evt_type except ValueError: return Obj(**data) - return super().deserialize(data) + evt = super().deserialize(data) + evt.type = evt_type + return evt @staticmethod @deserializer(AccountDataEventContent) diff --git a/mautrix/types/event/ephemeral.py b/mautrix/types/event/ephemeral.py index 7252f5c5..a1483a23 100644 --- a/mautrix/types/event/ephemeral.py +++ b/mautrix/types/event/ephemeral.py @@ -70,13 +70,15 @@ class ReceiptEvent(BaseEvent, SerializableAttrs): def deserialize_ephemeral_event(data: JSON) -> EphemeralEvent: event_type = EventType.find(data.get("type", None)) if event_type == EventType.RECEIPT: - return ReceiptEvent.deserialize(data) + evt = ReceiptEvent.deserialize(data) elif event_type == EventType.TYPING: - return TypingEvent.deserialize(data) + evt = TypingEvent.deserialize(data) elif event_type == EventType.PRESENCE: - return PresenceEvent.deserialize(data) + evt = PresenceEvent.deserialize(data) else: - return GenericEvent.deserialize(data) + evt = GenericEvent.deserialize(data) + evt.type = event_type + return evt setattr(EphemeralEvent, "deserialize", deserialize_ephemeral_event) diff --git a/mautrix/types/event/state.py b/mautrix/types/event/state.py index d1638fff..1ef18e3b 100644 --- a/mautrix/types/event/state.py +++ b/mautrix/types/event/state.py @@ -310,7 +310,9 @@ def deserialize(cls, data: JSON) -> "StateEvent": data.get("unsigned", {}).get("prev_content", {})["__mautrix_event_type"] = event_type except ValueError: return Obj(**data) - return super().deserialize(data) + evt = super().deserialize(data) + evt.type = event_type + return evt @staticmethod @deserializer(StateEventContent) diff --git a/mautrix/types/event/to_device.py b/mautrix/types/event/to_device.py index d1f19c1a..2f5d4864 100644 --- a/mautrix/types/event/to_device.py +++ b/mautrix/types/event/to_device.py @@ -100,7 +100,9 @@ def deserialize(cls, data: JSON) -> "ToDeviceEvent": data.setdefault("content", {})["__mautrix_event_type"] = evt_type except ValueError: return Obj(**data) - return super().deserialize(data) + evt = super().deserialize(data) + evt.type = evt_type + return evt @staticmethod @deserializer(ToDeviceEventContent) From 690f4702cbdf1145646a6deebd21946a6338854c Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 14 Sep 2022 20:30:50 +0300 Subject: [PATCH 206/456] Replace homeserver.asmux flag with generic software field --- mautrix/bridge/config.py | 4 ++++ mautrix/bridge/user.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/mautrix/bridge/config.py b/mautrix/bridge/config.py index 0352f88b..e9e51a49 100644 --- a/mautrix/bridge/config.py +++ b/mautrix/bridge/config.py @@ -101,6 +101,10 @@ def do_update(self, helper: ConfigUpdateHelper) -> None: copy("homeserver.status_endpoint") copy("homeserver.message_send_checkpoint_endpoint") copy("homeserver.async_media") + if self.get("homeserver.asmux", False): + helper.base["homeserver.software"] = "asmux" + else: + copy("homeserver.software") copy("appservice.address") copy("appservice.hostname") diff --git a/mautrix/bridge/user.py b/mautrix/bridge/user.py index 9948c286..cbb63141 100644 --- a/mautrix/bridge/user.py +++ b/mautrix/bridge/user.py @@ -128,7 +128,7 @@ async def update_direct_chats(self, dms: dict[UserID, list[RoomID]] | None = Non self.log.debug("Updating m.direct list on homeserver") replace = dms is None dms = dms or await self.get_direct_chats() - if self.bridge.config.get("homeserver.asmux", False): + if self.bridge.config.get("homeserver.software", "standard") == "asmux": # This uses a secret endpoint for atomically updating the DM list await puppet.intent.api.request( Method.PUT if replace else Method.PATCH, From 2a905cd66a8fce097f79075a97ef164d897a5a38 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 14 Sep 2022 20:45:07 +0300 Subject: [PATCH 207/456] Log transaction contents --- mautrix/appservice/as_handler.py | 30 +++++++++++++++++++++++++----- mautrix/types/misc.py | 3 +++ 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/mautrix/appservice/as_handler.py b/mautrix/appservice/as_handler.py index 8fd318a1..48c77217 100644 --- a/mautrix/appservice/as_handler.py +++ b/mautrix/appservice/as_handler.py @@ -161,8 +161,11 @@ async def _read_transaction_header(self, request: web.Request) -> tuple[str, dic async def _http_handle_transaction(self, request: web.Request) -> web.Response: transaction_id, data = await self._read_transaction_header(request) + txn_content_log = [] try: events = data.pop("events") + if events: + txn_content_log.append(f"{len(events)} PDUs") except KeyError: raise web.HTTPBadRequest( content_type="application/json", @@ -171,11 +174,12 @@ async def _http_handle_transaction(self, request: web.Request) -> web.Response: ), ) - ephemeral = ( - self._get_with_fallback(data, "ephemeral", "de.sorunome.msc2409") - if self.ephemeral_events - else None - ) + if self.ephemeral_events: + ephemeral = self._get_with_fallback(data, "ephemeral", "de.sorunome.msc2409") + if ephemeral: + txn_content_log.append(f"{len(ephemeral)} EDUs") + else: + ephemeral = None if self.encryption_events: to_device = self._get_with_fallback(data, "to_device", "de.sorunome.msc2409") device_lists = DeviceLists.deserialize( @@ -190,11 +194,27 @@ async def _http_handle_transaction(self, request: web.Request) -> web.Response: data, "device_one_time_keys_count", "org.matrix.msc3202", default={} ).items() } + if to_device: + txn_content_log.append(f"{len(to_device)} to-device events") + if device_lists.changed: + txn_content_log.append(f"{len(device_lists.changed)} device list changes") + if otk_counts: + txn_content_log.append( + f"{sum(len(vals) for vals in otk_counts.values())} OTK counts" + ) else: otk_counts = {} device_lists = None to_device = None + if len(txn_content_log) > 2: + txn_content_log = [", ".join(txn_content_log[:-1]), txn_content_log[-1]] + if not txn_content_log: + txn_description = "nothing?" + else: + txn_description = " and ".join(txn_content_log) + self.log.debug(f"Handling transaction {transaction_id} with {txn_description}") + try: output = await self.handle_transaction( transaction_id, diff --git a/mautrix/types/misc.py b/mautrix/types/misc.py index 6be085c5..7a978bd1 100644 --- a/mautrix/types/misc.py +++ b/mautrix/types/misc.py @@ -19,6 +19,9 @@ class DeviceLists(SerializableAttrs): changed: List[UserID] = attr.ib(factory=lambda: []) left: List[UserID] = attr.ib(factory=lambda: []) + def __bool__(self) -> bool: + return bool(self.changed or self.left) + @dataclass class DeviceOTKCount(SerializableAttrs): From 06caff41f10d371e6658a3eb73116f6796abec79 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 14 Sep 2022 21:01:14 +0300 Subject: [PATCH 208/456] Update changelog --- CHANGELOG.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index baafedcc..ebba0eea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,27 @@ +## v0.18.0 (unreleased) + +* **Breaking change *(util.async_db)*** Added checks to prevent calling + `.start()` on a database multiple times. +* *(appservice)* Fixed [MSC2409] support to read to-device events from the + correct field. +* *(appservice)* Added support for automatically calling functions when a + transaction contains [MSC2409] to-device events or [MSC3202] encryption data. +* *(bridge)* Added option to use [MSC2409] and [MSC3202] for end-to-bridge + encryption. However, this may not work with the Synapse implementation as it + hasn't been tested yet. +* *(bridge)* Replaced `homeserver` -> `asmux` flag with more generic `software` + field. +* *(bridge)* Added support for overriding parts of config with environment + variables. + * If the value starts with `json::`, it'll be parsed as JSON instead of using + as a raw string. +* *(client.api)* Added support for [MSC3870] for both uploading and downloading + media. +* *(types)* Added `knock_restricted` join rule to `JoinRule` enum. +* *(crypto)* Added warning logs if claiming one-time keys for other users fails. + +[MSC3870]: https://github.com/matrix-org/matrix-spec-proposals/pull/3870 + ## v0.17.8 (2022-08-22) * *(crypto)* Fixed parsing `/keys/claim` responses with no `failures` field. From 117698b3b060e892f99539a17bd2792fb9a4838d Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 14 Sep 2022 22:24:07 +0300 Subject: [PATCH 209/456] Fix withheld unauthorized error code --- mautrix/types/event/to_device.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/types/event/to_device.py b/mautrix/types/event/to_device.py index 2f5d4864..6a17e36e 100644 --- a/mautrix/types/event/to_device.py +++ b/mautrix/types/event/to_device.py @@ -17,7 +17,7 @@ class RoomKeyWithheldCode(ExtensibleEnum): BLACKLISTED: "RoomKeyWithheldCode" = "m.blacklisted" UNVERIFIED: "RoomKeyWithheldCode" = "m.unverified" - UNAUTHORIZED: "RoomKeyWithheldCode" = "m.unauthorized" + UNAUTHORIZED: "RoomKeyWithheldCode" = "m.unauthorised" UNAVAILABLE: "RoomKeyWithheldCode" = "m.unavailable" NO_OLM_SESSION: "RoomKeyWithheldCode" = "m.no_olm" From 6569ce0362007590ba62b00a9db1fa217e7a78bd Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 15 Sep 2022 16:29:49 +0300 Subject: [PATCH 210/456] Use new type hints in errors --- mautrix/errors/request.py | 36 ++++++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/mautrix/errors/request.py b/mautrix/errors/request.py index 73824c3f..ebff4d76 100644 --- a/mautrix/errors/request.py +++ b/mautrix/errors/request.py @@ -3,7 +3,9 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from typing import Callable, Dict, Optional, Type +from __future__ import annotations + +from typing import Callable, Type from .base import MatrixError @@ -12,25 +14,30 @@ class MatrixRequestError(MatrixError): """An error that was returned by the homeserver.""" http_status: int - message: Optional[str] + message: str | None errcode: str class MatrixUnknownRequestError(MatrixRequestError): """An unknown error type returned by the homeserver.""" + http_status: int + text: str + errcode: str | None + message: str | None + def __init__( self, http_status: int = 0, text: str = "", - errcode: Optional[str] = None, - message: Optional[str] = None, + errcode: str | None = None, + message: str | None = None, ) -> None: super().__init__(f"{http_status}: {text}") - self.http_status: int = http_status - self.text: str = text - self.errcode: Optional[str] = errcode - self.message: Optional[str] = message + self.http_status = http_status + self.text = text + self.errcode = errcode + self.message = message class MatrixStandardRequestError(MatrixRequestError): @@ -45,11 +52,11 @@ def __init__(self, http_status: int, message: str = "") -> None: MxSRE = Type[MatrixStandardRequestError] -ec_map: Dict[str, MxSRE] = {} -uec_map: Dict[str, MxSRE] = {} +ec_map: dict[str, MxSRE] = {} +uec_map: dict[str, MxSRE] = {} -def standard_error(code: str, unstable: Optional[str] = None) -> Callable[[MxSRE], MxSRE]: +def standard_error(code: str, unstable: str | None = None) -> Callable[[MxSRE], MxSRE]: def decorator(cls: MxSRE) -> MxSRE: cls.errcode = code ec_map[code] = cls @@ -62,7 +69,11 @@ def decorator(cls: MxSRE) -> MxSRE: def make_request_error( - http_status: int, text: str, errcode: str, message: str, unstable_errcode: Optional[str] = None + http_status: int, + text: str, + errcode: str | None, + message: str | None, + unstable_errcode: str | None = None, ) -> MatrixRequestError: """ Determine the correct exception class for the error code and create an instance of that class @@ -73,6 +84,7 @@ def make_request_error( text: The raw response text. errcode: The errcode field in the response JSON. message: The error field in the response JSON. + unstable_errcode: The MSC3848 error code field in the response JSON. """ if unstable_errcode: try: From 4472f306afd9b2c09f59d4d4da347ae6bf64ec67 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 15 Sep 2022 16:30:01 +0300 Subject: [PATCH 211/456] Clean up upload to external URL code --- .../client/api/modules/media_repository.py | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/mautrix/client/api/modules/media_repository.py b/mautrix/client/api/modules/media_repository.py index 965e0713..64df3ab4 100644 --- a/mautrix/client/api/modules/media_repository.py +++ b/mautrix/client/api/modules/media_repository.py @@ -141,7 +141,7 @@ async def upload_media( path = MediaPath.unstable["fi.mau.msc2246"].upload[server_name][media_id].complete if upload_url is not None: - task = self._upload_to_url(upload_url, path, data) + task = self._upload_to_url(upload_url, path, headers, data) else: task = self.api.request( method, path, content=data, headers=headers, query_params=query @@ -279,22 +279,29 @@ async def get_media_repo_config(self) -> MediaRepoConfig: except SerializerError as e: raise MatrixResponseError("Invalid MediaRepoConfig in response") from e - async def _upload_to_url(self, upload_url: str, post_upload_path: str, data: Any): + async def _upload_to_url( + self, + upload_url: str, + post_upload_path: str, + headers: dict[str, str], + data: bytes | bytearray | AsyncIterable[bytes], + ) -> None: retry_count = self.api.default_retry_count backoff = 4 while True: - upload_response = await self.api.session.request( - Method.PUT.name, upload_url, data=data - ) + self.log.debug("Uploading media to external URL %s", upload_url) + upload_response = await self.api.session.put(upload_url, data=data, headers=headers) if not upload_response.ok: if retry_count == 0: raise make_request_error( http_status=upload_response.status, text=await upload_response.text(), + errcode="COM.BEEPER.EXTERNAL_UPLOAD_ERROR", + message=None, ) self.log.warning( - f"non-ok http response from upload URL: PUT {upload_url} returned" - f" {upload_response.status}, retrying in {backoff} seconds" + f"Uploading media to external URL {upload_url} failed with HTTP " + f"{upload_response.status}, retrying in {backoff} seconds" ) await asyncio.sleep(backoff) backoff *= 2 From c90dc08f20535a11a2b0fc6f3aad32983ccce902 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 15 Sep 2022 16:52:22 +0300 Subject: [PATCH 212/456] Use ternary operator instead of or for deciding which data to dispatch --- mautrix/client/syncer.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/mautrix/client/syncer.py b/mautrix/client/syncer.py index f3392a4a..9f942665 100644 --- a/mautrix/client/syncer.py +++ b/mautrix/client/syncer.py @@ -258,7 +258,9 @@ async def run_internal_event( ) -> None: kwargs["source"] = SyncStream.INTERNAL tasks = self.dispatch_manual_event( - event_type, custom_type or kwargs, include_global_handlers=False + event_type, + custom_type if custom_type is not None else kwargs, + include_global_handlers=False, ) await asyncio.gather(*tasks) @@ -267,7 +269,9 @@ def dispatch_internal_event( ) -> list[asyncio.Task]: kwargs["source"] = SyncStream.INTERNAL return self.dispatch_manual_event( - event_type, custom_type or kwargs, include_global_handlers=False + event_type, + custom_type if custom_type is not None else kwargs, + include_global_handlers=False, ) def _try_deserialize(self, type: Type[T], data: JSON) -> T | GenericEvent: From ed3c4c0b195241c06971a409e8d15a13f4d2101d Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 15 Sep 2022 17:24:37 +0300 Subject: [PATCH 213/456] Set env_prefix before reading env --- mautrix/bridge/bridge.py | 11 +++++++---- mautrix/bridge/config.py | 6 +++++- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/mautrix/bridge/bridge.py b/mautrix/bridge/bridge.py index e7c3cde8..78f6250b 100644 --- a/mautrix/bridge/bridge.py +++ b/mautrix/bridge/bridge.py @@ -107,7 +107,9 @@ def preinit(self) -> None: def prepare(self) -> None: if self.config.env: - self.log.debug("Loaded config overrides from environment: %s", self.config.env.keys()) + self.log.debug( + "Loaded config overrides from environment: %s", list(self.config.env.keys()) + ) super().prepare() self.prepare_db() self.prepare_appservice() @@ -115,10 +117,11 @@ def prepare(self) -> None: def prepare_config(self) -> None: self.config = self.config_class( - self.args.config, self.args.registration, self.args.base_config + self.args.config, + self.args.registration, + self.args.base_config, + env_prefix=self.module.upper(), ) - if not self.config.env_prefix: - self.config.env_prefix = self.module.upper() if self.args.generate_registration: self.config._check_tokens = False self.load_and_update_config() diff --git a/mautrix/bridge/config.py b/mautrix/bridge/config.py index e9e51a49..012512a6 100644 --- a/mautrix/bridge/config.py +++ b/mautrix/bridge/config.py @@ -29,12 +29,16 @@ class BaseBridgeConfig(BaseFileConfig, BaseValidatableConfig, ABC): _check_tokens: bool env: dict[str, Any] - def __init__(self, path: str, registration_path: str, base_path: str) -> None: + def __init__( + self, path: str, registration_path: str, base_path: str, env_prefix: str | None = None + ) -> None: super().__init__(path, base_path) self.registration_path = registration_path self._registration = None self._check_tokens = True self.env = {} + if not self.env_prefix: + self.env_prefix = env_prefix if self.env_prefix: env_prefix = f"{self.env_prefix}_" for key, value in os.environ.items(): From 50792fdb6d39a3104e81297cff1e3d31585e73ad Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 15 Sep 2022 17:27:17 +0300 Subject: [PATCH 214/456] Bump version to 0.18.0 --- CHANGELOG.md | 2 +- mautrix/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ebba0eea..a0350232 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -## v0.18.0 (unreleased) +## v0.18.0 (2022-09-15) * **Breaking change *(util.async_db)*** Added checks to prevent calling `.start()` on a database multiple times. diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 50569673..81194a65 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.17.8" +__version__ = "0.18.0" __author__ = "Tulir Asokan " __all__ = [ "api", From ae7d8db3df77a29377d717efa6f2594b9d31ea1d Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 15 Sep 2022 21:46:23 +0300 Subject: [PATCH 215/456] Don't fail sending message if key claim fails --- mautrix/crypto/encrypt_megolm.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mautrix/crypto/encrypt_megolm.py b/mautrix/crypto/encrypt_megolm.py index fa15d846..9b37e486 100644 --- a/mautrix/crypto/encrypt_megolm.py +++ b/mautrix/crypto/encrypt_megolm.py @@ -221,7 +221,10 @@ async def _share_group_session(self, room_id: RoomID, users: List[UserID]) -> No if missing_sessions: self.log.debug(f"Creating missing outbound sessions {missing_sessions}") - await self._create_outbound_sessions(missing_sessions) + try: + await self._create_outbound_sessions(missing_sessions) + except Exception: + self.log.exception("Failed to create missing outbound sessions") for user_id, devices in missing_sessions.items(): for device_id, device in devices.items(): From 92abe92b1dd6b9176dc15dce277432b5a8ac3b72 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 15 Sep 2022 21:52:29 +0300 Subject: [PATCH 216/456] Bump version to 0.18.1 --- CHANGELOG.md | 5 +++++ mautrix/__init__.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a0350232..1bde0e34 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +## v0.18.1 (2022-09-15) + +* *(crypto)* Fixed error sharing megolm session if a single recipient device + has ran out of one-time keys. + ## v0.18.0 (2022-09-15) * **Breaking change *(util.async_db)*** Added checks to prevent calling diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 81194a65..5923a918 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.18.0" +__version__ = "0.18.1" __author__ = "Tulir Asokan " __all__ = [ "api", From 283c68a536a9b614dbd43cfdce8af91c54b4359b Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 20 Sep 2022 18:41:02 +0300 Subject: [PATCH 217/456] Add log and fix type hint --- mautrix/appservice/as_handler.py | 2 ++ mautrix/bridge/bridge.py | 11 ++--------- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/mautrix/appservice/as_handler.py b/mautrix/appservice/as_handler.py index 48c77217..67610125 100644 --- a/mautrix/appservice/as_handler.py +++ b/mautrix/appservice/as_handler.py @@ -228,6 +228,8 @@ async def _http_handle_transaction(self, request: web.Request) -> web.Response: except Exception: self.log.exception("Exception in transaction handler") output = None + finally: + self.log.debug(f"Finished handling transaction {transaction_id}") self.transactions.add(transaction_id) diff --git a/mautrix/bridge/bridge.py b/mautrix/bridge/bridge.py index 78f6250b..9a3f9115 100644 --- a/mautrix/bridge/bridge.py +++ b/mautrix/bridge/bridge.py @@ -17,14 +17,7 @@ from mautrix.client.state_store.asyncpg import PgStateStore as PgClientStateStore from mautrix.errors import MExclusive, MUnknownToken from mautrix.types import RoomID, UserID -from mautrix.util.async_db import ( - Database, - DatabaseException, - DatabaseNotOwned, - ForeignTablesFound, - UnsupportedDatabaseVersion, - UpgradeTable, -) +from mautrix.util.async_db import Database, DatabaseException, UpgradeTable from mautrix.util.bridge_state import BridgeState, BridgeStateEvent, GlobalBridgeState from mautrix.util.program import Program @@ -49,7 +42,7 @@ class Bridge(Program, ABC): matrix: br.BaseMatrixHandler repo_url: str markdown_version: str - manhole: br.manhole.ManholeState | None + manhole: br.commands.manhole.ManholeState | None def __init__( self, From 9e38b9e3a77ac77948b875a0c08040a5258070b2 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 20 Sep 2022 18:41:10 +0300 Subject: [PATCH 218/456] Fix key requests on AS mode e2be --- mautrix/crypto/machine.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mautrix/crypto/machine.py b/mautrix/crypto/machine.py index 14e9e95f..a1fd4e16 100644 --- a/mautrix/crypto/machine.py +++ b/mautrix/crypto/machine.py @@ -119,6 +119,8 @@ async def handle_as_to_device_event(self, evt: ASToDeviceEvent) -> None: return if evt.type == EventType.TO_DEVICE_ENCRYPTED: await self.handle_to_device_event(evt) + elif evt.type == EventType.ROOM_KEY_REQUEST: + await self.handle_room_key_request(evt) else: self.log.debug(f"Got unknown to-device event {evt.type} from {evt.sender}") From fe17bad0f3b67922d3a20158e1060e626a3f17ca Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 24 Sep 2022 13:43:55 +0300 Subject: [PATCH 219/456] Work around bugs in some servers Closes #118 --- mautrix/appservice/as_handler.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/mautrix/appservice/as_handler.py b/mautrix/appservice/as_handler.py index 67610125..e744ce43 100644 --- a/mautrix/appservice/as_handler.py +++ b/mautrix/appservice/as_handler.py @@ -237,6 +237,11 @@ async def _http_handle_transaction(self, request: web.Request) -> web.Response: @staticmethod def _fix_prev_content(raw_event: JSON) -> None: + try: + if raw_event["unsigned"] is None: + del raw_event["unsigned"] + except KeyError: + pass try: raw_event["unsigned"]["prev_content"] except KeyError: From e4b089ebc7eda620ee7c878c4ff542ded66aaac7 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 24 Sep 2022 13:47:02 +0300 Subject: [PATCH 220/456] Bump version to 0.18.2 --- CHANGELOG.md | 7 +++++++ mautrix/__init__.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1bde0e34..a8e8d552 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## v0.18.2 (2022-09-24) + +* *(crypto)* Fixed handling key requests when using appservice-mode (MSC2409) + encryption. +* *(appservice)* Added workaround for dumb servers that send `"unsigned": null` + in events. + ## v0.18.1 (2022-09-15) * *(crypto)* Fixed error sharing megolm session if a single recipient device diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 5923a918..af5c3a2d 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.18.1" +__version__ = "0.18.2" __author__ = "Tulir Asokan " __all__ = [ "api", From 61d8e6f0ac9f50685ebb2032d07f3e2eb541973e Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 30 Sep 2022 11:01:35 +0300 Subject: [PATCH 221/456] Fix mistake in noop exception handler --- mautrix/util/async_db/connection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/util/async_db/connection.py b/mautrix/util/async_db/connection.py index ec4c9957..1d27c9f0 100644 --- a/mautrix/util/async_db/connection.py +++ b/mautrix/util/async_db/connection.py @@ -43,7 +43,7 @@ async def wrapper(self: LoggingConnection, arg: str, *args: Any, **kwargs: str) return wrapper -async def handle_exception_noop() -> None: +async def handle_exception_noop(_: Exception) -> None: pass From 398f9af1e31d86f6f03cfd8bf35f6a2d876297c3 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 30 Sep 2022 11:08:40 +0300 Subject: [PATCH 222/456] Catch unique errors when inserting megolm sessions --- mautrix/crypto/store/asyncpg/store.py | 30 +++++++++++++++++---------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/mautrix/crypto/store/asyncpg/store.py b/mautrix/crypto/store/asyncpg/store.py index 9af2d075..cde6ad71 100644 --- a/mautrix/crypto/store/asyncpg/store.py +++ b/mautrix/crypto/store/asyncpg/store.py @@ -8,6 +8,8 @@ from collections import defaultdict from datetime import timedelta +from asyncpg import UniqueViolationError + from mautrix.client.state_store import SyncStore from mautrix.client.state_store.asyncpg import PgStateStore from mautrix.types import ( @@ -33,13 +35,16 @@ from .upgrade import upgrade_table try: - from sqlite3 import sqlite_version_info as sqlite_version + from sqlite3 import IntegrityError, sqlite_version_info as sqlite_version from aiosqlite import Cursor except ImportError: Cursor = None sqlite_version = (0, 0, 0) + class IntegrityError(Exception): + pass + class PgCryptoStateStore(PgStateStore, StateStore): """ @@ -231,16 +236,19 @@ async def put_group_session( session_id, sender_key, signing_key, room_id, session, forwarding_chains, account_id ) VALUES ($1, $2, $3, $4, $5, $6, $7) """ - await self.db.execute( - q, - session_id, - sender_key, - session.signing_key, - room_id, - pickle, - forwarding_chains, - self.account_id, - ) + try: + await self.db.execute( + q, + session_id, + sender_key, + session.signing_key, + room_id, + pickle, + forwarding_chains, + self.account_id, + ) + except (IntegrityError, UniqueViolationError): + self.log.exception(f"Failed to insert megolm session {session_id}") async def get_group_session( self, room_id: RoomID, session_id: SessionID From 97ad6fe1f8b7782056a6772ea0e2c31c181b3d71 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 1 Oct 2022 17:11:31 +0300 Subject: [PATCH 223/456] Add black cat override in variation selector utility --- mautrix/util/variation_selector.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/mautrix/util/variation_selector.py b/mautrix/util/variation_selector.py index a04c1ade..498f1abe 100644 --- a/mautrix/util/variation_selector.py +++ b/mautrix/util/variation_selector.py @@ -61,6 +61,10 @@ async def fetch_data() -> dict[str, str]: ) SKIN_TONE_MODIFIERS = ("\U0001F3FB", "\U0001F3FC", "\U0001F3FD", "\U0001F3FE", "\U0001F3FF") SKIN_TONE_REPLACEMENTS = {f"{VARIATION_SELECTOR_16}{mod}": mod for mod in SKIN_TONE_MODIFIERS} +VARIATION_SELECTOR_REPLACEMENTS = { + **SKIN_TONE_REPLACEMENTS, + "\U0001F408\ufe0f\u200d\u2b1b\ufe0f": "\U0001F408\u200d\u2b1b", +} def add(val: str) -> str: @@ -88,7 +92,7 @@ def add(val: str) -> str: The string with variation selectors added. """ added = remove(val).translate(ADD_VARIATION_TRANSLATION) - for invalid_selector, replacement in SKIN_TONE_REPLACEMENTS.items(): + for invalid_selector, replacement in VARIATION_SELECTOR_REPLACEMENTS.items(): added = added.replace(invalid_selector, replacement) return added From 5df16b0c7b35520148abfad499cdfd9c9e7f76c8 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 1 Oct 2022 17:14:21 +0300 Subject: [PATCH 224/456] Catch all errors in external URL uploads --- mautrix/client/api/modules/media_repository.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/mautrix/client/api/modules/media_repository.py b/mautrix/client/api/modules/media_repository.py index 64df3ab4..dec2ecd3 100644 --- a/mautrix/client/api/modules/media_repository.py +++ b/mautrix/client/api/modules/media_repository.py @@ -290,18 +290,23 @@ async def _upload_to_url( backoff = 4 while True: self.log.debug("Uploading media to external URL %s", upload_url) - upload_response = await self.api.session.put(upload_url, data=data, headers=headers) - if not upload_response.ok: + upload_response = None + try: + upload_response = await self.api.session.put( + upload_url, data=data, headers=headers + ) + upload_response.raise_for_status() + except Exception as e: if retry_count == 0: raise make_request_error( - http_status=upload_response.status, - text=await upload_response.text(), + http_status=upload_response.status if upload_response else -1, + text=(await upload_response.text()) if upload_response else "", errcode="COM.BEEPER.EXTERNAL_UPLOAD_ERROR", message=None, ) self.log.warning( - f"Uploading media to external URL {upload_url} failed with HTTP " - f"{upload_response.status}, retrying in {backoff} seconds" + f"Uploading media to external URL {upload_url} failed: {e}, " + f"retrying in {backoff} seconds", ) await asyncio.sleep(backoff) backoff *= 2 From 9867ff4eb4d05bc5b3e38da2cad60b2a81c73538 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 10 Oct 2022 23:46:10 +0300 Subject: [PATCH 225/456] Handle missing signatures in verify_signature_json --- mautrix/crypto/base.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mautrix/crypto/base.py b/mautrix/crypto/base.py index d7b0f9b9..4d12e6cf 100644 --- a/mautrix/crypto/base.py +++ b/mautrix/crypto/base.py @@ -87,7 +87,10 @@ def verify_signature_json( data_copy.pop("unsigned", None) signatures = data_copy.pop("signatures") key_id = str(KeyID(EncryptionKeyAlgorithm.ED25519, key_name)) - signature = signatures[user_id][key_id] + try: + signature = signatures[user_id][key_id] + except KeyError: + return False signed_data = canonical_json(data_copy) try: olm.ed25519_verify(key, signed_data, signature) From 5aa8d91755a3f5b42ca1a7b42fed3bcc4e3c1d3d Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 11 Oct 2022 15:05:46 +0300 Subject: [PATCH 226/456] Add a no-op mode to SimpleLock --- mautrix/util/simple_lock.py | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/mautrix/util/simple_lock.py b/mautrix/util/simple_lock.py index ecfa1e1c..ad979aec 100644 --- a/mautrix/util/simple_lock.py +++ b/mautrix/util/simple_lock.py @@ -13,31 +13,41 @@ class SimpleLock: _event: asyncio.Event log: logging.Logger | None message: str | None - - def __init__(self, message: str | None = None, log: logging.Logger | None = None) -> None: - self._event = asyncio.Event() - self._event.set() + noop_mode: bool + + def __init__( + self, + message: str | None = None, + log: logging.Logger | None = None, + noop_mode: bool = False, + ) -> None: + self.noop_mode = noop_mode + if not noop_mode: + self._event = asyncio.Event() + self._event.set() self.log = log self.message = message def __enter__(self) -> None: - self._event.clear() + if not self.noop_mode: + self._event.clear() async def __aenter__(self) -> None: self.__enter__() def __exit__(self, exc_type, exc_val, exc_tb) -> None: - self._event.set() + if not self.noop_mode: + self._event.set() def __aexit__(self, exc_type, exc_val, exc_tb) -> None: self.__exit__(exc_type, exc_val, exc_tb) @property def locked(self) -> bool: - return not self._event.is_set() + return not self.noop_mode and not self._event.is_set() async def wait(self, task: str | None = None) -> None: - if not self._event.is_set(): + if not self.noop_mode and not self._event.is_set(): if self.log and self.message: self.log.debug(self.message, task) await self._event.wait() From bc7a01c77ca302e10b3e875f6175b59827cb4ab6 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 11 Oct 2022 15:21:29 +0300 Subject: [PATCH 227/456] Parse homeserver.software into enum --- mautrix/bridge/__init__.py | 3 ++- mautrix/bridge/bridge.py | 21 +++++++++++++++++++++ mautrix/bridge/matrix.py | 8 ++++++++ mautrix/bridge/user.py | 2 +- 4 files changed, 32 insertions(+), 2 deletions(-) diff --git a/mautrix/bridge/__init__.py b/mautrix/bridge/__init__.py index e628eeff..ff6c0f7b 100644 --- a/mautrix/bridge/__init__.py +++ b/mautrix/bridge/__init__.py @@ -1,5 +1,5 @@ from ..util.async_getter_lock import async_getter_lock -from .bridge import Bridge +from .bridge import Bridge, HomeserverSoftware from .config import BaseBridgeConfig from .custom_puppet import ( AutologinError, @@ -21,6 +21,7 @@ __all__ = [ "async_getter_lock", "Bridge", + "HomeserverSoftware", "BaseBridgeConfig", "AutologinError", "CustomPuppetError", diff --git a/mautrix/bridge/bridge.py b/mautrix/bridge/bridge.py index 9a3f9115..7005d775 100644 --- a/mautrix/bridge/bridge.py +++ b/mautrix/bridge/bridge.py @@ -7,6 +7,7 @@ from typing import Any from abc import ABC, abstractmethod +from enum import Enum import sys from aiohttp import web @@ -30,6 +31,20 @@ uvloop = None +class HomeserverSoftware(Enum): + STANDARD = "standard" + ASMUX = "asmux" + HUNGRY = "hungry" + + @property + def is_hungry(self) -> bool: + return self == self.HUNGRY + + @property + def is_asmux(self) -> bool: + return self == self.ASMUX + + class Bridge(Program, ABC): db: Database az: AppService @@ -43,6 +58,7 @@ class Bridge(Program, ABC): repo_url: str markdown_version: str manhole: br.commands.manhole.ManholeState | None + homeserver_software: HomeserverSoftware def __init__( self, @@ -104,6 +120,11 @@ def prepare(self) -> None: "Loaded config overrides from environment: %s", list(self.config.env.keys()) ) super().prepare() + try: + self.homeserver_software = HomeserverSoftware(self.config["homeserver.software"]) + except Exception: + self.log.fatal("Invalid value for homeserver.software in config") + sys.exit(11) self.prepare_db() self.prepare_appservice() self.prepare_bridge() diff --git a/mautrix/bridge/matrix.py b/mautrix/bridge/matrix.py index cf7b962a..6cd6b1c2 100644 --- a/mautrix/bridge/matrix.py +++ b/mautrix/bridge/matrix.py @@ -207,6 +207,14 @@ async def check_versions(self) -> None: self.minimum_spec_version, ) sys.exit(18) + if self.bridge.homeserver_software.is_hungry and not self.versions.supports( + "com.beeper.hungry" + ): + self.log.fatal( + "The config claims the homeserver is hungryserv, " + "but the /versions response didn't confirm it" + ) + sys.exit(18) async def wait_for_connection(self) -> None: self.log.info("Ensuring connectivity to homeserver") diff --git a/mautrix/bridge/user.py b/mautrix/bridge/user.py index cbb63141..bea6bdf7 100644 --- a/mautrix/bridge/user.py +++ b/mautrix/bridge/user.py @@ -128,7 +128,7 @@ async def update_direct_chats(self, dms: dict[UserID, list[RoomID]] | None = Non self.log.debug("Updating m.direct list on homeserver") replace = dms is None dms = dms or await self.get_direct_chats() - if self.bridge.config.get("homeserver.software", "standard") == "asmux": + if self.bridge.homeserver_software.is_asmux: # This uses a secret endpoint for atomically updating the DM list await puppet.intent.api.request( Method.PUT if replace else Method.PATCH, From d8e73a05e7ae679531691b222086a683665b03a8 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 11 Oct 2022 15:49:33 +0300 Subject: [PATCH 228/456] Bump version to 0.18.3 --- CHANGELOG.md | 9 +++++++++ mautrix/__init__.py | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a8e8d552..22bdc664 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,12 @@ +## v0.18.3 (2022-10-11) + +* *(util.async_db)* Fixed mistake in default no-op database error handler + causing the wrong exception to be raised. +* *(crypto.store.asyncpg)* Updated `put_group_session` to catch unique key + errors and log instead of raising. +* *(client.api)* Updated [MSC3870] support to catch and retry on all + connection errors instead of only non-200 status codes when uploading. + ## v0.18.2 (2022-09-24) * *(crypto)* Fixed handling key requests when using appservice-mode (MSC2409) diff --git a/mautrix/__init__.py b/mautrix/__init__.py index af5c3a2d..bdfb00d4 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.18.2" +__version__ = "0.18.3" __author__ = "Tulir Asokan " __all__ = [ "api", From c5d8540c50a6ad1800a2e981792465e237db73d3 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 11 Oct 2022 16:21:12 +0300 Subject: [PATCH 229/456] Allow passing custom data to /createRoom --- mautrix/client/api/rooms.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/mautrix/client/api/rooms.py b/mautrix/client/api/rooms.py index eb6d352d..61ea16d7 100644 --- a/mautrix/client/api/rooms.py +++ b/mautrix/client/api/rooms.py @@ -68,6 +68,8 @@ async def create_room( room_version: str = None, creation_content: RoomCreateStateEventContent | dict[str, JSON] | None = None, power_level_override: PowerLevelStateEventContent | dict[str, JSON] | None = None, + beeper_auto_join_invites: bool = False, + custom_request_fields: dict[str, Any] | None = None, ) -> RoomID: """ Create a new room with various configuration options. @@ -111,6 +113,10 @@ async def create_room( power_level_override: The power level content to override in the default power level event. This object is applied on top of the generated ``m.room.power_levels`` event content prior to it being sent to the room. Defaults to overriding nothing. + beeper_auto_join_invites: A Beeper-specific extension which auto-joins all members in + the invite array instead of sending invites. + custom_request_fields: Additional fields to put in the top-level /createRoom content. + Non-custom fields take precedence over fields here. Returns: The ID of the newly created room. @@ -124,6 +130,7 @@ async def create_room( .. _m.room.member: https://spec.matrix.org/v1.1/client-server-api/#mroommember """ content = { + **(custom_request_fields or {}), "visibility": visibility.value, "is_direct": is_direct, "preset": preset.value, @@ -132,6 +139,8 @@ async def create_room( content["room_alias_name"] = alias_localpart if invitees: content["invite"] = invitees + if beeper_auto_join_invites: + content["com.beeper.auto_join_invites"] = True if name: content["name"] = name if topic: From 98172d7d7c8b78234eb4a5ab6062196e214ff9c7 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 12 Oct 2022 00:35:32 +0300 Subject: [PATCH 230/456] Make all fields optional in QueryKeysResponse --- mautrix/types/crypto.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mautrix/types/crypto.py b/mautrix/types/crypto.py index 56fd3fb8..821599de 100644 --- a/mautrix/types/crypto.py +++ b/mautrix/types/crypto.py @@ -79,10 +79,10 @@ def first_key_with_algorithm(self, alg: EncryptionKeyAlgorithm) -> Optional[Sign @dataclass class QueryKeysResponse(SerializableAttrs): - device_keys: Dict[UserID, Dict[DeviceID, DeviceKeys]] - master_keys: Dict[UserID, CrossSigningKeys] - self_signing_keys: Dict[UserID, CrossSigningKeys] - user_signing_keys: Dict[UserID, CrossSigningKeys] + device_keys: Dict[UserID, Dict[DeviceID, DeviceKeys]] = field(factory=lambda: {}) + master_keys: Dict[UserID, CrossSigningKeys] = field(factory=lambda: {}) + self_signing_keys: Dict[UserID, CrossSigningKeys] = field(factory=lambda: {}) + user_signing_keys: Dict[UserID, CrossSigningKeys] = field(factory=lambda: {}) failures: Dict[str, Any] = field(factory=lambda: {}) From 2322bf760ca74208f77a1f93dfb4ae6e6a134fe8 Mon Sep 17 00:00:00 2001 From: Nick Barrett Date: Wed, 12 Oct 2022 08:24:09 +0100 Subject: [PATCH 231/456] Pass filename in query to upload complete endpoint --- mautrix/client/api/modules/media_repository.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mautrix/client/api/modules/media_repository.py b/mautrix/client/api/modules/media_repository.py index dec2ecd3..b1de6751 100644 --- a/mautrix/client/api/modules/media_repository.py +++ b/mautrix/client/api/modules/media_repository.py @@ -141,7 +141,7 @@ async def upload_media( path = MediaPath.unstable["fi.mau.msc2246"].upload[server_name][media_id].complete if upload_url is not None: - task = self._upload_to_url(upload_url, path, headers, data) + task = self._upload_to_url(upload_url, path, headers, data, post_upload_query=query) else: task = self.api.request( method, path, content=data, headers=headers, query_params=query @@ -285,6 +285,7 @@ async def _upload_to_url( post_upload_path: str, headers: dict[str, str], data: bytes | bytearray | AsyncIterable[bytes], + post_upload_query: dict[str, str], ) -> None: retry_count = self.api.default_retry_count backoff = 4 @@ -314,4 +315,4 @@ async def _upload_to_url( else: break - await self.api.request(Method.POST, post_upload_path) + await self.api.request(Method.POST, post_upload_path, query_params=post_upload_query) From 9ea23d15f5d9ac038f80711ff829f2edd191dbfd Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 12 Oct 2022 17:25:01 +0300 Subject: [PATCH 232/456] Fix edge case when splitting entity strings. Fixes mautrix/telegram#850 --- mautrix/util/formatter/entity_string.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mautrix/util/formatter/entity_string.py b/mautrix/util/formatter/entity_string.py index af1a3c81..f40beb0b 100644 --- a/mautrix/util/formatter/entity_string.py +++ b/mautrix/util/formatter/entity_string.py @@ -39,6 +39,8 @@ def adjust_offset(self, offset: int, max_length: int = -1) -> SemiAbstractEntity entity.offset += offset if entity.offset < 0: entity.length += entity.offset + if entity.length < 0: + return None entity.offset = 0 elif entity.offset > max_length > -1: return None From 136d4893a96983041ffa0bdc762e3f1a692613e9 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 13 Oct 2022 10:53:41 +0300 Subject: [PATCH 233/456] Clear reply metadata when marking as edit --- mautrix/types/event/message.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/mautrix/types/event/message.py b/mautrix/types/event/message.py index 45c34ce6..eab4ba95 100644 --- a/mautrix/types/event/message.py +++ b/mautrix/types/event/message.py @@ -119,6 +119,12 @@ def set_thread_parent( def set_edit(self, edits: Union[EventID, "MessageEvent"]) -> None: self.relates_to.rel_type = RelationType.REPLACE self.relates_to.event_id = edits if isinstance(edits, str) else edits.event_id + # Library consumers may create message content by setting a reply first, + # then later marking it as an edit. As edits can't change the reply, just remove + # the reply metadata when marking as a reply. + if self.relates_to.in_reply_to: + self.relates_to.in_reply_to = None + self.relates_to.is_falling_back = None def serialize(self) -> JSON: data = SerializableAttrs.serialize(self) From 7c6fb062efcebec932c3c23fed3e395f86979bd7 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 13 Oct 2022 15:31:30 +0300 Subject: [PATCH 234/456] Bump version to 0.18.4 --- CHANGELOG.md | 11 +++++++++++ mautrix/__init__.py | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 22bdc664..66b1fafe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,14 @@ +## v0.18.4 (2022-10-13) + +* *(client.api)* Added option to pass custom data to `/createRoom` to enable + using custom fields and testing MSCs without changing the library. +* *(client.api)* Updated [MSC3870] support to send file name in upload complete + call. +* *(types)* Changed `set_edit` to clear reply metadata as edits can't change + the reply status. +* *(util.formatter)* Fixed edge case causing negative entity lengths when + splitting entity strings. + ## v0.18.3 (2022-10-11) * *(util.async_db)* Fixed mistake in default no-op database error handler diff --git a/mautrix/__init__.py b/mautrix/__init__.py index bdfb00d4..47857202 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.18.3" +__version__ = "0.18.4" __author__ = "Tulir Asokan " __all__ = [ "api", From 8a4d22eeb30fac53fdf894d6fb4b8f0f4be49683 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 18 Oct 2022 12:26:09 +0300 Subject: [PATCH 235/456] Catch errors in AS e2ee things instead of failing transaction --- mautrix/appservice/as_handler.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/mautrix/appservice/as_handler.py b/mautrix/appservice/as_handler.py index e744ce43..88715100 100644 --- a/mautrix/appservice/as_handler.py +++ b/mautrix/appservice/as_handler.py @@ -267,11 +267,20 @@ async def handle_transaction( except SerializerError: self.log.exception("Failed to deserialize to-device event %s", raw_td) else: - await self.to_device_handler(td) + try: + await self.to_device_handler(td) + except: + self.log.exception("Exception in Matrix to-device event handler") if device_lists and self.device_list_handler: - await self.device_list_handler(device_lists) + try: + await self.device_list_handler(device_lists) + except Exception: + self.log.exception("Exception in Matrix device list change handler") if otk_counts and self.otk_handler: - await self.otk_handler(otk_counts) + try: + await self.otk_handler(otk_counts) + except Exception: + self.log.exception("Exception in Matrix OTK count handler") for raw_edu in ephemeral or []: try: edu = EphemeralEvent.deserialize(raw_edu) @@ -304,6 +313,7 @@ async def try_handle(handler_func: HandlerFunc): self.log.exception("Exception in Matrix event handler") for handler in self.event_handlers: + # TODO add option to handle events synchronously asyncio.create_task(try_handle(handler)) def matrix_event_handler(self, func: HandlerFunc) -> HandlerFunc: From 9c03902f500892262ea35b6c0f6a3a05239fb6c4 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 20 Oct 2022 15:18:58 +0300 Subject: [PATCH 236/456] Bump version to 0.18.5 --- CHANGELOG.md | 6 ++++++ mautrix/__init__.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 66b1fafe..b30a62b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## v0.18.5 (2022-10-20) + +* *(appservice)* Added try blocks around [MSC3202] handler functions to log + errors instead of failing the entire transaction. This matches the behavior + of errors in normal appservice event handlers. + ## v0.18.4 (2022-10-13) * *(client.api)* Added option to pass custom data to `/createRoom` to enable diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 47857202..47b1a40c 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.18.4" +__version__ = "0.18.5" __author__ = "Tulir Asokan " __all__ = [ "api", From 8691786ffc711ad5323189695270b0da0fa7a160 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 21 Oct 2022 16:42:52 +0300 Subject: [PATCH 237/456] Add default HTML converter for horizontal lines --- mautrix/util/formatter/parser.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/mautrix/util/formatter/parser.py b/mautrix/util/formatter/parser.py index 76c3d270..dcff1eb8 100644 --- a/mautrix/util/formatter/parser.py +++ b/mautrix/util/formatter/parser.py @@ -106,6 +106,9 @@ async def blockquote_to_fstring(self, node: HTMLNode, ctx: RecursionContext) -> msg = await self.tag_aware_parse_node(node, ctx) return msg.format(self.e.BLOCKQUOTE) + async def hr_to_fstring(self, node: HTMLNode, ctx: RecursionContext) -> T: + return self.fs("---") + async def header_to_fstring(self, node: HTMLNode, ctx: RecursionContext) -> T: children = await self.node_to_fstrings(node, ctx) length = int(node.tag[1]) @@ -194,6 +197,8 @@ async def node_to_fstring(self, node: HTMLNode, ctx: RecursionContext) -> T: return self.fs("") elif node.tag == "blockquote": return await self.blockquote_to_fstring(node, ctx) + elif node.tag == "hr": + return await self.hr_to_fstring(node, ctx) elif node.tag == "ol": return await self.list_to_fstring(node, ctx) elif node.tag == "ul": From 1de10f069f14dff5bca422f874dfa3c5182184fd Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 24 Oct 2022 22:00:19 +0300 Subject: [PATCH 238/456] Bump version to 0.18.6 --- CHANGELOG.md | 5 +++++ mautrix/__init__.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b30a62b8..e0f40acb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +## v0.18.6 (2022-10-24) + +* *(util.formatter)* Added conversion method for `
` tag and defaulted to + converting back to `---`. + ## v0.18.5 (2022-10-20) * *(appservice)* Added try blocks around [MSC3202] handler functions to log diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 47b1a40c..bb37c6b4 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.18.5" +__version__ = "0.18.6" __author__ = "Tulir Asokan " __all__ = [ "api", From 923582caf5471971616a1735b6c0f5f8424a9c35 Mon Sep 17 00:00:00 2001 From: Andrew Ferrazzutti Date: Fri, 28 Oct 2022 17:10:51 -0400 Subject: [PATCH 239/456] Check AppService fields for None on stop This prevents getting an AttributeError when AppService.stop is called before AppService.start --- mautrix/appservice/appservice.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/mautrix/appservice/appservice.py b/mautrix/appservice/appservice.py index 4c9bf471..cceacf56 100644 --- a/mautrix/appservice/appservice.py +++ b/mautrix/appservice/appservice.py @@ -56,7 +56,9 @@ class AppService(AppServiceServerMixin): loop: asyncio.AbstractEventLoop log: TraceLogger app: web.Application - runner: web.AppRunner + runner: web.AppRunner | None + + _http_session: aiohttp.ClientSession | None def __init__( self, @@ -178,10 +180,12 @@ async def start(self, host: str = "127.0.0.1", port: int = 8080) -> None: async def stop(self) -> None: self.log.debug("Stopping appservice web server") - await self.runner.cleanup() + if self.runner: + await self.runner.cleanup() self._intent = None - await self._http_session.close() - self._http_session = None + if self._http_session: + await self._http_session.close() + self._http_session = None await self.state_store.close() async def _liveness_probe(self, _: web.Request) -> web.Response: From 837afc51c7d293ca2120d74fcf815e81c1c1aa95 Mon Sep 17 00:00:00 2001 From: Andrew Ferrazzutti Date: Fri, 28 Oct 2022 17:11:59 -0400 Subject: [PATCH 240/456] Add None to some AppService field typehints --- mautrix/appservice/appservice.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mautrix/appservice/appservice.py b/mautrix/appservice/appservice.py index cceacf56..551eab5c 100644 --- a/mautrix/appservice/appservice.py +++ b/mautrix/appservice/appservice.py @@ -36,8 +36,8 @@ class AppService(AppServiceServerMixin): domain: str id: str verify_ssl: bool - tls_cert: str - tls_key: str + tls_cert: str | None + tls_key: str | None as_token: str hs_token: str bot_mxid: UserID @@ -59,6 +59,7 @@ class AppService(AppServiceServerMixin): runner: web.AppRunner | None _http_session: aiohttp.ClientSession | None + _intent: IntentAPI | None def __init__( self, From d7475a29b9c9f6d5cfa21cc9e96ab44ae99df410 Mon Sep 17 00:00:00 2001 From: Sumner Evans Date: Mon, 7 Nov 2022 15:46:29 -0700 Subject: [PATCH 241/456] intent/batch send: add option to set com.beeper.mark_read_by Signed-off-by: Sumner Evans --- mautrix/appservice/api/intent.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mautrix/appservice/api/intent.py b/mautrix/appservice/api/intent.py index 2d91f8db..e5d17103 100644 --- a/mautrix/appservice/api/intent.py +++ b/mautrix/appservice/api/intent.py @@ -501,6 +501,7 @@ async def batch_send( events: Iterable[BatchSendEvent], state_events_at_start: Iterable[BatchSendStateEvent] = (), beeper_new_messages: bool = False, + beeper_mark_read_by: UserID | None = None, ) -> BatchSendResponse: """ Send a batch of historical events into a room. See `MSC2716`_ for more info. @@ -530,6 +531,8 @@ async def batch_send( query["batch_id"] = batch_id if beeper_new_messages: query["com.beeper.new_messages"] = "true" + if beeper_mark_read_by: + query["com.beeper.mark_read_by"] = beeper_mark_read_by resp = await self.api.request( Method.POST, path, From 873d8d4929c4bb274a91b089e5f94340a61e1b1d Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 8 Nov 2022 18:17:29 +0200 Subject: [PATCH 242/456] Bump version to 0.18.7 --- CHANGELOG.md | 2 ++ mautrix/__init__.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e0f40acb..c88a0a9a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,5 @@ +## v0.18.7 (2022-11-08) + ## v0.18.6 (2022-10-24) * *(util.formatter)* Added conversion method for `
` tag and defaulted to diff --git a/mautrix/__init__.py b/mautrix/__init__.py index bb37c6b4..6a515f62 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.18.6" +__version__ = "0.18.7" __author__ = "Tulir Asokan " __all__ = [ "api", From b2ad957aab76f4abe9e92afbd6e4321b7e0ae657 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 11 Nov 2022 19:14:19 +0200 Subject: [PATCH 243/456] Give a logger to PgCryptoStore --- mautrix/crypto/store/asyncpg/store.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mautrix/crypto/store/asyncpg/store.py b/mautrix/crypto/store/asyncpg/store.py index cde6ad71..8609e12d 100644 --- a/mautrix/crypto/store/asyncpg/store.py +++ b/mautrix/crypto/store/asyncpg/store.py @@ -70,6 +70,7 @@ def __init__(self, account_id: str, pickle_key: str, db: Database) -> None: self.db = db self.account_id = account_id self.pickle_key = pickle_key + self.log = db.log self._sync_token = None self._device_id = DeviceID("") From 8684e2444b1612ead93dd14fe08c5948efda86bb Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 18 Nov 2022 17:11:26 +0200 Subject: [PATCH 244/456] Update state store after creating room --- mautrix/client/api/rooms.py | 4 ++-- mautrix/client/store_updater.py | 22 ++++++++++++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/mautrix/client/api/rooms.py b/mautrix/client/api/rooms.py index 61ea16d7..9f935430 100644 --- a/mautrix/client/api/rooms.py +++ b/mautrix/client/api/rooms.py @@ -5,7 +5,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import annotations -from typing import Any, Awaitable, Callable, Iterable +from typing import Any, Awaitable, Callable import asyncio from multidict import CIMultiDict @@ -64,7 +64,7 @@ async def create_room( topic: str | None = None, is_direct: bool = False, invitees: list[UserID] | None = None, - initial_state: Iterable[StateEvent | StrippedStateEvent | dict[str, JSON]] | None = None, + initial_state: list[StateEvent | StrippedStateEvent | dict[str, JSON]] | None = None, room_version: str = None, creation_content: RoomCreateStateEventContent | dict[str, JSON] | None = None, power_level_override: PowerLevelStateEventContent | dict[str, JSON] | None = None, diff --git a/mautrix/client/store_updater.py b/mautrix/client/store_updater.py index 929b378d..a5530bde 100644 --- a/mautrix/client/store_updater.py +++ b/mautrix/client/store_updater.py @@ -149,6 +149,28 @@ async def get_state(self, room_id: RoomID) -> list[StateEvent]: ) return state + async def create_room(self, *args, **kwargs) -> RoomID: + room_id = await super().create_room(*args, **kwargs) + if self.state_store: + invitee_membership = Membership.INVITE + if kwargs.get("beeper_auto_join_invites"): + invitee_membership = Membership.JOIN + for user_id in kwargs.get("invitees", []): + await self.state_store.set_membership(room_id, user_id, invitee_membership) + for evt in kwargs.get("initial_state", []): + await self.state_store.update_state( + StateEvent( + type=EventType.find(evt["type"], t_class=EventType.Class.STATE), + room_id=room_id, + event_id=EventID("$fake-create-id"), + sender=self.mxid, + state_key=evt.get("state_key", ""), + timestamp=0, + content=evt["content"], + ) + ) + return room_id + async def send_state_event( self, room_id: RoomID, From effbf5f2db91cab7390813e692c4a8580d66e06c Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 18 Nov 2022 18:34:33 +0200 Subject: [PATCH 245/456] Allow passing event ID in batch sends --- mautrix/types/event/batch.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/mautrix/types/event/batch.py b/mautrix/types/event/batch.py index df45b742..48e2d661 100644 --- a/mautrix/types/event/batch.py +++ b/mautrix/types/event/batch.py @@ -3,26 +3,28 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from typing import Any +from typing import Any, Optional from attr import dataclass import attr -from ..primitive import UserID +from ..primitive import UserID, EventID from ..util import SerializableAttrs from .base import BaseEvent -@dataclass +@dataclass(kw_only=True) class BatchSendEvent(BaseEvent, SerializableAttrs): """Base event class for events sent via a batch send request.""" sender: UserID timestamp: int = attr.ib(metadata={"json": "origin_server_ts"}) content: Any + # N.B. Overriding event IDs is not allowed in standard room versions + event_id: Optional[EventID] = None -@dataclass +@dataclass(kw_only=True) class BatchSendStateEvent(BatchSendEvent, SerializableAttrs): """ State events to be used as initial state events on batch send events. These never need to be From 666cc7270435c6282ef24538d14b095fd1f66549 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 18 Nov 2022 18:56:22 +0200 Subject: [PATCH 246/456] Fix import order --- mautrix/types/event/batch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/types/event/batch.py b/mautrix/types/event/batch.py index 48e2d661..b08ad21e 100644 --- a/mautrix/types/event/batch.py +++ b/mautrix/types/event/batch.py @@ -8,7 +8,7 @@ from attr import dataclass import attr -from ..primitive import UserID, EventID +from ..primitive import EventID, UserID from ..util import SerializableAttrs from .base import BaseEvent From af063d4b7c63f06a0560018af194dc612494cd95 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 18 Nov 2022 19:23:26 +0200 Subject: [PATCH 247/456] Bump version to 0.18.8 --- CHANGELOG.md | 8 ++++++++ mautrix/__init__.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c88a0a9a..dc547c82 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,11 @@ +## v0.18.8 (2022-11-18) + +* *(crypto.store.asyncpg)* Fixed bug causing `put_group_session` to fail when + trying to log unique key errors. +* *(client)* Added wrapper for `create_room` to update the state store with + initial state and invites (applies to anything extending `StoreUpdatingAPI`, + such as the high-level `Client` and appservice `IntentAPI` classes). + ## v0.18.7 (2022-11-08) ## v0.18.6 (2022-10-24) diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 6a515f62..b66f6369 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.18.7" +__version__ = "0.18.8" __author__ = "Tulir Asokan " __all__ = [ "api", From 4d79e1c71de12ccd493be8df4093650d9c89a492 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 30 Nov 2022 15:48:38 +0200 Subject: [PATCH 248/456] Enable foreign keys and WAL mode by default on SQLite --- mautrix/util/async_db/aiosqlite.py | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/mautrix/util/async_db/aiosqlite.py b/mautrix/util/async_db/aiosqlite.py index 444aaef4..9b7fa16a 100644 --- a/mautrix/util/async_db/aiosqlite.py +++ b/mautrix/util/async_db/aiosqlite.py @@ -110,7 +110,29 @@ def __init__( self._db_args.pop("max_size", None) self._stopped = False self._conns = 0 - self._init_commands = self._db_args.pop("init_commands", []) + self._init_commands = self._add_missing_pragmas(self._db_args.pop("init_commands", [])) + + @staticmethod + def _add_missing_pragmas(init_commands: list[str]) -> list[str]: + has_foreign_keys = False + has_journal_mode = False + has_busy_timeout = False + for cmd in init_commands: + if "PRAGMA" not in cmd: + continue + if "foreign_keys" in cmd: + has_foreign_keys = True + elif "journal_mode" in cmd: + has_journal_mode = True + elif "busy_timeout" in cmd: + has_busy_timeout = True + if not has_foreign_keys: + init_commands.append("PRAGMA foreign_keys = ON") + if not has_journal_mode: + init_commands.append("PRAGMA journal_mode = WAL") + if not has_busy_timeout: + init_commands.append("PRAGMA busy_timeout = 5000") + return init_commands async def start(self) -> None: if self._conns: @@ -118,12 +140,13 @@ async def start(self) -> None: elif self._stopped: raise RuntimeError("database pool can't be restarted") self.log.debug(f"Connecting to {self.url}") + self.log.debug(f"Database connection init commands: {self._init_commands}") for _ in range(self._pool.maxsize): conn = await TxnConnection(self._path, **self._db_args) if self._init_commands: cur = await conn.cursor() for command in self._init_commands: - self.log.debug("Executing command: %s", command) + self.log.trace("Executing init command: %s", command) await cur.execute(command) await conn.commit() conn.row_factory = sqlite3.Row From 116729a875db8d14ff1b36cdaaef31523a0596c8 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 6 Dec 2022 11:22:56 +0200 Subject: [PATCH 249/456] Add additional forbidden example value for homeserver address --- mautrix/bridge/config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mautrix/bridge/config.py b/mautrix/bridge/config.py index 012512a6..16d8f6a3 100644 --- a/mautrix/bridge/config.py +++ b/mautrix/bridge/config.py @@ -76,6 +76,7 @@ def _new_token() -> str: def forbidden_defaults(self) -> list[ForbiddenDefault]: return [ ForbiddenDefault("homeserver.address", "https://example.com"), + ForbiddenDefault("homeserver.address", "https://matrix.example.com"), ForbiddenDefault("homeserver.domain", "example.com"), ] + ( [ From fba7541c847f3cc807021fb4267272733e42225e Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 6 Dec 2022 13:01:51 +0200 Subject: [PATCH 250/456] Add more workarounds for Conduit doing dumb stuff Fixes mautrix/telegram#873 --- mautrix/types/event/state.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/mautrix/types/event/state.py b/mautrix/types/event/state.py index 1ef18e3b..c6b351c6 100644 --- a/mautrix/types/event/state.py +++ b/mautrix/types/event/state.py @@ -258,7 +258,9 @@ def deserialize(cls, data: JSON) -> "StrippedStateEvent": try: event_type = EventType.find(data.get("type", None)) data.get("content", {})["__mautrix_event_type"] = event_type - data.get("unsigned", {}).get("prev_content", {})["__mautrix_event_type"] = event_type + (data.get("unsigned") or {}).get("prev_content", {})[ + "__mautrix_event_type" + ] = event_type except ValueError: pass return super().deserialize(data) @@ -305,7 +307,10 @@ def deserialize(cls, data: JSON) -> "StateEvent": try: event_type = EventType.find(data.get("type"), t_class=EventType.Class.STATE) data.get("content", {})["__mautrix_event_type"] = event_type - if "prev_content" in data and "prev_content" not in data.get("unsigned", {}): + if "prev_content" in data and "prev_content" not in (data.get("unsigned") or {}): + # This if is a workaround for Conduit being extremely dumb + if data.get("unsigned", {}) is None: + data["unsigned"] = {} data.setdefault("unsigned", {})["prev_content"] = data["prev_content"] data.get("unsigned", {}).get("prev_content", {})["__mautrix_event_type"] = event_type except ValueError: From 478577ebbbbd005c9cda5cb022c551dce7769bac Mon Sep 17 00:00:00 2001 From: Alejandro Herrera <50601186+bramenn@users.noreply.github.com> Date: Tue, 13 Dec 2022 14:26:48 -0500 Subject: [PATCH 251/456] Allow raw dicts in `MemoryStateStore.set_power_levels` (#127) --- mautrix/client/state_store/memory.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mautrix/client/state_store/memory.py b/mautrix/client/state_store/memory.py index 5d2d5b48..8f2edac5 100644 --- a/mautrix/client/state_store/memory.py +++ b/mautrix/client/state_store/memory.py @@ -170,8 +170,10 @@ async def get_power_levels(self, room_id: RoomID) -> PowerLevelStateEventContent return self.power_levels.get(room_id) async def set_power_levels( - self, room_id: RoomID, content: PowerLevelStateEventContent + self, room_id: RoomID, content: PowerLevelStateEventContent | dict[str, Any] ) -> None: + if not isinstance(content, PowerLevelStateEventContent): + content = PowerLevelStateEventContent.deserialize(content) self.power_levels[room_id] = content async def has_encryption_info_cached(self, room_id: RoomID) -> bool: From 11a092712530d3901af4cdf26d2eb6ee0b0f895a Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 14 Dec 2022 01:15:11 +0200 Subject: [PATCH 252/456] Bump version to 0.18.9 --- CHANGELOG.md | 14 ++++++++++++++ mautrix/__init__.py | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dc547c82..af2f5fc9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,17 @@ +## v0.18.9 (2022-12-14) + +* *(util.async_db)* Changed aiosqlite connector to force-enable foreign keys, + WAL mode and busy_timeout. + * The values can be changed by manually specifying the same PRAGMAs in the + `init_commands` db arg, e.g. `- PRAGMA foreign_keys = OFF`. +* *(types)* Added workaround to `StateEvent.deserialize` to handle Conduit's + broken `unsigned` fields. +* *(client.state_store)* Fixed `set_power_level` to allow raw dicts the same + way as `set_encryption_info` does (thanks to [@bramenn] in [#127]). + +[@bramenn]: https://github.com/bramenn +[#127]: https://github.com/mautrix/python/pull/127 + ## v0.18.8 (2022-11-18) * *(crypto.store.asyncpg)* Fixed bug causing `put_group_session` to fail when diff --git a/mautrix/__init__.py b/mautrix/__init__.py index b66f6369..ae628d89 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.18.8" +__version__ = "0.18.9" __author__ = "Tulir Asokan " __all__ = [ "api", From f6b10699c16b90bf763614014a94d35e24a1efd7 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 21 Dec 2022 22:40:20 +0200 Subject: [PATCH 253/456] Fix decoding json values in env vars --- mautrix/bridge/config.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/mautrix/bridge/config.py b/mautrix/bridge/config.py index 16d8f6a3..f48b20aa 100644 --- a/mautrix/bridge/config.py +++ b/mautrix/bridge/config.py @@ -46,7 +46,7 @@ def __init__( continue key = key.removeprefix(env_prefix) if value.startswith("json::"): - value = json.loads(value) + value = json.loads(value.removeprefix("json::")) self.env[key] = value def __getitem__(self, item: str) -> Any: @@ -57,8 +57,6 @@ def __getitem__(self, item: str) -> Any: except KeyError: pass else: - if val.startswith("json::"): - val = json.loads(val.removeprefix("json::")) return val return super().__getitem__(item) From 9d734319d3674185345f857e1ae0423b2b8f4d87 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 21 Dec 2022 22:43:32 +0200 Subject: [PATCH 254/456] Simplify try except block --- mautrix/bridge/config.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/mautrix/bridge/config.py b/mautrix/bridge/config.py index f48b20aa..4289b178 100644 --- a/mautrix/bridge/config.py +++ b/mautrix/bridge/config.py @@ -53,11 +53,9 @@ def __getitem__(self, item: str) -> Any: if self.env: try: sanitized_item = item.replace(".", "_").replace("[", "").replace("]", "").upper() - val = self.env[sanitized_item] + return self.env[sanitized_item] except KeyError: pass - else: - return val return super().__getitem__(item) def save(self) -> None: From e6fd197b2fd7ccecd3e338ec5979f4f6bb798f19 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 27 Dec 2022 22:19:33 +0200 Subject: [PATCH 255/456] Use hungryserv room yeeting endpoint --- mautrix/appservice/api/intent.py | 6 ++++++ mautrix/bridge/portal.py | 10 ++++++++++ 2 files changed, 16 insertions(+) diff --git a/mautrix/appservice/api/intent.py b/mautrix/appservice/api/intent.py index e5d17103..f16441e4 100644 --- a/mautrix/appservice/api/intent.py +++ b/mautrix/appservice/api/intent.py @@ -544,6 +544,12 @@ async def batch_send( ) return BatchSendResponse.deserialize(resp) + async def beeper_delete_room(self, room_id: RoomID) -> None: + versions = await self.versions() + if not versions.supports("com.beeper.room_yeeting"): + raise RuntimeError("Homeserver does not support yeeting rooms") + await self.api.request(Method.POST, Path.unstable["com.beeper.yeet"].rooms[room_id].delete) + # endregion # region Ensure functions diff --git a/mautrix/bridge/portal.py b/mautrix/bridge/portal.py index 0d4b110f..7ade73da 100644 --- a/mautrix/bridge/portal.py +++ b/mautrix/bridge/portal.py @@ -458,6 +458,16 @@ async def cleanup_room( message: str = "Cleaning room", puppets_only: bool = False, ) -> None: + if not puppets_only and cls.bridge.homeserver_software.is_hungry: + try: + await intent.beeper_delete_room(room_id) + return + except Exception: + cls.log.warning( + f"Failed to delete {room_id} using hungryserv yeet endpoint, " + f"falling back to normal method", + exc_info=True, + ) try: members = await intent.get_room_members(room_id) except MatrixError: From 883b7fb42017bbf3624f779137745023ca4ed809 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 27 Dec 2022 22:24:44 +0200 Subject: [PATCH 256/456] Remove double retry loop when accepting invites as bridge bot --- mautrix/bridge/matrix.py | 22 +++++----------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/mautrix/bridge/matrix.py b/mautrix/bridge/matrix.py index 6cd6b1c2..144264aa 100644 --- a/mautrix/bridge/matrix.py +++ b/mautrix/bridge/matrix.py @@ -472,23 +472,11 @@ async def send_permission_error(self, room_id: RoomID) -> None: ) async def accept_bot_invite(self, room_id: RoomID, inviter: br.BaseUser) -> None: - tries = 0 - while tries < 5: - try: - await self.az.intent.join_room(room_id) - break - except (IntentError, MatrixError): - tries += 1 - wait_for_seconds = (tries + 1) * 10 - if tries < 5: - self.log.exception( - f"Failed to join room {room_id} with bridge bot, " - f"retrying in {wait_for_seconds} seconds..." - ) - await asyncio.sleep(wait_for_seconds) - else: - self.log.exception(f"Failed to join room {room_id}, giving up.") - return + try: + await self.az.intent.join_room(room_id) + except Exception: + self.log.exception(f"Failed to join room {room_id} as bridge bot") + return if not await self.allow_command(inviter): await self.send_permission_error(room_id) From 26de4b2e5ca24d28ca7e55371d34cd5c27a3382d Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 2 Jan 2023 13:46:34 +0200 Subject: [PATCH 257/456] Remove appservice typing state store --- mautrix/appservice/api/intent.py | 12 ++++++++---- mautrix/appservice/state_store/memory.py | 21 --------------------- 2 files changed, 8 insertions(+), 25 deletions(-) diff --git a/mautrix/appservice/api/intent.py b/mautrix/appservice/api/intent.py index f16441e4..92966313 100644 --- a/mautrix/appservice/api/intent.py +++ b/mautrix/appservice/api/intent.py @@ -407,13 +407,17 @@ async def set_typing( room_id: RoomID, is_typing: bool = True, timeout: int = 5000, - ignore_cache: bool = False, ) -> None: + """ + Args: + room_id: The ID of the room in which the user is typing. + is_typing: Whether the user is typing. + .. deprecated:: 0.18.10 + Use ``timeout=0`` instead of setting this flag. + timeout: The length of time in seconds to mark this user as typing. + """ await self.ensure_joined(room_id) - if not ignore_cache and is_typing == self.state_store.is_typing(room_id, self.mxid): - return await super().set_typing(room_id, timeout if is_typing else 0) - self.state_store.set_typing(room_id, self.mxid, is_typing, timeout) async def error_and_leave( self, room_id: RoomID, text: str | None = None, html: str | None = None diff --git a/mautrix/appservice/state_store/memory.py b/mautrix/appservice/state_store/memory.py index f7af8a36..a9aba4ea 100644 --- a/mautrix/appservice/state_store/memory.py +++ b/mautrix/appservice/state_store/memory.py @@ -13,7 +13,6 @@ class ASStateStore(ClientStateStore, ABC): _presence: Dict[UserID, str] - _typing: Dict[Tuple[RoomID, UserID], int] _read: Dict[Tuple[RoomID, UserID], EventID] _registered: Dict[UserID, bool] @@ -21,7 +20,6 @@ def __init__(self) -> None: self._registered = {} # Non-persistent storage self._presence = {} - self._typing = {} self._read = {} async def is_registered(self, user_id: UserID) -> bool: @@ -69,22 +67,3 @@ def get_read(self, room_id: RoomID, user_id: UserID) -> Optional[EventID]: return self._read[(room_id, user_id)] except KeyError: return None - - def set_typing( - self, room_id: RoomID, user_id: UserID, is_typing: bool, timeout: int = 0 - ) -> None: - if is_typing: - ts = int(round(time.time() * 1000)) - self._typing[(room_id, user_id)] = ts + timeout - else: - try: - del self._typing[(room_id, user_id)] - except KeyError: - pass - - def is_typing(self, room_id: RoomID, user_id: UserID) -> bool: - ts = int(round(time.time() * 1000)) - try: - return self._typing[(room_id, user_id)] > ts - except KeyError: - return False From b717598ccdc850895db001e998aff28c47752194 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 2 Jan 2023 13:46:43 +0200 Subject: [PATCH 258/456] Update changelog --- CHANGELOG.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index af2f5fc9..70c15c8f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,12 @@ +## unreleased + +* *(bridge)* Removed accidentally nested reply loop when accepting invites as + the bridge bot. +* *(bridge)* Fixed decoding JSON values in config override env vars. +* *(appservice)* Removed typing status from state store. + * Additionally, the `is_typing` boolean in `set_typing` is now deprecated, + and `timeout=0` should be used instead to match the `ClientAPI` behavior. + ## v0.18.9 (2022-12-14) * *(util.async_db)* Changed aiosqlite connector to force-enable foreign keys, From 11593cc8e3a18b7d851dbd535a768b7a9ded07cf Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 2 Jan 2023 20:16:11 +0200 Subject: [PATCH 259/456] Fix manhole python compiler --- mautrix/util/manhole.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/mautrix/util/manhole.py b/mautrix/util/manhole.py index 727ae36d..c6505822 100644 --- a/mautrix/util/manhole.py +++ b/mautrix/util/manhole.py @@ -76,7 +76,13 @@ class StatefulCommandCompiler(codeop.CommandCompiler): def __init__(self) -> None: super().__init__() self.compiler = functools.partial( - compile, optimize=1, flags=ast.PyCF_ONLY_AST | codeop.PyCF_DONT_IMPLY_DEDENT + compile, + optimize=1, + flags=( + ast.PyCF_ONLY_AST + | codeop.PyCF_DONT_IMPLY_DEDENT + | codeop.PyCF_ALLOW_INCOMPLETE_INPUT + ), ) self.buf = BytesIO() From 2b004c642127f2a1b8d8b714acdfeddbdc691c51 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 3 Jan 2023 21:20:51 +0200 Subject: [PATCH 260/456] Ignore M_NOT_FOUND in hungryserv yeet endpoint --- mautrix/bridge/portal.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mautrix/bridge/portal.py b/mautrix/bridge/portal.py index 7ade73da..f51a0ae8 100644 --- a/mautrix/bridge/portal.py +++ b/mautrix/bridge/portal.py @@ -462,6 +462,9 @@ async def cleanup_room( try: await intent.beeper_delete_room(room_id) return + except MNotFound as err: + cls.log.debug(f"Hungryserv yeet returned {err}, assuming the room is already gone") + return except Exception: cls.log.warning( f"Failed to delete {room_id} using hungryserv yeet endpoint, " From ef0856c70e2dd6b2e929f46c760fc87d8097d730 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 4 Jan 2023 17:44:11 +0200 Subject: [PATCH 261/456] Remove legacy fields in Beeper MSS events --- mautrix/types/event/beeper.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/mautrix/types/event/beeper.py b/mautrix/types/event/beeper.py index dc588800..23a9f5cd 100644 --- a/mautrix/types/event/beeper.py +++ b/mautrix/types/event/beeper.py @@ -49,19 +49,8 @@ class BeeperMessageStatusEventContent(SerializableAttrs): error: Optional[str] = None message: Optional[str] = None - success: Optional[bool] = None - still_working: Optional[bool] = None - can_retry: Optional[bool] = None - is_certain: Optional[bool] = None - last_retry: Optional[EventID] = None - def fill_legacy_booleans(self) -> None: - self.success = self.status == MessageStatus.SUCCESS - if not self.success: - self.still_working = self.status == MessageStatus.PENDING - self.can_retry = self.status in (MessageStatus.PENDING, MessageStatus.RETRIABLE) - @dataclass class BeeperMessageStatusEvent(BaseRoomEvent, SerializableAttrs): From beade51174334d5c3ef4fd5f023ae05b772cb8ee Mon Sep 17 00:00:00 2001 From: Scott Weber Date: Mon, 9 Jan 2023 17:01:23 -0500 Subject: [PATCH 262/456] Add probe_path and probe_bytes functions (#129) * Add probe_path and probe_bytes functions * Run black and isort * Update mautrix/util/ffmpeg.py Co-authored-by: Tulir Asokan Co-authored-by: Tulir Asokan --- mautrix/util/ffmpeg.py | 102 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 97 insertions(+), 5 deletions(-) diff --git a/mautrix/util/ffmpeg.py b/mautrix/util/ffmpeg.py index f158ae52..7d195c43 100644 --- a/mautrix/util/ffmpeg.py +++ b/mautrix/util/ffmpeg.py @@ -5,9 +5,11 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import annotations -from typing import Iterable +from typing import Any, Iterable from pathlib import Path import asyncio +import json +import logging import mimetypes import os import shutil @@ -34,7 +36,89 @@ def __init__(self) -> None: ffmpeg_path = _abswhich("ffmpeg") -ffmpeg_default_params = ("-hide_banner", "-loglevel", "warning") +ffmpeg_default_params = ("-hide_banner", "-loglevel", "warning", "-y") + +ffprobe_path = _abswhich("ffprobe") +ffprobe_default_params = ( + "-loglevel", + "quiet", + "-print_format", + "json", + "-show_optional_fields", + "1", + "-show_format", + "-show_streams", +) + + +async def probe_path( + input_file: os.PathLike[str] | str, + logger: logging.Logger | None = None, +) -> Any: + """ + Probes a media file on the disk using ffprobe. + + Args: + input_file: The full path to the file. + + Returns: + A Python object containing the parsed JSON response from ffprobe + + Raises: + ConverterError: if ffprobe returns a non-zero exit code. + """ + if ffprobe_path is None: + raise NotInstalledError() + + input_file = Path(input_file) + proc = await asyncio.create_subprocess_exec( + ffprobe_path, + *ffprobe_default_params, + str(input_file), + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + stdin=asyncio.subprocess.PIPE, + ) + stdout, stderr = await proc.communicate() + if proc.returncode != 0: + err_text = stderr.decode("utf-8") if stderr else f"unknown ({proc.returncode})" + raise ConverterError(f"ffprobe error: {err_text}") + elif stderr and logger: + logger.warn(f"ffprobe warning: {stderr.decode('utf-8')}") + return json.loads(stdout) + + +async def probe_bytes( + data: bytes, + input_mime: str | None = None, + logger: logging.Logger | None = None, +) -> Any: + """ + Probe media file data using ffprobe. + + Args: + data: The bytes of the file to probe. + input_mime: The mime type of the input data. If not specified, will be guessed using magic. + + Returns: + A Python object containing the parsed JSON response from ffprobe + + Raises: + ConverterError: if ffprobe returns a non-zero exit code. + """ + if ffprobe_path is None: + raise NotInstalledError() + + if input_mime is None: + if magic is None: + raise ValueError("input_mime was not specified and magic is not installed") + input_mime = magic.mimetype(data) + input_extension = mimetypes.guess_extension(input_mime) + with tempfile.TemporaryDirectory(prefix="mautrix_ffmpeg_") as tmpdir: + input_file = Path(tmpdir) / f"data{input_extension}" + with open(input_file, "wb") as file: + file.write(data) + return await probe_path(input_file=input_file, logger=logger) async def convert_path( @@ -44,6 +128,7 @@ async def convert_path( output_args: Iterable[str] | None = None, remove_input: bool = False, output_path_override: os.PathLike[str] | str | None = None, + logger: logging.Logger | None = None, ) -> Path | bytes: """ Convert a media file on the disk using ffmpeg. @@ -76,6 +161,10 @@ async def convert_path( else: input_file = Path(input_file) output_file = input_file.parent / f"{input_file.stem}{output_extension}" + if input_file == output_file: + output_file = Path(output_file) + output_file = output_file.parent / f"{output_file.stem}-new{output_extension}" + proc = await asyncio.create_subprocess_exec( ffmpeg_path, *ffmpeg_default_params, @@ -92,9 +181,8 @@ async def convert_path( if proc.returncode != 0: err_text = stderr.decode("utf-8") if stderr else f"unknown ({proc.returncode})" raise ConverterError(f"ffmpeg error: {err_text}") - elif stderr: - # TODO log warnings? - pass + elif stderr and logger: + logger.warn(f"ffmpeg warning: {stderr.decode('utf-8')}") if remove_input and isinstance(input_file, Path): input_file.unlink(missing_ok=True) return stdout if output_file == "-" else output_file @@ -106,6 +194,7 @@ async def convert_bytes( input_args: Iterable[str] | None = None, output_args: Iterable[str] | None = None, input_mime: str | None = None, + logger: logging.Logger | None = None, ) -> bytes: """ Convert media file data using ffmpeg. @@ -140,6 +229,7 @@ async def convert_bytes( output_extension=output_extension, input_args=input_args, output_args=output_args, + logger=logger, ) with open(output_file, "rb") as file: return file.read() @@ -152,4 +242,6 @@ async def convert_bytes( "NotInstalledError", "convert_bytes", "convert_path", + "probe_bytes", + "probe_path", ] From 3a34fdcfa3855f38b95123aa83b2e652e6569d62 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 10 Jan 2023 15:22:12 +0200 Subject: [PATCH 263/456] Remove is_typing parameter in IntentAPI.set_typing --- mautrix/appservice/api/intent.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/mautrix/appservice/api/intent.py b/mautrix/appservice/api/intent.py index 92966313..8a486eca 100644 --- a/mautrix/appservice/api/intent.py +++ b/mautrix/appservice/api/intent.py @@ -405,19 +405,10 @@ async def get_room_member_info( async def set_typing( self, room_id: RoomID, - is_typing: bool = True, - timeout: int = 5000, + timeout: int = 0, ) -> None: - """ - Args: - room_id: The ID of the room in which the user is typing. - is_typing: Whether the user is typing. - .. deprecated:: 0.18.10 - Use ``timeout=0`` instead of setting this flag. - timeout: The length of time in seconds to mark this user as typing. - """ await self.ensure_joined(room_id) - await super().set_typing(room_id, timeout if is_typing else 0) + await super().set_typing(room_id, timeout) async def error_and_leave( self, room_id: RoomID, text: str | None = None, html: str | None = None From b0701627bca11dbbf37c552e8eda62f4b3c85918 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 10 Jan 2023 15:23:59 +0200 Subject: [PATCH 264/456] Add Dendrite support to hacky already joined error check --- mautrix/appservice/api/intent.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mautrix/appservice/api/intent.py b/mautrix/appservice/api/intent.py index 8a486eca..8b5ca16f 100644 --- a/mautrix/appservice/api/intent.py +++ b/mautrix/appservice/api/intent.py @@ -248,7 +248,9 @@ async def invite_user( await self.state_store.joined(room_id, user_id) except MatrixRequestError as e: # TODO remove this once MSC3848 is released and minimum spec version is bumped - if e.errcode == "M_FORBIDDEN" and "is already in the room" in e.message: + if e.errcode == "M_FORBIDDEN" and ( + "already in the room" in e.message or "is already joined to room" in e.message + ): await self.state_store.joined(room_id, user_id) else: raise From bee00cffca58d7ae566c2d216f6c21498d262a16 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 10 Jan 2023 15:25:30 +0200 Subject: [PATCH 265/456] Bump version to 0.19.0 --- CHANGELOG.md | 10 ++++++---- mautrix/__init__.py | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 70c15c8f..9027b641 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,11 +1,13 @@ -## unreleased +## v0.19.0 (2023-01-10) +* **Breaking change *(appservice)*** Removed typing status from state store. +* **Breaking change *(appservice)*** Removed `is_typing` parameter from + `IntentAPI.set_typing` to make the signature match `ClientAPI.set_typing`. + `timeout=0` is equivalent to the old `is_typing=False`. +* **Breaking change *(types)*** Removed legacy fields in Beeper MSS events. * *(bridge)* Removed accidentally nested reply loop when accepting invites as the bridge bot. * *(bridge)* Fixed decoding JSON values in config override env vars. -* *(appservice)* Removed typing status from state store. - * Additionally, the `is_typing` boolean in `set_typing` is now deprecated, - and `timeout=0` should be used instead to match the `ClientAPI` behavior. ## v0.18.9 (2022-12-14) diff --git a/mautrix/__init__.py b/mautrix/__init__.py index ae628d89..0b02a1fc 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.18.9" +__version__ = "0.19.0" __author__ = "Tulir Asokan " __all__ = [ "api", From a1f5a5bc5b07a25564e817789927218dcad6cdd3 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 11 Jan 2023 12:29:57 +0200 Subject: [PATCH 266/456] Add byte iter memory optimization to async media uploads --- mautrix/api.py | 10 ++---- .../client/api/modules/media_repository.py | 8 ++++- mautrix/util/__init__.py | 31 ++++++++++--------- mautrix/util/async_iter_bytes.py | 28 +++++++++++++++++ 4 files changed, 54 insertions(+), 23 deletions(-) create mode 100644 mautrix/util/async_iter_bytes.py diff --git a/mautrix/api.py b/mautrix/api.py index a0561d08..a1ccd07e 100644 --- a/mautrix/api.py +++ b/mautrix/api.py @@ -22,6 +22,7 @@ from mautrix import __optional_imports__, __version__ as mautrix_version from mautrix.errors import MatrixConnectionError, MatrixRequestError, make_request_error +from mautrix.util.async_iter_bytes import AsyncBody, async_iter_bytes from mautrix.util.logging import TraceLogger from mautrix.util.opt_prometheus import Counter @@ -155,7 +156,6 @@ def replace(self, find: str, replace: str) -> PathBuilder: """ _req_id = 0 -AsyncBody = AsyncGenerator[Union[bytes, bytearray, memoryview], None] def _next_global_req_id() -> int: @@ -164,12 +164,6 @@ def _next_global_req_id() -> int: return _req_id -async def _async_iter_bytes(data: bytearray | bytes, chunk_size: int = 1024**2) -> AsyncBody: - with memoryview(data) as mv: - for i in range(0, len(data), chunk_size): - yield mv[i : i + chunk_size] - - class HTTPAPI: """HTTPAPI is a simple asyncio Matrix API request sender.""" @@ -395,7 +389,7 @@ async def request( method, log_url, content, orig_content, query_params, headers, req_id, sensitive ) API_CALLS.labels(method=metrics_method).inc() - req_content = _async_iter_bytes(content) if do_fake_iter else content + req_content = async_iter_bytes(content) if do_fake_iter else content start = time.monotonic() try: resp_data, resp = await self._send( diff --git a/mautrix/client/api/modules/media_repository.py b/mautrix/client/api/modules/media_repository.py index b1de6751..2b76e881 100644 --- a/mautrix/client/api/modules/media_repository.py +++ b/mautrix/client/api/modules/media_repository.py @@ -20,6 +20,7 @@ MXOpenGraph, SerializerError, ) +from mautrix.util.async_iter_bytes import async_iter_bytes from mautrix.util.opt_prometheus import Histogram from ..base import BaseClientAPI @@ -286,15 +287,20 @@ async def _upload_to_url( headers: dict[str, str], data: bytes | bytearray | AsyncIterable[bytes], post_upload_query: dict[str, str], + min_iter_size: int = 25 * 1024 * 1024, ) -> None: retry_count = self.api.default_retry_count backoff = 4 + do_fake_iter = data and hasattr(data, "__len__") and len(data) > min_iter_size + if do_fake_iter: + headers["Content-Length"] = str(len(data)) while True: self.log.debug("Uploading media to external URL %s", upload_url) upload_response = None try: + req_data = async_iter_bytes(data) if do_fake_iter else data upload_response = await self.api.session.put( - upload_url, data=data, headers=headers + upload_url, data=req_data, headers=headers ) upload_response.raise_for_status() except Exception as e: diff --git a/mautrix/util/__init__.py b/mautrix/util/__init__.py index 6a7827d1..0212dd7e 100644 --- a/mautrix/util/__init__.py +++ b/mautrix/util/__init__.py @@ -1,22 +1,25 @@ __all__ = [ + "async_db", + "config", + "db", "formatter", "logging", - "config", - "signed_token", - "simple_template", - "manhole", - "markdown", - "simple_lock", + "async_getter_lock", + "async_iter_bytes", + "bridge_state", + "color_log", + "ffmpeg", "file_store", - "program", - "async_db", - "db", - "opt_prometheus", + "format_duration", "magic", - "bridge_state", + "manhole", + "markdown", "message_send_checkpoint", - "variation_selector", - "format_duration", - "ffmpeg", + "opt_prometheus", + "program", + "signed_token", + "simple_lock", + "simple_template", "utf16_surrogate", + "variation_selector", ] diff --git a/mautrix/util/async_iter_bytes.py b/mautrix/util/async_iter_bytes.py new file mode 100644 index 00000000..25e3806d --- /dev/null +++ b/mautrix/util/async_iter_bytes.py @@ -0,0 +1,28 @@ +# Copyright (c) 2023 Tulir Asokan +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +from typing import AsyncGenerator, Union + +AsyncBody = AsyncGenerator[Union[bytes, bytearray, memoryview], None] + + +async def async_iter_bytes(data: bytearray | bytes, chunk_size: int = 1024**2) -> AsyncBody: + """ + Return memory views into a byte array in chunks. This is used to prevent aiohttp from copying + the entire request body. + + Args: + data: The underlying data to iterate through. + chunk_size: How big each returned chunk should be. + + Returns: + An async generator that yields the given data in chunks. + """ + with memoryview(data) as mv: + for i in range(0, len(data), chunk_size): + yield mv[i : i + chunk_size] + + +__all__ = ["AsyncBody", "async_iter_bytes"] From d3486f22c0a6d3180519c261c5e0767d98404abb Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 11 Jan 2023 12:49:08 +0200 Subject: [PATCH 267/456] Add missing import --- mautrix/util/async_iter_bytes.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mautrix/util/async_iter_bytes.py b/mautrix/util/async_iter_bytes.py index 25e3806d..088baed4 100644 --- a/mautrix/util/async_iter_bytes.py +++ b/mautrix/util/async_iter_bytes.py @@ -3,6 +3,8 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. +from __future__ import annotations + from typing import AsyncGenerator, Union AsyncBody = AsyncGenerator[Union[bytes, bytearray, memoryview], None] From 2a71f0176701627765cd5a659f092d17b1798908 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 11 Jan 2023 12:50:49 +0200 Subject: [PATCH 268/456] Update CI and linter versions --- .github/workflows/python-package.yml | 6 +++--- .pre-commit-config.yaml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 81c07712..7c2264b4 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -9,7 +9,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.8", "3.9", "3.10"] + python-version: ["3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v3 @@ -50,14 +50,14 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-python@v3 with: - python-version: "3.10" + python-version: "3.11" - uses: isort/isort-action@master with: sortPaths: "./mautrix" - uses: psf/black@stable with: src: "./mautrix" - version: "22.3.0" + version: "22.12.0" - name: pre-commit run: | pip install pre-commit diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 56919be3..69af4a5a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.1.0 + rev: v4.3.0 hooks: - id: trailing-whitespace exclude_types: [markdown] @@ -8,13 +8,13 @@ repos: - id: check-yaml - id: check-added-large-files - repo: https://github.com/psf/black - rev: 22.3.0 + rev: 22.12.0 hooks: - id: black language_version: python3 files: ^mautrix/.*\.pyi?$ - repo: https://github.com/PyCQA/isort - rev: 5.10.1 + rev: 5.11.4 hooks: - id: isort files: ^mautrix/.*\.pyi?$ From ac82892bc06f102ff8bf3262182c9c722e1d8891 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 11 Jan 2023 15:09:27 +0200 Subject: [PATCH 269/456] Mark Python 3.11 as supported in setup.py --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f7596154..4b3c9f78 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ ], extras_require={ "detect_mimetype": ["python-magic>=0.4.15,<0.5"], - "lint": ["black==22.1.0", "isort"], + "lint": ["black~=22.1", "isort"], "test": ["pytest", "pytest-asyncio", *test_dependencies], "encryption": encryption_dependencies, }, @@ -45,6 +45,7 @@ "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", ], package_data={ From 51c4e4b77a9cba43b3854ccb534fd4a53a463197 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 11 Jan 2023 15:12:21 +0200 Subject: [PATCH 270/456] Bump version to 0.19.1 --- CHANGELOG.md | 11 +++++++++++ mautrix/__init__.py | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9027b641..bd69c304 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,14 @@ +## v0.19.1 (2023-01-11) + +* Marked Python 3.11 as supported. Python 3.8 support will likely be dropped in + the coming months. +* *(client.api)* Added request payload memory optimization to MSC3870 URL uploads. + * aiohttp will duplicate the entire request body if it's raw bytes, which + wastes a lot of memory. The optimization is passing an iterator instead of + raw bytes, so aiohttp won't accidentally duplicate the whole thing. + * The main `HTTPAPI` has had the optimization for a while, but uploading to + URL calls aiohttp manually. + ## v0.19.0 (2023-01-10) * **Breaking change *(appservice)*** Removed typing status from state store. diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 0b02a1fc..224a35ca 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.19.0" +__version__ = "0.19.1" __author__ = "Tulir Asokan " __all__ = [ "api", From 933af513fc78aa06a4e5fa5243a461b36200ae04 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 11 Jan 2023 16:54:09 +0200 Subject: [PATCH 271/456] Add utility for reading aiohttp response into bytearray --- mautrix/api.py | 2 +- .../client/api/modules/media_repository.py | 2 +- mautrix/util/__init__.py | 4 +- mautrix/util/async_body.py | 95 +++++++++++++++++++ mautrix/util/async_iter_bytes.py | 30 ------ 5 files changed, 100 insertions(+), 33 deletions(-) create mode 100644 mautrix/util/async_body.py delete mode 100644 mautrix/util/async_iter_bytes.py diff --git a/mautrix/api.py b/mautrix/api.py index a1ccd07e..f6c9f475 100644 --- a/mautrix/api.py +++ b/mautrix/api.py @@ -22,7 +22,7 @@ from mautrix import __optional_imports__, __version__ as mautrix_version from mautrix.errors import MatrixConnectionError, MatrixRequestError, make_request_error -from mautrix.util.async_iter_bytes import AsyncBody, async_iter_bytes +from mautrix.util.async_body import AsyncBody, async_iter_bytes from mautrix.util.logging import TraceLogger from mautrix.util.opt_prometheus import Counter diff --git a/mautrix/client/api/modules/media_repository.py b/mautrix/client/api/modules/media_repository.py index 2b76e881..f91ba456 100644 --- a/mautrix/client/api/modules/media_repository.py +++ b/mautrix/client/api/modules/media_repository.py @@ -20,7 +20,7 @@ MXOpenGraph, SerializerError, ) -from mautrix.util.async_iter_bytes import async_iter_bytes +from mautrix.util.async_body import async_iter_bytes from mautrix.util.opt_prometheus import Histogram from ..base import BaseClientAPI diff --git a/mautrix/util/__init__.py b/mautrix/util/__init__.py index 0212dd7e..e731dfa4 100644 --- a/mautrix/util/__init__.py +++ b/mautrix/util/__init__.py @@ -1,11 +1,13 @@ __all__ = [ + # Directory modules "async_db", "config", "db", "formatter", "logging", + # File modules + "async_body", "async_getter_lock", - "async_iter_bytes", "bridge_state", "color_log", "ffmpeg", diff --git a/mautrix/util/async_body.py b/mautrix/util/async_body.py new file mode 100644 index 00000000..4db4d1e5 --- /dev/null +++ b/mautrix/util/async_body.py @@ -0,0 +1,95 @@ +# Copyright (c) 2023 Tulir Asokan +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +from __future__ import annotations + +from typing import AsyncGenerator, Union +import logging + +import aiohttp + +AsyncBody = AsyncGenerator[Union[bytes, bytearray, memoryview], None] + + +async def async_iter_bytes(data: bytearray | bytes, chunk_size: int = 1024**2) -> AsyncBody: + """ + Return memory views into a byte array in chunks. This is used to prevent aiohttp from copying + the entire request body. + + Args: + data: The underlying data to iterate through. + chunk_size: How big each returned chunk should be. + + Returns: + An async generator that yields the given data in chunks. + """ + with memoryview(data) as mv: + for i in range(0, len(data), chunk_size): + yield mv[i : i + chunk_size] + + +class FileTooLargeError(Exception): + def __init__(self, max_size: int) -> None: + super().__init__(f"File size larger than maximum ({max_size / 1024 / 1024} MiB)") + + +_default_dl_log = logging.getLogger("mau.util.download") + + +async def read_response_chunks( + resp: aiohttp.ClientResponse, max_size: int, log: logging.Logger = _default_dl_log +) -> bytearray: + """ + Read the body from an aiohttp response in chunks into a mutable bytearray. + + Args: + resp: The aiohttp response object to read the body from. + max_size: The maximum size to read. FileTooLargeError will be raised if the Content-Length + is higher than this, or if the body exceeds this size during reading. + log: A logger for logging download status. + + Returns: + The body data as a byte array. + + Raises: + FileTooLargeError: if the body is larger than the provided max_size. + """ + content_length = int(resp.headers.get("Content-Length", "0")) + if 0 < max_size < content_length: + raise FileTooLargeError(max_size) + size_str = "unknown length" if content_length == 0 else f"{content_length} bytes" + log.info(f"Reading file download response with {size_str} (max: {max_size})") + data = bytearray(content_length) + mv = memoryview(data) if content_length > 0 else None + read_size = 0 + max_size += 1 + while True: + block = await resp.content.readany() + if not block: + break + max_size -= len(block) + if max_size <= 0: + raise FileTooLargeError(max_size) + if len(data) >= read_size + len(block): + mv[read_size : read_size + len(block)] = block + elif len(data) > read_size: + log.warning("File being downloaded is bigger than expected") + mv[read_size:] = block[: len(data) - read_size] + mv.release() + mv = None + data.extend(block[len(data) - read_size :]) + else: + if mv is not None: + mv.release() + mv = None + data.extend(block) + read_size += len(block) + if mv is not None: + mv.release() + log.info(f"Successfully read {read_size} bytes of file download response") + return data + + +__all__ = ["AsyncBody", "FileTooLargeError", "async_iter_bytes", "async_read_bytes"] diff --git a/mautrix/util/async_iter_bytes.py b/mautrix/util/async_iter_bytes.py deleted file mode 100644 index 088baed4..00000000 --- a/mautrix/util/async_iter_bytes.py +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright (c) 2023 Tulir Asokan -# -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import annotations - -from typing import AsyncGenerator, Union - -AsyncBody = AsyncGenerator[Union[bytes, bytearray, memoryview], None] - - -async def async_iter_bytes(data: bytearray | bytes, chunk_size: int = 1024**2) -> AsyncBody: - """ - Return memory views into a byte array in chunks. This is used to prevent aiohttp from copying - the entire request body. - - Args: - data: The underlying data to iterate through. - chunk_size: How big each returned chunk should be. - - Returns: - An async generator that yields the given data in chunks. - """ - with memoryview(data) as mv: - for i in range(0, len(data), chunk_size): - yield mv[i : i + chunk_size] - - -__all__ = ["AsyncBody", "async_iter_bytes"] From e0baf7c0e9b7ee5d633ed9ebb066e5853ad5ed52 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 11 Jan 2023 17:10:36 +0200 Subject: [PATCH 272/456] Bump latest version in asyncpg crypto store v6 is only for existing databases so it can be skipped for new ones --- mautrix/crypto/store/asyncpg/upgrade.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/crypto/store/asyncpg/upgrade.py b/mautrix/crypto/store/asyncpg/upgrade.py index f7f50a74..7a55c10d 100644 --- a/mautrix/crypto/store/asyncpg/upgrade.py +++ b/mautrix/crypto/store/asyncpg/upgrade.py @@ -16,7 +16,7 @@ ) -@upgrade_table.register(description="Latest revision", upgrades_to=5) +@upgrade_table.register(description="Latest revision", upgrades_to=6) async def upgrade_blank_to_v4(conn: Connection) -> None: await conn.execute( """CREATE TABLE IF NOT EXISTS crypto_account ( From 62b515e718edccf2838bd879fa71fb75cb8746c3 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 14 Jan 2023 14:16:39 +0200 Subject: [PATCH 273/456] Fix external URL upload retry loop --- mautrix/client/api/modules/media_repository.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mautrix/client/api/modules/media_repository.py b/mautrix/client/api/modules/media_repository.py index f91ba456..e516fbd0 100644 --- a/mautrix/client/api/modules/media_repository.py +++ b/mautrix/client/api/modules/media_repository.py @@ -304,7 +304,7 @@ async def _upload_to_url( ) upload_response.raise_for_status() except Exception as e: - if retry_count == 0: + if retry_count <= 0: raise make_request_error( http_status=upload_response.status if upload_response else -1, text=(await upload_response.text()) if upload_response else "", @@ -317,7 +317,7 @@ async def _upload_to_url( ) await asyncio.sleep(backoff) backoff *= 2 - retry_count = -1 + retry_count -= 1 else: break From 95f61b9dcc4c64c9aab59a5017e88bd53324a76b Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 14 Jan 2023 14:18:01 +0200 Subject: [PATCH 274/456] Reduce base backoff in external URL uploads --- mautrix/client/api/modules/media_repository.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/client/api/modules/media_repository.py b/mautrix/client/api/modules/media_repository.py index e516fbd0..f247285a 100644 --- a/mautrix/client/api/modules/media_repository.py +++ b/mautrix/client/api/modules/media_repository.py @@ -290,7 +290,7 @@ async def _upload_to_url( min_iter_size: int = 25 * 1024 * 1024, ) -> None: retry_count = self.api.default_retry_count - backoff = 4 + backoff = 2 do_fake_iter = data and hasattr(data, "__len__") and len(data) > min_iter_size if do_fake_iter: headers["Content-Length"] = str(len(data)) From e808bea4774ff5035ac57a18d30ee1b4016662c5 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 14 Jan 2023 14:20:09 +0200 Subject: [PATCH 275/456] Update changelog --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bd69c304..a11d8f56 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## v0.19.2 (unreleased) + +* *(util.async_body)* Added utility for reading aiohttp response into a bytearray + (so that the output is mutable, e.g. for decrypting or encrypting media). +* *(client.api)* Fixed retry loop for MSC3870 URL uploads not exiting properly + after too many errors. + ## v0.19.1 (2023-01-11) * Marked Python 3.11 as supported. Python 3.8 support will likely be dropped in From f4b1dfbf2f7a4fdbe441424284b68d1cca57b404 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 14 Jan 2023 14:25:48 +0200 Subject: [PATCH 276/456] Bump version to 0.19.2 --- CHANGELOG.md | 2 +- mautrix/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a11d8f56..84b027db 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -## v0.19.2 (unreleased) +## v0.19.2 (2023-01-14) * *(util.async_body)* Added utility for reading aiohttp response into a bytearray (so that the output is mutable, e.g. for decrypting or encrypting media). diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 224a35ca..67d8ba2e 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.19.1" +__version__ = "0.19.2" __author__ = "Tulir Asokan " __all__ = [ "api", From 6d261cc5c004f7fee7d2ed5c5b526b61b4f5b0ff Mon Sep 17 00:00:00 2001 From: Sumner Evans Date: Thu, 19 Jan 2023 11:32:07 -0700 Subject: [PATCH 277/456] checkpoints: add delivery failed Signed-off-by: Sumner Evans --- mautrix/util/message_send_checkpoint.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mautrix/util/message_send_checkpoint.py b/mautrix/util/message_send_checkpoint.py index 61eb691d..5dd023b5 100644 --- a/mautrix/util/message_send_checkpoint.py +++ b/mautrix/util/message_send_checkpoint.py @@ -29,6 +29,7 @@ class MessageSendCheckpointStatus(SerializableEnum): PERM_FAILURE = "PERM_FAILURE" UNSUPPORTED = "UNSUPPORTED" TIMEOUT = "TIMEOUT" + DELIVERY_FAILED = "DELIVERY_FAILED" class MessageSendCheckpointReportedBy(SerializableEnum): From dd3e11860471e55cd97a5174cd792c04001dd65a Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 24 Jan 2023 16:46:50 +0200 Subject: [PATCH 278/456] Bump maximum delay for receiving decryption keys --- mautrix/bridge/matrix.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mautrix/bridge/matrix.py b/mautrix/bridge/matrix.py index 144264aa..ffd2eb9a 100644 --- a/mautrix/bridge/matrix.py +++ b/mautrix/bridge/matrix.py @@ -583,7 +583,6 @@ async def _send_crypto_status_error( error=msg, message=err.human_message if err else None, ) - status_content.fill_legacy_booleans() await self.az.intent.send_message_event( evt.room_id, EventType.BEEPER_MESSAGE_STATUS, status_content ) @@ -784,7 +783,7 @@ async def handle_encrypted(self, evt: EncryptedEvent) -> None: try: decrypted = await self.e2ee.decrypt(evt, wait_session_timeout=3) except SessionNotFound as e: - await self._handle_encrypted_wait(evt, e, wait=6) + await self._handle_encrypted_wait(evt, e, wait=22) except DecryptionError as e: self.log.warning(f"Failed to decrypt {evt.event_id}: {e}") self.log.trace("%s decryption traceback:", evt.event_id, exc_info=True) From 0c5debaebc678c05c52090ffb2f3d66b19141918 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 26 Jan 2023 00:36:04 +0200 Subject: [PATCH 279/456] Deduplicate bridge states more --- mautrix/util/bridge_state.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mautrix/util/bridge_state.py b/mautrix/util/bridge_state.py index b7c346f0..dcb42977 100644 --- a/mautrix/util/bridge_state.py +++ b/mautrix/util/bridge_state.py @@ -62,8 +62,8 @@ class BridgeStateEvent(SerializableEnum): class BridgeState(SerializableAttrs): human_readable_errors: ClassVar[Dict[Optional[str], str]] = {} default_source: ClassVar[str] = "bridge" - default_error_ttl: ClassVar[int] = 60 - default_ok_ttl: ClassVar[int] = 240 + default_error_ttl: ClassVar[int] = 3600 + default_ok_ttl: ClassVar[int] = 21600 state_event: BridgeStateEvent user_id: Optional[UserID] = None @@ -106,8 +106,8 @@ def should_deduplicate(self, prev_state: Optional["BridgeState"]) -> bool: ): # If there's no previous state or the state was different, send this one. return False - # If there's more than ⅘ of the previous pong's time-to-live left, drop this one - return prev_state.timestamp + (prev_state.ttl / 5) > self.timestamp + # If the previous state is recent, drop this one + return prev_state.timestamp + prev_state.ttl > self.timestamp async def send(self, url: str, token: str, log: logging.Logger, log_sent: bool = True) -> bool: if not url: From fa9f9fd294eab84ac654b114078186563f9633cc Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 27 Jan 2023 15:07:37 +0200 Subject: [PATCH 280/456] Bump version to 0.19.3 --- CHANGELOG.md | 4 ++++ mautrix/__init__.py | 2 +- optional-requirements.txt | 2 +- setup.py | 2 +- 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 84b027db..1a94abdf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## v0.19.3 (2023-01-27) + +* *(bridge)* Bumped default timeouts for decrypting incoming messages. + ## v0.19.2 (2023-01-14) * *(util.async_body)* Added utility for reading aiohttp response into a bytearray diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 67d8ba2e..4a89ae2f 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.19.2" +__version__ = "0.19.3" __author__ = "Tulir Asokan " __all__ = [ "api", diff --git a/optional-requirements.txt b/optional-requirements.txt index 6cfdcece..a5660f4a 100644 --- a/optional-requirements.txt +++ b/optional-requirements.txt @@ -1,6 +1,6 @@ python-magic ruamel.yaml -SQLAlchemy +SQLAlchemy<2 commonmark lxml asyncpg diff --git a/setup.py b/setup.py index 4b3c9f78..cabdf15a 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ from mautrix import __version__ encryption_dependencies = ["python-olm", "unpaddedbase64", "pycryptodome"] -test_dependencies = ["aiosqlite", "sqlalchemy", "asyncpg", *encryption_dependencies] +test_dependencies = ["aiosqlite", "sqlalchemy<2", "asyncpg", *encryption_dependencies] setuptools.setup( name="mautrix", From 6b3b17c5d9a4203eed2a847c21c4b9c908817d5c Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 31 Jan 2023 16:32:48 +0200 Subject: [PATCH 281/456] Use target event thread parent if event is already in thread Previously set_thread_parent didn't work correctly if you passed an event that's already in a thread and not the root. --- mautrix/types/event/message.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/mautrix/types/event/message.py b/mautrix/types/event/message.py index eab4ba95..47c4e2e6 100644 --- a/mautrix/types/event/message.py +++ b/mautrix/types/event/message.py @@ -112,6 +112,12 @@ def set_thread_parent( self.relates_to.event_id = ( thread_parent if isinstance(thread_parent, str) else thread_parent.event_id ) + if isinstance(thread_parent, MessageEvent) and isinstance( + thread_parent.content, BaseMessageEventContentFuncs + ): + self.relates_to.event_id = ( + thread_parent.content.get_thread_parent() or self.relates_to.event_id + ) if not disable_reply_fallback: self.set_reply(last_event_in_thread or thread_parent, **kwargs) self.relates_to.is_falling_back = True From 98140aa4b3168bbe64e3dc3278e6a11057e94536 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 11 Feb 2023 22:32:18 +0200 Subject: [PATCH 282/456] Update linters --- .github/workflows/python-package.yml | 2 +- .pre-commit-config.yaml | 6 +++--- dev-requirements.txt | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 7c2264b4..96b55773 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -57,7 +57,7 @@ jobs: - uses: psf/black@stable with: src: "./mautrix" - version: "22.12.0" + version: "23.1.0" - name: pre-commit run: | pip install pre-commit diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 69af4a5a..77beb7ac 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.3.0 + rev: v4.4.0 hooks: - id: trailing-whitespace exclude_types: [markdown] @@ -8,13 +8,13 @@ repos: - id: check-yaml - id: check-added-large-files - repo: https://github.com/psf/black - rev: 22.12.0 + rev: 23.1.0 hooks: - id: black language_version: python3 files: ^mautrix/.*\.pyi?$ - repo: https://github.com/PyCQA/isort - rev: 5.11.4 + rev: 5.12.0 hooks: - id: isort files: ^mautrix/.*\.pyi?$ diff --git a/dev-requirements.txt b/dev-requirements.txt index e513c0df..5cd14c23 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,3 +1,3 @@ pre-commit>=2.10.1,<3 isort>=5.10.1,<6 -black>=22.3,<23 +black>=23,<24 From 4c7503d2533de00ab37d20a9c2ba3bb736fdddfc Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 11 Feb 2023 22:36:11 +0200 Subject: [PATCH 283/456] Add wrapper for creating background tasks The wrapper ensures that the reference to the task is not lost before it completes, and also logs uncaught errors. --- mautrix/appservice/as_handler.py | 3 +- mautrix/bridge/custom_puppet.py | 3 +- mautrix/bridge/matrix.py | 8 +-- mautrix/bridge/portal.py | 7 +-- mautrix/bridge/user.py | 3 +- .../client/api/modules/media_repository.py | 3 +- mautrix/crypto/decrypt_olm.py | 7 +-- mautrix/crypto/machine.py | 3 +- mautrix/util/__init__.py | 1 + mautrix/util/background_task.py | 53 +++++++++++++++++++ 10 files changed, 76 insertions(+), 15 deletions(-) create mode 100644 mautrix/util/background_task.py diff --git a/mautrix/appservice/as_handler.py b/mautrix/appservice/as_handler.py index 88715100..78cd4307 100644 --- a/mautrix/appservice/as_handler.py +++ b/mautrix/appservice/as_handler.py @@ -27,6 +27,7 @@ SerializerError, UserID, ) +from mautrix.util import background_task HandlerFunc = Callable[[Event], Awaitable] @@ -314,7 +315,7 @@ async def try_handle(handler_func: HandlerFunc): for handler in self.event_handlers: # TODO add option to handle events synchronously - asyncio.create_task(try_handle(handler)) + background_task.create(try_handle(handler)) def matrix_event_handler(self, func: HandlerFunc) -> HandlerFunc: self.event_handlers.append(func) diff --git a/mautrix/bridge/custom_puppet.py b/mautrix/bridge/custom_puppet.py index 3d5c7f9e..e2759513 100644 --- a/mautrix/bridge/custom_puppet.py +++ b/mautrix/bridge/custom_puppet.py @@ -41,6 +41,7 @@ SyncToken, UserID, ) +from mautrix.util import background_task from .. import bridge as br @@ -409,7 +410,7 @@ def _handle_sync(self, sync_resp: dict) -> None: # Deserialize and handle all events for event in chain(ephemeral_events, presence_events): - asyncio.create_task(self.mx.try_handle_sync_event(Event.deserialize(event))) + background_task.create(self.mx.try_handle_sync_event(Event.deserialize(event))) async def _try_sync(self) -> None: try: diff --git a/mautrix/bridge/matrix.py b/mautrix/bridge/matrix.py index ffd2eb9a..1ba2ac7c 100644 --- a/mautrix/bridge/matrix.py +++ b/mautrix/bridge/matrix.py @@ -56,7 +56,7 @@ Version, VersionsResponse, ) -from mautrix.util import markdown +from mautrix.util import background_task, markdown from mautrix.util.logging import TraceLogger from mautrix.util.message_send_checkpoint import ( CHECKPOINT_TYPES, @@ -382,7 +382,7 @@ async def handle_puppet_nonportal_invite( members = await intent.get_room_members(room_id) except MatrixError: self.log.exception(f"Failed to get state after joining {room_id} as {intent.mxid}") - asyncio.create_task(intent.leave_room(room_id, reason="Internal error")) + background_task.create(intent.leave_room(room_id, reason="Internal error")) return if create_evt.type == RoomType.SPACE: await self.handle_puppet_space_invite(room_id, puppet, invited_by, evt) @@ -798,7 +798,7 @@ async def _handle_encrypted_wait( f"Couldn't find session {err.session_id} trying to decrypt {evt.event_id}," " waiting even longer" ) - asyncio.create_task( + background_task.create( self.e2ee.crypto.request_room_key( evt.room_id, evt.content.sender_key, @@ -875,7 +875,7 @@ def _send_message_checkpoint( info=str(err) if err else None, retry_num=retry_num, ) - asyncio.create_task(checkpoint.send(endpoint, self.az.as_token, self.log)) + background_task.create(checkpoint.send(endpoint, self.az.as_token, self.log)) allowed_event_classes: tuple[type, ...] = ( MessageEvent, diff --git a/mautrix/bridge/portal.py b/mautrix/bridge/portal.py index f51a0ae8..1a6a1343 100644 --- a/mautrix/bridge/portal.py +++ b/mautrix/bridge/portal.py @@ -30,6 +30,7 @@ TextMessageEventContent, UserID, ) +from mautrix.util import background_task from mautrix.util.logging import TraceLogger from mautrix.util.simple_lock import SimpleLock @@ -402,7 +403,7 @@ async def restart_scheduled_disappearing(cls) -> None: for msg in msgs: portal = await cls.bridge.get_portal(msg.room_id) if portal and portal.mxid: - asyncio.create_task(portal._disappear_event(msg)) + background_task.create(portal._disappear_event(msg)) else: await msg.delete() @@ -418,7 +419,7 @@ async def schedule_disappearing(self) -> None: for msg in msgs: msg.start_timer() await msg.update() - asyncio.create_task(self._disappear_event(msg)) + background_task.create(self._disappear_event(msg)) async def _send_message( self, @@ -431,7 +432,7 @@ async def _send_message( event_type, content = await self.matrix.e2ee.encrypt(self.mxid, event_type, content) event_id = await intent.send_message_event(self.mxid, event_type, content, **kwargs) if intent.api.is_real_user: - asyncio.create_task(intent.mark_read(self.mxid, event_id)) + background_task.create(intent.mark_read(self.mxid, event_id)) return event_id @property diff --git a/mautrix/bridge/user.py b/mautrix/bridge/user.py index bea6bdf7..66860c04 100644 --- a/mautrix/bridge/user.py +++ b/mautrix/bridge/user.py @@ -16,6 +16,7 @@ from mautrix.appservice import AppService from mautrix.errors import MNotFound from mautrix.types import EventID, EventType, Membership, MessageType, RoomID, UserID +from mautrix.util import background_task from mautrix.util.bridge_state import BridgeState, BridgeStateEvent from mautrix.util.logging import TraceLogger from mautrix.util.message_send_checkpoint import ( @@ -244,7 +245,7 @@ def send_remote_checkpoint( """ if not self.bridge.config["homeserver.message_send_checkpoint_endpoint"]: return WrappedTask(task=None) - task = asyncio.create_task( + task = background_task.create( MessageSendCheckpoint( event_id=event_id, room_id=room_id, diff --git a/mautrix/client/api/modules/media_repository.py b/mautrix/client/api/modules/media_repository.py index f247285a..edf5e701 100644 --- a/mautrix/client/api/modules/media_repository.py +++ b/mautrix/client/api/modules/media_repository.py @@ -20,6 +20,7 @@ MXOpenGraph, SerializerError, ) +from mautrix.util import background_task from mautrix.util.async_body import async_iter_bytes from mautrix.util.opt_prometheus import Histogram @@ -157,7 +158,7 @@ async def _try_upload(): except Exception as e: self.log.error(f"Failed to upload {mxc}: {type(e).__name__}: {e}") - asyncio.create_task(_try_upload()) + background_task.create(_try_upload()) return mxc else: with self._observe_upload_time(size): diff --git a/mautrix/crypto/decrypt_olm.py b/mautrix/crypto/decrypt_olm.py index 8182b160..6eef76f5 100644 --- a/mautrix/crypto/decrypt_olm.py +++ b/mautrix/crypto/decrypt_olm.py @@ -19,6 +19,7 @@ ToDeviceEvent, UserID, ) +from mautrix.util import background_task from .base import BaseOlmMachine from .sessions import Session @@ -74,19 +75,19 @@ async def _decrypt_olm_ciphertext( f"Found matching session yet decryption failed for sender {sender}" f" with key {sender_key}" ) - asyncio.create_task(self._unwedge_session(sender, sender_key)) + background_task.create(self._unwedge_session(sender, sender_key)) raise if not plaintext: if message.type != OlmMsgType.PREKEY: - asyncio.create_task(self._unwedge_session(sender, sender_key)) + background_task.create(self._unwedge_session(sender, sender_key)) raise DecryptionError("Decryption failed for normal message") self.log.trace(f"Trying to create inbound session for {sender}/{sender_key}") try: session = await self._create_inbound_session(sender_key, message.body) except olm.OlmSessionError as e: - asyncio.create_task(self._unwedge_session(sender, sender_key)) + background_task.create(self._unwedge_session(sender, sender_key)) raise DecryptionError("Failed to create new session from prekey message") from e self.log.debug( f"Created inbound session {session.id} for {sender} (sender key: {sender_key})" diff --git a/mautrix/crypto/machine.py b/mautrix/crypto/machine.py index a1fd4e16..3e845b44 100644 --- a/mautrix/crypto/machine.py +++ b/mautrix/crypto/machine.py @@ -24,6 +24,7 @@ TrustState, UserID, ) +from mautrix.util import background_task from mautrix.util.logging import TraceLogger from .account import OlmAccount @@ -109,7 +110,7 @@ async def handle_as_otk_counts( self.log.warning(f"Got OTK count for unknown device {user_id}/{device_id}") async def handle_as_device_lists(self, device_lists: DeviceLists) -> None: - asyncio.create_task(self.handle_device_lists(device_lists)) + background_task.create(self.handle_device_lists(device_lists)) async def handle_as_to_device_event(self, evt: ASToDeviceEvent) -> None: if evt.to_user_id != self.client.mxid or evt.to_device_id != self.client.device_id: diff --git a/mautrix/util/__init__.py b/mautrix/util/__init__.py index e731dfa4..fd349bef 100644 --- a/mautrix/util/__init__.py +++ b/mautrix/util/__init__.py @@ -8,6 +8,7 @@ # File modules "async_body", "async_getter_lock", + "background_task", "bridge_state", "color_log", "ffmpeg", diff --git a/mautrix/util/background_task.py b/mautrix/util/background_task.py new file mode 100644 index 00000000..e22e74f1 --- /dev/null +++ b/mautrix/util/background_task.py @@ -0,0 +1,53 @@ +# Copyright (c) 2023 Tulir Asokan +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +from __future__ import annotations + +from typing import Coroutine +import asyncio +import logging + +_tasks = set() +log = logging.getLogger("mau.background_task") + + +async def catch(coro: Coroutine, caller: str) -> None: + try: + await coro + except Exception: + log.exception(f"Uncaught error in background task (created in {caller})") + + +# Logger.findCaller finds the 3rd stack frame, so add an intermediate function +# to get the caller of create(). +def _find_caller() -> tuple[str, int, str, None]: + return log.findCaller() + + +def create(coro: Coroutine, *, name: str | None = None, catch_errors: bool = True) -> asyncio.Task: + """ + Create a background asyncio task safely, ensuring a reference is kept until the task completes. + It also catches and logs uncaught errors (unless disabled via the parameter). + + Args: + coro: The coroutine to wrap in a task and execute. + name: An optional name for the created task. + catch_errors: Should the task be wrapped in a try-except block to log any uncaught errors? + + Returns: + An asyncio Task object wrapping the given coroutine. + """ + if catch_errors: + try: + file_name, line_number, function_name, _ = _find_caller() + caller = f"{function_name} at {file_name}:{line_number}" + except ValueError: + caller = "unknown function" + task = asyncio.create_task(catch(coro, caller), name=name) + else: + task = asyncio.create_task(coro, name=name) + _tasks.add(task) + task.add_done_callback(_tasks.discard) + return task From 94a258c3b379485c0a4a299fa0eb2dae9694350f Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 12 Feb 2023 12:17:47 +0200 Subject: [PATCH 284/456] Bump version to 0.19.4 --- CHANGELOG.md | 9 +++++++++ mautrix/__init__.py | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a94abdf..70304b94 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,12 @@ +## v0.19.4 (2023-02-12) + +* *(types)* Changed `set_thread_parent` to inherit the existing thread parent + if a `MessageEvent` is passed, as starting threads from a message in a thread + is not allowed. +* *(util.background_task)* Added new utility for creating background tasks + safely, by ensuring that the task is not garbage collected before finishing + and logging uncaught exceptions immediately. + ## v0.19.3 (2023-01-27) * *(bridge)* Bumped default timeouts for decrypting incoming messages. diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 4a89ae2f..02a72acd 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.19.3" +__version__ = "0.19.4" __author__ = "Tulir Asokan " __all__ = [ "api", From 1410a3a9e1545ecd8c79254fefec7fbe2b8d7a23 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 15 Feb 2023 21:54:52 +0200 Subject: [PATCH 285/456] Fix usages of log.warn --- mautrix/util/ffmpeg.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mautrix/util/ffmpeg.py b/mautrix/util/ffmpeg.py index 7d195c43..41cebf32 100644 --- a/mautrix/util/ffmpeg.py +++ b/mautrix/util/ffmpeg.py @@ -84,7 +84,7 @@ async def probe_path( err_text = stderr.decode("utf-8") if stderr else f"unknown ({proc.returncode})" raise ConverterError(f"ffprobe error: {err_text}") elif stderr and logger: - logger.warn(f"ffprobe warning: {stderr.decode('utf-8')}") + logger.warning(f"ffprobe warning: {stderr.decode('utf-8')}") return json.loads(stdout) @@ -182,7 +182,7 @@ async def convert_path( err_text = stderr.decode("utf-8") if stderr else f"unknown ({proc.returncode})" raise ConverterError(f"ffmpeg error: {err_text}") elif stderr and logger: - logger.warn(f"ffmpeg warning: {stderr.decode('utf-8')}") + logger.warning(f"ffmpeg warning: {stderr.decode('utf-8')}") if remove_input and isinstance(input_file, Path): input_file.unlink(missing_ok=True) return stdout if output_file == "-" else output_file From 41608a8ba9c7ffe636bd20e5d1ccbbb5c6ae2385 Mon Sep 17 00:00:00 2001 From: Malte E Date: Mon, 20 Feb 2023 15:01:09 +0100 Subject: [PATCH 286/456] block parallel invites to the same room --- mautrix/bridge/matrix.py | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/mautrix/bridge/matrix.py b/mautrix/bridge/matrix.py index 1ba2ac7c..08382abc 100644 --- a/mautrix/bridge/matrix.py +++ b/mautrix/bridge/matrix.py @@ -5,6 +5,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import annotations +from collections import defaultdict import asyncio import logging import sys @@ -143,6 +144,7 @@ class BaseMatrixHandler: media_config: MediaRepoConfig versions: VersionsResponse minimum_spec_version: Version = SpecVersions.V11 + room_locks: dict[str, asyncio.Lock] user_id_prefix: str user_id_suffix: str @@ -159,6 +161,7 @@ def __init__( self.media_config = MediaRepoConfig(upload_size=50 * 1024 * 1024) self.versions = VersionsResponse.deserialize({"versions": ["v1.3"]}) self.az.matrix_event_handler(self.int_handle_event) + self.room_locks = defaultdict(asyncio.Lock) self.e2ee = None self.require_e2ee = False @@ -400,19 +403,20 @@ async def handle_puppet_invite( await intent.leave_room(room_id, reason="You're not allowed to invite this ghost.") return - portal = await self.bridge.get_portal(room_id) - if portal: - try: - await portal.handle_matrix_invite(invited_by, puppet) - except br.RejectMatrixInvite as e: - await intent.leave_room(room_id, reason=e.message) - except br.IgnoreMatrixInvite: - pass + async with self.room_locks[room_id]: + portal = await self.bridge.get_portal(room_id) + if portal: + try: + await portal.handle_matrix_invite(invited_by, puppet) + except br.RejectMatrixInvite as e: + await intent.leave_room(room_id, reason=e.message) + except br.IgnoreMatrixInvite: + pass + else: + await intent.join_room(room_id) + return else: - await intent.join_room(room_id) - return - else: - await self.handle_puppet_nonportal_invite(room_id, puppet, invited_by, evt) + await self.handle_puppet_nonportal_invite(room_id, puppet, invited_by, evt) async def handle_invite( self, room_id: RoomID, user_id: UserID, invited_by: br.BaseUser, evt: StateEvent From 06adf2f6d71b80b072eb709ab149d7dfb15e4064 Mon Sep 17 00:00:00 2001 From: Nick Mills-Barrett Date: Thu, 2 Mar 2023 19:30:27 +0400 Subject: [PATCH 287/456] Add proxy util from IG/FB bridges (#134) * Add proxy util from IG/FB bridges * Add comment about blocking request --- mautrix/util/proxy.py | 113 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100644 mautrix/util/proxy.py diff --git a/mautrix/util/proxy.py b/mautrix/util/proxy.py new file mode 100644 index 00000000..c73fd81f --- /dev/null +++ b/mautrix/util/proxy.py @@ -0,0 +1,113 @@ +from __future__ import annotations + +from typing import Awaitable, Callable, TypeVar +import asyncio +import json +import logging +import urllib.request + +from aiohttp import ClientConnectionError +from yarl import URL + +from mautrix.util.logging import TraceLogger + +try: + from aiohttp_socks import ProxyConnectionError, ProxyError, ProxyTimeoutError +except ImportError: + + class ProxyError(Exception): + pass + + ProxyConnectionError = ProxyTimeoutError = ProxyError + +RETRYABLE_PROXY_EXCEPTIONS = ( + ProxyError, + ProxyTimeoutError, + ProxyConnectionError, + ClientConnectionError, + ConnectionError, + asyncio.TimeoutError, +) + + +class ProxyHandler: + current_proxy_url: str | None = None + log = logging.getLogger("mau.proxy") + + def __init__(self, api_url: str | None) -> None: + self.api_url = api_url + + def get_proxy_url_from_api(self, reason: str | None = None) -> str | None: + assert self.api_url is not None + + api_url = str(URL(self.api_url).update_query({"reason": reason} if reason else {})) + + # NOTE: using urllib.request to intentionally block the whole bridge until the proxy change applied + request = urllib.request.Request(api_url, method="GET") + self.log.debug("Requesting proxy from: %s", api_url) + + try: + with urllib.request.urlopen(request) as f: + response = json.loads(f.read().decode()) + except Exception: + self.log.exception("Failed to retrieve proxy from API") + else: + return response["proxy_url"] + + return None + + def update_proxy_url(self, reason: str | None = None) -> bool: + old_proxy = self.current_proxy_url + new_proxy = None + + if self.api_url is not None: + new_proxy = self.get_proxy_url_from_api(reason) + else: + new_proxy = urllib.request.getproxies().get("http") + + if old_proxy != new_proxy: + self.log.debug("Set new proxy URL: %s", new_proxy) + self.current_proxy_url = new_proxy + return True + + self.log.debug("Got same proxy URL: %s", new_proxy) + return False + + def get_proxy_url(self) -> str | None: + if not self.current_proxy_url: + self.update_proxy_url() + + return self.current_proxy_url + + +T = TypeVar("T") + + +async def proxy_with_retry( + name: str, + func: Callable[[], Awaitable[T]], + logger: TraceLogger, + proxy_handler: ProxyHandler, + on_proxy_change: Callable[[], Awaitable[None]], + max_retries: int = 10, +) -> T: + errors = 0 + + while True: + try: + return await func() + except RETRYABLE_PROXY_EXCEPTIONS as e: + errors += 1 + if errors > max_retries: + raise + wait = min(errors * 10, 60) + logger.warning( + "%s while trying to %s, retrying in %d seconds", + e.__class__.__name__, + name, + wait, + ) + if errors > 1 and proxy_handler.update_proxy_url( + f"{e.__class__.__name__} while trying to {name}" + ): + await on_proxy_change() From 090f33361fd7925331c6a3f71a70317ae647d394 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 4 Mar 2023 14:57:49 +0200 Subject: [PATCH 288/456] Add default value for max upload size. Closes #133 --- mautrix/types/media.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/types/media.py b/mautrix/types/media.py index 72aa6c4f..0b277382 100644 --- a/mautrix/types/media.py +++ b/mautrix/types/media.py @@ -20,7 +20,7 @@ class MediaRepoConfig(SerializableAttrs): https://spec.matrix.org/v1.2/client-server-api/#get_matrixmediav3config """ - upload_size: int = field(json="m.upload.size") + upload_size: int = field(default=50 * 1024 * 1024, json="m.upload.size") @dataclass From eee6d1b599c79d0964815293074fbf2287e09fd6 Mon Sep 17 00:00:00 2001 From: Nick Mills-Barrett Date: Tue, 7 Mar 2023 15:37:21 +0400 Subject: [PATCH 289/456] More `proxy_with_retry` helper arguments (#136) * Add `min_wait_seconds` argument to `proxy_with_retry` helper * Add `retryable_exceptions` argument to `proxy_with_retry` helper This allows non-aiohttp calls to use the helper to retry proxy related requests. * Stricter type for `retryable_exceptions` Co-authored-by: Tulir Asokan --------- Co-authored-by: Tulir Asokan --- mautrix/util/proxy.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mautrix/util/proxy.py b/mautrix/util/proxy.py index c73fd81f..57364c2c 100644 --- a/mautrix/util/proxy.py +++ b/mautrix/util/proxy.py @@ -90,17 +90,19 @@ async def proxy_with_retry( proxy_handler: ProxyHandler, on_proxy_change: Callable[[], Awaitable[None]], max_retries: int = 10, + min_wait_seconds: int = 60, + retryable_exceptions: tuple[Exception] = RETRYABLE_PROXY_EXCEPTIONS, ) -> T: errors = 0 while True: try: return await func() - except RETRYABLE_PROXY_EXCEPTIONS as e: + except retryable_exceptions as e: errors += 1 if errors > max_retries: raise - wait = min(errors * 10, 60) + wait = min(errors * 10, min_wait_seconds) logger.warning( "%s while trying to %s, retrying in %d seconds", e.__class__.__name__, From 741995e17ada6c18c539059e132aec0861d942ec Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 7 Mar 2023 14:15:43 +0200 Subject: [PATCH 290/456] Bump version to 0.19.5 --- CHANGELOG.md | 10 ++++++++++ mautrix/__init__.py | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 70304b94..f1e0a13c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,13 @@ +## v0.19.5 (2023-03-07) + +* *(util.proxy)* Added utility for dynamic proxies (from mautrix-instagram/facebook). +* *(types)* Added default value for `upload_size` in `MediaRepoConfig` as the + field is optional in the spec. +* *(bridge)* Changed ghost invite handling to only process one per room at a time + (thanks to [@maltee1] in [#132]). + +[#132]: https://github.com/mautrix/python/pull/132 + ## v0.19.4 (2023-02-12) * *(types)* Changed `set_thread_parent` to inherit the existing thread parent diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 02a72acd..2bf008d5 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.19.4" +__version__ = "0.19.5" __author__ = "Tulir Asokan " __all__ = [ "api", From 03f54e2be035c5ec711ac7a72ee7130cc0b9fa68 Mon Sep 17 00:00:00 2001 From: Nick Mills-Barrett Date: Sat, 11 Mar 2023 18:28:38 +0400 Subject: [PATCH 291/456] Fix wait second handling in `proxy_with_retry` (#137) * Fix wait second handling in `proxy_with_retry` Adds new argument `max_wait_seconds` which actually was the broken behaviour of the previous `min_wait_seconds` which is now correctly handled. * Add `multiply_wait_seconds` argument --- mautrix/util/proxy.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/mautrix/util/proxy.py b/mautrix/util/proxy.py index 57364c2c..5edd6531 100644 --- a/mautrix/util/proxy.py +++ b/mautrix/util/proxy.py @@ -90,7 +90,9 @@ async def proxy_with_retry( proxy_handler: ProxyHandler, on_proxy_change: Callable[[], Awaitable[None]], max_retries: int = 10, - min_wait_seconds: int = 60, + min_wait_seconds: int = 0, + max_wait_seconds: int = 60, + multiply_wait_seconds: int = 10, retryable_exceptions: tuple[Exception] = RETRYABLE_PROXY_EXCEPTIONS, ) -> T: errors = 0 @@ -102,7 +104,9 @@ async def proxy_with_retry( errors += 1 if errors > max_retries: raise - wait = min(errors * 10, min_wait_seconds) + wait = errors * multiply_wait_seconds + wait = max(wait, min_wait_seconds) + wait = min(wait, max_wait_seconds) logger.warning( "%s while trying to %s, retrying in %d seconds", e.__class__.__name__, From 41b288a9dae85f42e8a62c53690a0ee1b3bca01b Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 13 Mar 2023 15:32:58 +0200 Subject: [PATCH 292/456] Log event ID and source in member change session invalidations --- mautrix/crypto/machine.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mautrix/crypto/machine.py b/mautrix/crypto/machine.py index 3e845b44..ed820c79 100644 --- a/mautrix/crypto/machine.py +++ b/mautrix/crypto/machine.py @@ -171,9 +171,11 @@ async def handle_member_event(self, evt: StateEvent) -> None: } if prev == cur or ignored_changes.get(prev) == cur: return + src = getattr(evt, "source", None) self.log.debug( f"Got membership state event in {evt.room_id} changing {evt.state_key} from " - f"{prev} to {cur}, invalidating group session" + f"{prev} to {cur} (event ID: {evt.event_id}, sync source: {src}), " + "invalidating group session" ) await self.crypto_store.remove_outbound_group_session(evt.room_id) From 72fb0f61f2a8cf8f062ddb8bacaa98ef4cbd0aa5 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 13 Mar 2023 15:50:28 +0200 Subject: [PATCH 293/456] Check cache before invalidating group sessions on member event --- mautrix/client/state_store/abstract.py | 3 +++ mautrix/crypto/machine.py | 13 +++++++++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/mautrix/client/state_store/abstract.py b/mautrix/client/state_store/abstract.py index e241d8f1..3d37087d 100644 --- a/mautrix/client/state_store/abstract.py +++ b/mautrix/client/state_store/abstract.py @@ -143,6 +143,9 @@ async def update_state(self, evt: StateEvent) -> None: if evt.type == EventType.ROOM_POWER_LEVELS: await self.set_power_levels(evt.room_id, evt.content) elif evt.type == EventType.ROOM_MEMBER: + evt.unsigned["mautrix_prev_membership"] = await self.get_member( + evt.room_id, UserID(evt.state_key) + ) await self.set_member(evt.room_id, UserID(evt.state_key), evt.content) elif evt.type == EventType.ROOM_ENCRYPTION: await self.set_encryption_info(evt.room_id, evt.content) diff --git a/mautrix/crypto/machine.py b/mautrix/crypto/machine.py index ed820c79..fe0ef3fa 100644 --- a/mautrix/crypto/machine.py +++ b/mautrix/crypto/machine.py @@ -18,6 +18,7 @@ DeviceOTKCount, EncryptionAlgorithm, EventType, + Member, Membership, StateEvent, ToDeviceEvent, @@ -172,10 +173,18 @@ async def handle_member_event(self, evt: StateEvent) -> None: if prev == cur or ignored_changes.get(prev) == cur: return src = getattr(evt, "source", None) + prev_cache = evt.unsigned.get("mautrix_prev_membership") + if isinstance(prev_cache, Member) and prev_cache.membership == cur: + self.log.debug( + f"Got duplicate membership state event in {evt.room_id} changing {evt.state_key} " + f"from {prev} to {cur}, cached state was {prev_cache} (event ID: {evt.event_id}, " + f"sync source: {src})" + ) + return self.log.debug( f"Got membership state event in {evt.room_id} changing {evt.state_key} from " - f"{prev} to {cur} (event ID: {evt.event_id}, sync source: {src}), " - "invalidating group session" + f"{prev} to {cur} (event ID: {evt.event_id}, sync source: {src}, " + f"cached: {prev_cache.membership}), invalidating group session" ) await self.crypto_store.remove_outbound_group_session(evt.room_id) From 177e5685660e45480907c8e62a633253ce880738 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 13 Mar 2023 15:50:36 +0200 Subject: [PATCH 294/456] Add extra check to cross signing key parser --- mautrix/types/crypto.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mautrix/types/crypto.py b/mautrix/types/crypto.py index 821599de..3bf7e96b 100644 --- a/mautrix/types/crypto.py +++ b/mautrix/types/crypto.py @@ -71,6 +71,8 @@ def first_ed25519_key(self) -> Optional[SigningKey]: return self.first_key_with_algorithm(EncryptionKeyAlgorithm.ED25519) def first_key_with_algorithm(self, alg: EncryptionKeyAlgorithm) -> Optional[SigningKey]: + if not self.keys: + return None try: return next(key for key_id, key in self.keys.items() if key_id.algorithm == alg) except StopIteration: From 718b7bb959b320707953dbb53f33fc5c0e1dfd55 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 13 Mar 2023 15:56:25 +0200 Subject: [PATCH 295/456] Add missing check --- mautrix/crypto/machine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/crypto/machine.py b/mautrix/crypto/machine.py index fe0ef3fa..f9884f7b 100644 --- a/mautrix/crypto/machine.py +++ b/mautrix/crypto/machine.py @@ -184,7 +184,7 @@ async def handle_member_event(self, evt: StateEvent) -> None: self.log.debug( f"Got membership state event in {evt.room_id} changing {evt.state_key} from " f"{prev} to {cur} (event ID: {evt.event_id}, sync source: {src}, " - f"cached: {prev_cache.membership}), invalidating group session" + f"cached: {prev_cache.membership if prev_cache else None}), invalidating group session" ) await self.crypto_store.remove_outbound_group_session(evt.room_id) From 04072cb2f4a00e6e1b02e16389720a94aad71f0f Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 13 Mar 2023 15:59:26 +0200 Subject: [PATCH 296/456] Bump version to 0.19.6 --- CHANGELOG.md | 7 +++++++ mautrix/__init__.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f1e0a13c..087ffefc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## v0.19.6 (2023-03-13) + +* *(crypto)* Added cache checks to prevent invalidating group session when the + server sends a duplicate member event in /sync. +* *(util.proxy)* Fixed `min_wait_seconds` behavior and added `max_wait_seconds` + and `multiply_wait_seconds` to `proxy_with_retry`. + ## v0.19.5 (2023-03-07) * *(util.proxy)* Added utility for dynamic proxies (from mautrix-instagram/facebook). diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 2bf008d5..3fd06e98 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.19.5" +__version__ = "0.19.6" __author__ = "Tulir Asokan " __all__ = [ "api", From 22d9125136ec2a3f577771c7c1f85f63a59da2ab Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 22 Mar 2023 22:07:27 +0200 Subject: [PATCH 297/456] Resolve trust when checking if key sharing is allowed --- mautrix/bridge/e2ee.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mautrix/bridge/e2ee.py b/mautrix/bridge/e2ee.py index 516e54bd..804792c1 100644 --- a/mautrix/bridge/e2ee.py +++ b/mautrix/bridge/e2ee.py @@ -132,9 +132,9 @@ async def allow_key_share(self, device: DeviceIdentity, request: RequestedKeyInf f"Rejecting key request from blacklisted device " f"{device.user_id}/{device.device_id}", code=RoomKeyWithheldCode.BLACKLISTED, - reason="You have been blacklisted by this device", + reason="Your device has been blacklisted by the bridge", ) - elif device.trust >= self.crypto.share_keys_min_trust: + elif await self.crypto.resolve_trust(device) >= self.crypto.share_keys_min_trust: portal = await self.bridge.get_portal(request.room_id) if portal is None: raise RejectKeyShare( @@ -161,7 +161,7 @@ async def allow_key_share(self, device: DeviceIdentity, request: RequestedKeyInf f"Rejecting key request from unverified device " f"{device.user_id}/{device.device_id}", code=RoomKeyWithheldCode.UNVERIFIED, - reason="You have not been verified by this device", + reason="Your device is not trusted by the bridge", ) def _ignore_user(self, user_id: str) -> bool: From 423d42a065e79d7c9607412e302f6821c6692354 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 22 Mar 2023 22:16:17 +0200 Subject: [PATCH 298/456] Also resolve trust in non-bridge key share check --- mautrix/crypto/key_share.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/crypto/key_share.py b/mautrix/crypto/key_share.py index 58525bbf..8d6b6e73 100644 --- a/mautrix/crypto/key_share.py +++ b/mautrix/crypto/key_share.py @@ -76,7 +76,7 @@ async def default_allow_key_share( code=RoomKeyWithheldCode.BLACKLISTED, reason="You have been blacklisted by this device", ) - elif device.trust >= self.share_keys_min_trust: + elif await self.resolve_trust(device) >= self.share_keys_min_trust: self.log.debug(f"Accepting key request from trusted device {device.device_id}") return True else: From 0b674872c228f70b41d6443f665515caee828463 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 22 Mar 2023 22:17:16 +0200 Subject: [PATCH 299/456] Bump version to 0.19.7 --- CHANGELOG.md | 5 +++++ mautrix/__init__.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 087ffefc..24192de4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +## v0.19.7 (2023-03-22) + +* *(bridge, crypto)* Fixed key sharing trust checker not resolving cross-signing + signatures when minimum trust level is set to cross-signed. + ## v0.19.6 (2023-03-13) * *(crypto)* Added cache checks to prevent invalidating group session when the diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 3fd06e98..53ed1518 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.19.6" +__version__ = "0.19.7" __author__ = "Tulir Asokan " __all__ = [ "api", From 4d4441b910da3f06b7ce8eadc6c859d60b977863 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 23 Mar 2023 16:27:22 +0200 Subject: [PATCH 300/456] Sync crypto store schema with mautrix-go --- mautrix/crypto/store/asyncpg/store.py | 13 +- mautrix/crypto/store/asyncpg/upgrade.py | 189 +++++++++++++++++++++--- mautrix/errors/__init__.py | 1 + mautrix/errors/crypto.py | 6 + 4 files changed, 188 insertions(+), 21 deletions(-) diff --git a/mautrix/crypto/store/asyncpg/store.py b/mautrix/crypto/store/asyncpg/store.py index 8609e12d..51985caf 100644 --- a/mautrix/crypto/store/asyncpg/store.py +++ b/mautrix/crypto/store/asyncpg/store.py @@ -12,6 +12,7 @@ from mautrix.client.state_store import SyncStore from mautrix.client.state_store.asyncpg import PgStateStore +from mautrix.errors import GroupSessionWithheldError from mautrix.types import ( CrossSigner, CrossSigningUsage, @@ -117,7 +118,7 @@ async def put_account(self, account: OlmAccount) -> None: await self.db.execute( q, self.account_id, - self._device_id, + self._device_id or "", account.shared, self._sync_token or "", pickle, @@ -236,6 +237,10 @@ async def put_group_session( INSERT INTO crypto_megolm_inbound_session ( session_id, sender_key, signing_key, room_id, session, forwarding_chains, account_id ) VALUES ($1, $2, $3, $4, $5, $6, $7) + ON CONFLICT (session_id, account_id) DO UPDATE + SET withheld_code=NULL, withheld_reason=NULL, sender_key=excluded.sender_key, + signing_key=excluded.signing_key, room_id=excluded.room_id, session=excluded.session, + forwarding_chains=excluded.forwarding_chains """ try: await self.db.execute( @@ -255,13 +260,15 @@ async def get_group_session( self, room_id: RoomID, session_id: SessionID ) -> InboundGroupSession | None: q = """ - SELECT sender_key, signing_key, session, forwarding_chains + SELECT sender_key, signing_key, session, forwarding_chains, withheld_code FROM crypto_megolm_inbound_session WHERE room_id=$1 AND session_id=$2 AND account_id=$3 """ row = await self.db.fetchrow(q, room_id, session_id, self.account_id) if row is None: return None + if row["withheld_code"] is not None: + raise GroupSessionWithheldError(session_id, row["withheld_code"]) forwarding_chain = row["forwarding_chains"].split(",") if row["forwarding_chains"] else [] return InboundGroupSession.from_pickle( row["session"], @@ -275,7 +282,7 @@ async def get_group_session( async def has_group_session(self, room_id: RoomID, session_id: SessionID) -> bool: q = """ SELECT COUNT(session) FROM crypto_megolm_inbound_session - WHERE room_id=$1 AND session_id=$2 AND account_id=$3 + WHERE room_id=$1 AND session_id=$2 AND account_id=$3 AND session IS NOT NULL """ count = await self.db.fetchval(q, room_id, session_id, self.account_id) return count > 0 diff --git a/mautrix/crypto/store/asyncpg/upgrade.py b/mautrix/crypto/store/asyncpg/upgrade.py index 7a55c10d..b0f2299c 100644 --- a/mautrix/crypto/store/asyncpg/upgrade.py +++ b/mautrix/crypto/store/asyncpg/upgrade.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022 Tulir Asokan +# Copyright (c) 2023 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this @@ -16,15 +16,15 @@ ) -@upgrade_table.register(description="Latest revision", upgrades_to=6) -async def upgrade_blank_to_v4(conn: Connection) -> None: +@upgrade_table.register(description="Latest revision", upgrades_to=8) +async def upgrade_blank_to_latest(conn: Connection) -> None: await conn.execute( """CREATE TABLE IF NOT EXISTS crypto_account ( - account_id TEXT PRIMARY KEY, - device_id TEXT, - shared BOOLEAN NOT NULL, - sync_token TEXT NOT NULL, - account bytea NOT NULL + account_id TEXT PRIMARY KEY, + device_id TEXT NOT NULL, + shared BOOLEAN NOT NULL, + sync_token TEXT NOT NULL, + account bytea NOT NULL )""" ) await conn.execute( @@ -68,16 +68,19 @@ async def upgrade_blank_to_v4(conn: Connection) -> None: ) await conn.execute( """CREATE TABLE IF NOT EXISTS crypto_megolm_inbound_session ( - account_id TEXT, - session_id CHAR(43), - sender_key CHAR(43) NOT NULL, - signing_key CHAR(43) NOT NULL, - room_id TEXT NOT NULL, - session bytea NOT NULL, - forwarding_chains TEXT NOT NULL, + account_id TEXT, + session_id CHAR(43), + sender_key CHAR(43) NOT NULL, + signing_key CHAR(43), + room_id TEXT NOT NULL, + session bytea, + forwarding_chains TEXT, + withheld_code TEXT, + withheld_reason TEXT, PRIMARY KEY (account_id, session_id) )""" ) + # TODO chnge max_age to BIGINT await conn.execute( """CREATE TABLE IF NOT EXISTS crypto_megolm_outbound_session ( account_id TEXT, @@ -97,8 +100,10 @@ async def upgrade_blank_to_v4(conn: Connection) -> None: """CREATE TABLE crypto_cross_signing_keys ( user_id TEXT, usage TEXT, - key CHAR(43), - first_seen_key CHAR(43), + key CHAR(43) NOT NULL, + + first_seen_key CHAR(43) NOT NULL, + PRIMARY KEY (user_id, usage) )""" ) @@ -108,7 +113,7 @@ async def upgrade_blank_to_v4(conn: Connection) -> None: signed_key TEXT, signer_user_id TEXT, signer_key TEXT, - signature TEXT, + signature CHAR(88) NOT NULL, PRIMARY KEY (signed_user_id, signed_key, signer_user_id, signer_key) )""" ) @@ -250,3 +255,151 @@ async def upgrade_v6(conn: Connection) -> None: await conn.execute("UPDATE crypto_device SET trust=300 WHERE trust=1") # verified await conn.execute("UPDATE crypto_device SET trust=-100 WHERE trust=2") # blacklisted await conn.execute("UPDATE crypto_device SET trust=0 WHERE trust=3") # ignored -> unset + + +@upgrade_table.register( + description="Synchronize schema with mautrix-go", upgrades_to=8, transaction=False +) +async def upgrade_v8(conn: Connection, scheme: Scheme) -> None: + if scheme == Scheme.POSTGRES: + async with conn.transaction(): + await upgrade_v8_postgres(conn) + else: + await upgrade_v8_sqlite(conn) + + +async def upgrade_v8_postgres(conn: Connection) -> None: + await conn.execute("UPDATE crypto_account SET device_id='' WHERE device_id IS NULL") + await conn.execute("ALTER TABLE crypto_account ALTER COLUMN device_id SET NOT NULL") + + await conn.execute( + "ALTER TABLE crypto_megolm_inbound_session ALTER COLUMN signing_key DROP NOT NULL" + ) + await conn.execute( + "ALTER TABLE crypto_megolm_inbound_session ALTER COLUMN session DROP NOT NULL" + ) + await conn.execute( + "ALTER TABLE crypto_megolm_inbound_session ALTER COLUMN forwarding_chains DROP NOT NULL" + ) + await conn.execute("ALTER TABLE crypto_megolm_inbound_session ADD COLUMN withheld_code TEXT") + await conn.execute("ALTER TABLE crypto_megolm_inbound_session ADD COLUMN withheld_reason TEXT") + + await conn.execute("DELETE FROM crypto_cross_signing_keys WHERE key IS NULL") + await conn.execute( + "UPDATE crypto_cross_signing_keys SET first_seen_key=key WHERE first_seen_key IS NULL" + ) + await conn.execute("ALTER TABLE crypto_cross_signing_keys ALTER COLUMN key SET NOT NULL") + await conn.execute( + "ALTER TABLE crypto_cross_signing_keys ALTER COLUMN first_seen_key SET NOT NULL" + ) + + await conn.execute("DELETE FROM crypto_cross_signing_signatures WHERE signature IS NULL") + await conn.execute( + "ALTER TABLE crypto_cross_signing_signatures ALTER COLUMN signature SET NOT NULL" + ) + + +async def upgrade_v8_sqlite(conn: Connection) -> None: + await conn.execute("PRAGMA foreign_keys = OFF") + async with conn.transaction(): + await conn.execute( + """CREATE TABLE new_crypto_account ( + account_id TEXT PRIMARY KEY, + device_id TEXT NOT NULL, + shared BOOLEAN NOT NULL, + sync_token TEXT NOT NULL, + account bytea NOT NULL + )""" + ) + await conn.execute( + """ + INSERT INTO new_crypto_account (account_id, device_id, shared, sync_token, account) + SELECT account_id, COALESCE(device_id, ''), shared, sync_token, account + FROM crypto_account + """ + ) + await conn.execute("DROP TABLE crypto_account") + await conn.execute("ALTER TABLE new_crypto_account RENAME TO crypto_account") + + await conn.execute( + """CREATE TABLE new_crypto_megolm_inbound_session ( + account_id TEXT, + session_id CHAR(43), + sender_key CHAR(43) NOT NULL, + signing_key CHAR(43), + room_id TEXT NOT NULL, + session bytea, + forwarding_chains TEXT, + withheld_code TEXT, + withheld_reason TEXT, + PRIMARY KEY (account_id, session_id) + )""" + ) + await conn.execute( + """ + INSERT INTO new_crypto_megolm_inbound_session ( + account_id, session_id, sender_key, signing_key, room_id, session, + forwarding_chains + ) + SELECT account_id, session_id, sender_key, signing_key, room_id, session, + forwarding_chains + FROM crypto_megolm_inbound_session + """ + ) + await conn.execute("DROP TABLE crypto_megolm_inbound_session") + await conn.execute( + "ALTER TABLE new_crypto_megolm_inbound_session RENAME TO crypto_megolm_inbound_session" + ) + + await conn.execute( + """CREATE TABLE new_crypto_cross_signing_keys ( + user_id TEXT, + usage TEXT, + key CHAR(43) NOT NULL, + + first_seen_key CHAR(43) NOT NULL, + + PRIMARY KEY (user_id, usage) + )""" + ) + await conn.execute( + """ + INSERT INTO new_crypto_cross_signing_keys (user_id, usage, key, first_seen_key) + SELECT user_id, usage, key, COALESCE(first_seen_key, key) + FROM crypto_cross_signing_keys + WHERE key IS NOT NULL + """ + ) + await conn.execute("DROP TABLE crypto_cross_signing_keys") + await conn.execute( + "ALTER TABLE new_crypto_cross_signing_keys RENAME TO crypto_cross_signing_keys" + ) + + await conn.execute( + """CREATE TABLE new_crypto_cross_signing_signatures ( + signed_user_id TEXT, + signed_key TEXT, + signer_user_id TEXT, + signer_key TEXT, + signature CHAR(88) NOT NULL, + PRIMARY KEY (signed_user_id, signed_key, signer_user_id, signer_key) + )""" + ) + await conn.execute( + """ + INSERT INTO new_crypto_cross_signing_signatures ( + signed_user_id, signed_key, signer_user_id, signer_key, signature + ) + SELECT signed_user_id, signed_key, signer_user_id, signer_key, signature + FROM crypto_cross_signing_signatures + WHERE signature IS NOT NULL + """ + ) + await conn.execute("DROP TABLE crypto_cross_signing_signatures") + await conn.execute( + "ALTER TABLE new_crypto_cross_signing_signatures " + "RENAME TO crypto_cross_signing_signatures" + ) + + await conn.execute("PRAGMA foreign_key_check") + await conn.execute("PRAGMA foreign_keys = ON") diff --git a/mautrix/errors/__init__.py b/mautrix/errors/__init__.py index fdec6e3a..d3646a37 100644 --- a/mautrix/errors/__init__.py +++ b/mautrix/errors/__init__.py @@ -6,6 +6,7 @@ DeviceValidationError, DuplicateMessageIndex, EncryptionError, + GroupSessionWithheldError, MatchingSessionDecryptionError, MismatchingRoomError, SessionNotFound, diff --git a/mautrix/errors/crypto.py b/mautrix/errors/crypto.py index 97592b05..4a65048c 100644 --- a/mautrix/errors/crypto.py +++ b/mautrix/errors/crypto.py @@ -36,6 +36,12 @@ class MatchingSessionDecryptionError(DecryptionError): pass +class GroupSessionWithheldError(DecryptionError): + def __init__(self, session_id: SessionID, withheld_code: str) -> None: + super().__init__(f"Session ID {session_id} was withheld ({withheld_code})") + self.withheld_code = withheld_code + + class SessionNotFound(DecryptionError): def __init__(self, session_id: SessionID, sender_key: IdentityKey | None = None) -> None: super().__init__( From 27732d4b7a42c53af72af2413ab2ee1b09c001fd Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 23 Mar 2023 16:35:29 +0200 Subject: [PATCH 301/456] Change max_age to BIGINT milliseconds --- mautrix/crypto/store/asyncpg/store.py | 9 ++------- mautrix/crypto/store/asyncpg/upgrade.py | 12 +++++++++--- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/mautrix/crypto/store/asyncpg/store.py b/mautrix/crypto/store/asyncpg/store.py index 51985caf..cb15be77 100644 --- a/mautrix/crypto/store/asyncpg/store.py +++ b/mautrix/crypto/store/asyncpg/store.py @@ -289,9 +289,7 @@ async def has_group_session(self, room_id: RoomID, session_id: SessionID) -> boo async def add_outbound_group_session(self, session: OutboundGroupSession) -> None: pickle = session.pickle(self.pickle_key) - max_age = session.max_age - if self.db.scheme == Scheme.SQLITE: - max_age = max_age.total_seconds() + max_age = int(session.max_age.total_seconds() * 1000) q = """ INSERT INTO crypto_megolm_outbound_session ( room_id, session_id, session, shared, max_messages, message_count, @@ -341,9 +339,6 @@ async def get_outbound_group_session(self, room_id: RoomID) -> OutboundGroupSess row = await self.db.fetchrow(q, room_id, self.account_id) if row is None: return None - max_age = row["max_age"] - if self.db.scheme == Scheme.SQLITE: - max_age = timedelta(seconds=max_age) return OutboundGroupSession.from_pickle( row["session"], passphrase=self.pickle_key, @@ -351,7 +346,7 @@ async def get_outbound_group_session(self, room_id: RoomID) -> OutboundGroupSess shared=row["shared"], max_messages=row["max_messages"], message_count=row["message_count"], - max_age=max_age, + max_age=timedelta(milliseconds=row["max_age"]), use_time=row["last_used"], creation_time=row["created_at"], ) diff --git a/mautrix/crypto/store/asyncpg/upgrade.py b/mautrix/crypto/store/asyncpg/upgrade.py index b0f2299c..9b8ce779 100644 --- a/mautrix/crypto/store/asyncpg/upgrade.py +++ b/mautrix/crypto/store/asyncpg/upgrade.py @@ -80,7 +80,6 @@ async def upgrade_blank_to_latest(conn: Connection) -> None: PRIMARY KEY (account_id, session_id) )""" ) - # TODO chnge max_age to BIGINT await conn.execute( """CREATE TABLE IF NOT EXISTS crypto_megolm_outbound_session ( account_id TEXT, @@ -90,7 +89,7 @@ async def upgrade_blank_to_latest(conn: Connection) -> None: shared BOOLEAN NOT NULL, max_messages INTEGER NOT NULL, message_count INTEGER NOT NULL, - max_age INTERVAL NOT NULL, + max_age BIGINT NOT NULL, created_at timestamp NOT NULL, last_used timestamp NOT NULL, PRIMARY KEY (account_id, room_id) @@ -167,7 +166,7 @@ async def upgrade_v2(conn: Connection, scheme: Scheme) -> None: shared BOOLEAN NOT NULL, max_messages INTEGER NOT NULL, message_count INTEGER NOT NULL, - max_age INTERVAL NOT NULL, + max_age BIGINT NOT NULL, created_at timestamp NOT NULL, last_used timestamp NOT NULL, PRIMARY KEY (account_id, room_id) @@ -298,6 +297,11 @@ async def upgrade_v8_postgres(conn: Connection) -> None: "ALTER TABLE crypto_cross_signing_signatures ALTER COLUMN signature SET NOT NULL" ) + await conn.execute( + "ALTER TABLE crypto_megolm_outbound_session ALTER COLUMN max_age TYPE BIGINT " + "USING (EXTRACT(EPOCH from max_age)*1000)::int" + ) + async def upgrade_v8_sqlite(conn: Connection) -> None: await conn.execute("PRAGMA foreign_keys = OFF") @@ -351,6 +355,8 @@ async def upgrade_v8_sqlite(conn: Connection) -> None: "ALTER TABLE new_crypto_megolm_inbound_session RENAME TO crypto_megolm_inbound_session" ) + await conn.execute("UPDATE crypto_megolm_outbound_session SET max_age=max_age*1000") + await conn.execute( """CREATE TABLE new_crypto_cross_signing_keys ( user_id TEXT, From 342728365d41cad4541e4b83970130e158c581e3 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 23 Mar 2023 16:40:03 +0200 Subject: [PATCH 302/456] Bump schema version to 9 --- mautrix/crypto/store/asyncpg/upgrade.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/mautrix/crypto/store/asyncpg/upgrade.py b/mautrix/crypto/store/asyncpg/upgrade.py index 9b8ce779..1f1b7e7e 100644 --- a/mautrix/crypto/store/asyncpg/upgrade.py +++ b/mautrix/crypto/store/asyncpg/upgrade.py @@ -257,17 +257,17 @@ async def upgrade_v6(conn: Connection) -> None: @upgrade_table.register( - description="Synchronize schema with mautrix-go", upgrades_to=8, transaction=False + description="Synchronize schema with mautrix-go", upgrades_to=9, transaction=False ) -async def upgrade_v8(conn: Connection, scheme: Scheme) -> None: +async def upgrade_v9(conn: Connection, scheme: Scheme) -> None: if scheme == Scheme.POSTGRES: async with conn.transaction(): - await upgrade_v8_postgres(conn) + await upgrade_v9_postgres(conn) else: - await upgrade_v8_sqlite(conn) + await upgrade_v9_sqlite(conn) -async def upgrade_v8_postgres(conn: Connection) -> None: +async def upgrade_v9_postgres(conn: Connection) -> None: await conn.execute("UPDATE crypto_account SET device_id='' WHERE device_id IS NULL") await conn.execute("ALTER TABLE crypto_account ALTER COLUMN device_id SET NOT NULL") @@ -303,7 +303,7 @@ async def upgrade_v8_postgres(conn: Connection) -> None: ) -async def upgrade_v8_sqlite(conn: Connection) -> None: +async def upgrade_v9_sqlite(conn: Connection) -> None: await conn.execute("PRAGMA foreign_keys = OFF") async with conn.transaction(): await conn.execute( From 9213dd0cdad42ed761aa315e52ae327b695564a3 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 23 Mar 2023 16:47:37 +0200 Subject: [PATCH 303/456] Fix upgrade index --- mautrix/crypto/store/asyncpg/upgrade.py | 11 +++++++++++ mautrix/util/async_db/upgrade.py | 4 ++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/mautrix/crypto/store/asyncpg/upgrade.py b/mautrix/crypto/store/asyncpg/upgrade.py index 1f1b7e7e..8e8da45a 100644 --- a/mautrix/crypto/store/asyncpg/upgrade.py +++ b/mautrix/crypto/store/asyncpg/upgrade.py @@ -267,6 +267,17 @@ async def upgrade_v9(conn: Connection, scheme: Scheme) -> None: await upgrade_v9_sqlite(conn) +# These two are never used because the previous one jumps from 6 to 9. +@upgrade_table.register +async def upgrade_noop_7_to_8(_: Connection) -> None: + pass + + +@upgrade_table.register +async def upgrade_noop_8_to_9(_: Connection) -> None: + pass + + async def upgrade_v9_postgres(conn: Connection) -> None: await conn.execute("UPDATE crypto_account SET device_id='' WHERE device_id IS NULL") await conn.execute("ALTER TABLE crypto_account ALTER COLUMN device_id SET NOT NULL") diff --git a/mautrix/util/async_db/upgrade.py b/mautrix/util/async_db/upgrade.py index d69dac56..9e593ece 100644 --- a/mautrix/util/async_db/upgrade.py +++ b/mautrix/util/async_db/upgrade.py @@ -21,7 +21,7 @@ UpgradeWithoutScheme = Callable[[LoggingConnection], Awaitable[Optional[int]]] -async def noop_upgrade(_: LoggingConnection) -> None: +async def noop_upgrade(_: LoggingConnection, _2: Scheme) -> None: pass @@ -178,6 +178,6 @@ def _find_upgrade_table(fn: Upgrade) -> UpgradeTable: def register_upgrade(index: int = -1, description: str = "") -> Callable[[Upgrade], Upgrade]: def actually_register(fn: Upgrade) -> Upgrade: - return _find_upgrade_table(fn).register(index, description, fn) + return _find_upgrade_table(fn).register(fn, index=index, description=description) return actually_register From 97f5e87b3c2e9aa3efcd25ecdf68624743098425 Mon Sep 17 00:00:00 2001 From: Sumner Evans Date: Wed, 5 Apr 2023 10:48:29 -0600 Subject: [PATCH 304/456] client/user_data: add beeper_update_profile method Signed-off-by: Sumner Evans --- mautrix/client/api/user_data.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/mautrix/client/api/user_data.py b/mautrix/client/api/user_data.py index f37cb769..4c3d437a 100644 --- a/mautrix/client/api/user_data.py +++ b/mautrix/client/api/user_data.py @@ -5,6 +5,8 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import annotations +from typing import Any + from mautrix.api import Method, Path from mautrix.errors import MatrixResponseError, MNotFound from mautrix.types import ContentURI, Member, SerializerError, User, UserID, UserSearchResults @@ -170,3 +172,16 @@ async def get_profile(self, user_id: UserID) -> Member: raise MatrixResponseError("Invalid member in response") from e # endregion + + # region Beeper Custom Fields API + + async def beeper_update_profile(self, custom_fields: dict[str, Any]) -> None: + """ + Set custom fields on the user's profile. Only works on Hungryserv. + + Args: + custom_fields: A dictionary of fields to set in the custom content of the profile. + """ + await self.api.request(Method.PATCH, Path.v3.profile[self.mxid], custom_fields) + + # endregion From ddf796ffc10b5573b3aee15560c238fad01ec6d3 Mon Sep 17 00:00:00 2001 From: Sumner Evans Date: Wed, 5 Apr 2023 14:31:58 -0600 Subject: [PATCH 305/456] bridge/init_as_bot: set Beeper contact info fields Signed-off-by: Sumner Evans --- mautrix/bridge/bridge.py | 10 ++++++++++ mautrix/bridge/matrix.py | 11 +++++++++++ 2 files changed, 21 insertions(+) diff --git a/mautrix/bridge/bridge.py b/mautrix/bridge/bridge.py index 7005d775..396a04a1 100644 --- a/mautrix/bridge/bridge.py +++ b/mautrix/bridge/bridge.py @@ -315,6 +315,16 @@ def is_bridge_ghost(self, user_id: UserID) -> bool: async def count_logged_in_users(self) -> int: return 0 + @staticmethod + @abstractmethod + def get_beeper_service_name() -> str: + pass + + @staticmethod + @abstractmethod + def get_beeper_network_name() -> str: + pass + async def manhole_global_namespace(self, user_id: UserID) -> dict[str, Any]: own_user = await self.get_user(user_id, create=False) try: diff --git a/mautrix/bridge/matrix.py b/mautrix/bridge/matrix.py index 08382abc..aba84567 100644 --- a/mautrix/bridge/matrix.py +++ b/mautrix/bridge/matrix.py @@ -272,6 +272,17 @@ async def init_as_bot(self) -> None: except Exception: self.log.exception("Failed to set bot avatar") + if self.bridge.homeserver_software.is_hungry: + self.log.debug("Setting contact info on the appservice bot") + await self.az.intent.beeper_update_profile( + { + "com.beeper.bridge.service": self.bridge.get_beeper_service_name(), + "com.beeper.bridge.network": self.bridge.get_beeper_network_name(), + "com.beeper.bridge.is_bridge_bot": True, + "com.beeper.bridge.is_bot": True, + } + ) + async def init_encryption(self) -> None: if self.e2ee: await self.e2ee.start() From 0d55eb589e8fa85307f48e13e67305f479f1d13e Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 6 Apr 2023 11:11:41 +0300 Subject: [PATCH 306/456] Disable reply fallback in set_thread_parent --- mautrix/types/event/message.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/mautrix/types/event/message.py b/mautrix/types/event/message.py index 47c4e2e6..021e91f9 100644 --- a/mautrix/types/event/message.py +++ b/mautrix/types/event/message.py @@ -119,7 +119,7 @@ def set_thread_parent( thread_parent.content.get_thread_parent() or self.relates_to.event_id ) if not disable_reply_fallback: - self.set_reply(last_event_in_thread or thread_parent, **kwargs) + self.set_reply(last_event_in_thread or thread_parent, disable_fallback=True, **kwargs) self.relates_to.is_falling_back = True def set_edit(self, edits: Union[EventID, "MessageEvent"]) -> None: @@ -315,12 +315,16 @@ class TextMessageEventContent(BaseMessageEventContent, SerializableAttrs): formatted_body: str = None def set_reply( - self, reply_to: Union["MessageEvent", EventID], *, displayname: Optional[str] = None + self, + reply_to: Union["MessageEvent", EventID], + *, + displayname: Optional[str] = None, + disable_fallback: bool = False, ) -> None: super().set_reply(reply_to) if isinstance(reply_to, str): return - if isinstance(reply_to, MessageEvent): + if isinstance(reply_to, MessageEvent) and not disable_fallback: self.ensure_has_html() if isinstance(reply_to.content, TextMessageEventContent): reply_to.content.trim_reply_fallback() From 251b8fcb9acee184f6d4fabe955072ab8f0fea8f Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 6 Apr 2023 11:22:38 +0300 Subject: [PATCH 307/456] Fix latest schema version --- mautrix/crypto/store/asyncpg/upgrade.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/crypto/store/asyncpg/upgrade.py b/mautrix/crypto/store/asyncpg/upgrade.py index 8e8da45a..789552a1 100644 --- a/mautrix/crypto/store/asyncpg/upgrade.py +++ b/mautrix/crypto/store/asyncpg/upgrade.py @@ -16,7 +16,7 @@ ) -@upgrade_table.register(description="Latest revision", upgrades_to=8) +@upgrade_table.register(description="Latest revision", upgrades_to=9) async def upgrade_blank_to_latest(conn: Connection) -> None: await conn.execute( """CREATE TABLE IF NOT EXISTS crypto_account ( From 9ff58f9af97d6f31abfc42fb4f9069de1d07e3e0 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 6 Apr 2023 16:51:44 +0300 Subject: [PATCH 308/456] Replace service/network name with properties --- mautrix/bridge/bridge.py | 12 ++---------- mautrix/bridge/matrix.py | 6 +++--- 2 files changed, 5 insertions(+), 13 deletions(-) diff --git a/mautrix/bridge/bridge.py b/mautrix/bridge/bridge.py index 396a04a1..d8241038 100644 --- a/mautrix/bridge/bridge.py +++ b/mautrix/bridge/bridge.py @@ -59,6 +59,8 @@ class Bridge(Program, ABC): markdown_version: str manhole: br.commands.manhole.ManholeState | None homeserver_software: HomeserverSoftware + beeper_network_name: str | None = None + beeper_service_name: str | None = None def __init__( self, @@ -315,16 +317,6 @@ def is_bridge_ghost(self, user_id: UserID) -> bool: async def count_logged_in_users(self) -> int: return 0 - @staticmethod - @abstractmethod - def get_beeper_service_name() -> str: - pass - - @staticmethod - @abstractmethod - def get_beeper_network_name() -> str: - pass - async def manhole_global_namespace(self, user_id: UserID) -> dict[str, Any]: own_user = await self.get_user(user_id, create=False) try: diff --git a/mautrix/bridge/matrix.py b/mautrix/bridge/matrix.py index aba84567..6ce3bfde 100644 --- a/mautrix/bridge/matrix.py +++ b/mautrix/bridge/matrix.py @@ -272,12 +272,12 @@ async def init_as_bot(self) -> None: except Exception: self.log.exception("Failed to set bot avatar") - if self.bridge.homeserver_software.is_hungry: + if self.bridge.homeserver_software.is_hungry and self.bridge.beeper_network_name: self.log.debug("Setting contact info on the appservice bot") await self.az.intent.beeper_update_profile( { - "com.beeper.bridge.service": self.bridge.get_beeper_service_name(), - "com.beeper.bridge.network": self.bridge.get_beeper_network_name(), + "com.beeper.bridge.service": self.bridge.beeper_service_name, + "com.beeper.bridge.network": self.bridge.beeper_network_name, "com.beeper.bridge.is_bridge_bot": True, "com.beeper.bridge.is_bot": True, } From 754ec520f3e6b13ec22624ce4c8aa84e87c03d1f Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 6 Apr 2023 16:54:58 +0300 Subject: [PATCH 309/456] Bump version to 0.19.8 --- CHANGELOG.md | 5 +++++ mautrix/__init__.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 24192de4..862da25a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +## v0.19.8 (2023-04-06) + +* *(crypto)* Updated crypto store schema to match mautrix-go. +* *(types)* Fixed `set_thread_parent` adding reply fallbacks to the message body. + ## v0.19.7 (2023-03-22) * *(bridge, crypto)* Fixed key sharing trust checker not resolving cross-signing diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 53ed1518..40300b56 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.19.7" +__version__ = "0.19.8" __author__ = "Tulir Asokan " __all__ = [ "api", From 4860ddf76c77b4624ce200c73fb13675688ef615 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 8 Apr 2023 22:37:13 +0300 Subject: [PATCH 310/456] Fix crypto store migration when there are long-lived megolm sessions --- mautrix/crypto/store/asyncpg/upgrade.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/crypto/store/asyncpg/upgrade.py b/mautrix/crypto/store/asyncpg/upgrade.py index 789552a1..914da23d 100644 --- a/mautrix/crypto/store/asyncpg/upgrade.py +++ b/mautrix/crypto/store/asyncpg/upgrade.py @@ -310,7 +310,7 @@ async def upgrade_v9_postgres(conn: Connection) -> None: await conn.execute( "ALTER TABLE crypto_megolm_outbound_session ALTER COLUMN max_age TYPE BIGINT " - "USING (EXTRACT(EPOCH from max_age)*1000)::int" + "USING (EXTRACT(EPOCH from max_age)*1000)::bigint" ) From 82a9929e089b908c89ca9116e4163303311938d8 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 12 Apr 2023 16:51:32 +0300 Subject: [PATCH 311/456] Bump version to 0.19.9 --- CHANGELOG.md | 5 +++++ mautrix/__init__.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 862da25a..e7cabdd6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +## v0.19.9 (2023-04-12) + +* *(crypto)* Fixed bug in crypto store migration when using outbound sessions + with max age higher than usual. + ## v0.19.8 (2023-04-06) * *(crypto)* Updated crypto store schema to match mautrix-go. diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 40300b56..de48a11a 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.19.8" +__version__ = "0.19.9" __author__ = "Tulir Asokan " __all__ = [ "api", From efd1eaaf6dffb58c9595b77589c2249171314a66 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 12 Apr 2023 20:09:23 +0300 Subject: [PATCH 312/456] Add options to automatically ratchet/delete megolm sessions (#144) --- mautrix/bridge/config.py | 7 ++ mautrix/bridge/e2ee.py | 59 +++++++++++- mautrix/bridge/portal.py | 2 +- mautrix/client/state_store/asyncpg/upgrade.py | 19 +++- mautrix/crypto/__init__.py | 2 +- mautrix/crypto/base.py | 8 ++ mautrix/crypto/decrypt_megolm.py | 81 +++++++++++++--- mautrix/crypto/device_lists.py | 10 ++ mautrix/crypto/encrypt_megolm.py | 63 ++++++++----- mautrix/crypto/key_request.py | 17 +++- mautrix/crypto/machine.py | 53 +++++++++++ mautrix/crypto/sessions.py | 67 +++++++++++++- mautrix/crypto/store/abstract.py | 40 ++++++++ mautrix/crypto/store/asyncpg/store.py | 92 ++++++++++++++++++- mautrix/crypto/store/asyncpg/upgrade.py | 23 ++++- mautrix/crypto/store/memory.py | 24 +++++ mautrix/types/event/beeper.py | 9 +- mautrix/types/event/to_device.py | 16 +++- mautrix/types/event/type.py | 1 + mautrix/types/event/type.pyi | 1 + 20 files changed, 539 insertions(+), 55 deletions(-) diff --git a/mautrix/bridge/config.py b/mautrix/bridge/config.py index 4289b178..f3e7720c 100644 --- a/mautrix/bridge/config.py +++ b/mautrix/bridge/config.py @@ -138,6 +138,13 @@ def do_update(self, helper: ConfigUpdateHelper) -> None: copy("bridge.encryption.default") copy("bridge.encryption.require") copy("bridge.encryption.appservice") + copy("bridge.encryption.delete_keys.delete_outbound_on_ack") + copy("bridge.encryption.delete_keys.dont_store_outbound") + copy("bridge.encryption.delete_keys.ratchet_on_decrypt") + copy("bridge.encryption.delete_keys.delete_fully_used_on_decrypt") + copy("bridge.encryption.delete_keys.delete_prev_on_new_session") + copy("bridge.encryption.delete_keys.delete_on_device_delete") + copy("bridge.encryption.delete_keys.periodically_delete_expired") copy("bridge.encryption.verification_levels.receive") copy("bridge.encryption.verification_levels.send") copy("bridge.encryption.verification_levels.share") diff --git a/mautrix/bridge/e2ee.py b/mautrix/bridge/e2ee.py index 804792c1..95b419f7 100644 --- a/mautrix/bridge/e2ee.py +++ b/mautrix/bridge/e2ee.py @@ -13,7 +13,7 @@ from mautrix.appservice import AppService from mautrix.client import Client, InternalEventType, SyncStore from mautrix.crypto import CryptoStore, OlmMachine, PgCryptoStore, RejectKeyShare, StateStore -from mautrix.errors import EncryptionError, SessionNotFound +from mautrix.errors import EncryptionError, MForbidden, MNotFound, SessionNotFound from mautrix.types import ( JSON, DeviceIdentity, @@ -34,6 +34,7 @@ StateFilter, TrustState, ) +from mautrix.util import background_task from mautrix.util.async_db import Database from mautrix.util.logging import TraceLogger @@ -61,6 +62,7 @@ class EncryptionManager: min_send_trust: TrustState key_sharing_enabled: bool appservice_mode: bool + periodically_delete_expired_keys: bool bridge: br.Bridge az: AppService @@ -68,6 +70,7 @@ class EncryptionManager: _id_suffix: str _share_session_events: dict[RoomID, asyncio.Event] + _key_delete_task: asyncio.Task | None def __init__( self, @@ -100,6 +103,7 @@ def __init__( sync_store=self.crypto_store, log=self.log.getChild("client"), default_retry_count=default_http_retry_count, + state_store=self.bridge.state_store, ) self.crypto = OlmMachine(self.client, self.crypto_store, self.state_store) self.client.add_event_handler(InternalEventType.SYNC_STOPPED, self._exit_on_sync_fail) @@ -115,6 +119,16 @@ def __init__( self.az.device_list_handler = self.crypto.handle_as_device_lists self.az.to_device_handler = self.crypto.handle_as_to_device_event + delete_cfg = bridge.config["bridge.encryption.delete_keys"] + self.crypto.delete_outbound_keys_on_ack = delete_cfg["delete_outbound_on_ack"] + self.crypto.dont_store_outbound_keys = delete_cfg["dont_store_outbound"] + self.crypto.delete_previous_keys_on_receive = delete_cfg["delete_prev_on_new_session"] + self.crypto.ratchet_keys_on_decrypt = delete_cfg["ratchet_on_decrypt"] + self.crypto.delete_fully_used_keys_on_decrypt = delete_cfg["delete_fully_used_on_decrypt"] + self.crypto.delete_keys_on_device_delete = delete_cfg["delete_on_device_delete"] + self.periodically_delete_expired_keys = delete_cfg["periodically_delete_expired"] + self._key_delete_task = None + async def _exit_on_sync_fail(self, data) -> None: if data["error"]: self.log.critical("Exiting due to crypto sync error") @@ -267,6 +281,37 @@ async def start(self) -> None: else: _ = self.client.start(self._filter) self.log.info("End-to-bridge encryption support is enabled (sync mode)") + if self.periodically_delete_expired_keys: + self._key_delete_task = background_task.create(self._periodically_delete_keys()) + background_task.create(self._resync_encryption_info()) + + async def _resync_encryption_info(self) -> None: + rows = await self.crypto_db.fetch( + """SELECT room_id FROM mx_room_state WHERE encryption='{"resync":true}'""" + ) + room_ids = [row["room_id"] for row in rows] + if not room_ids: + return + self.log.debug(f"Resyncing encryption state event in rooms: {room_ids}") + for room_id in room_ids: + try: + evt = await self.client.get_state_event(room_id, EventType.ROOM_ENCRYPTION) + except (MNotFound, MForbidden) as e: + self.log.debug(f"Failed to get encryption state in {room_id}: {e}") + q = """ + UPDATE mx_room_state SET encryption=NULL + WHERE room_id=$1 AND encryption='{"resync":true}' + """ + await self.crypto_db.execute(q, room_id) + else: + self.log.debug(f"Resynced encryption state in {room_id}: {evt}") + q = """ + UPDATE crypto_megolm_inbound_session SET max_age=$1, max_messages=$2 + WHERE room_id=$3 AND max_age IS NULL and max_messages IS NULL + """ + await self.crypto_db.execute( + q, evt.rotation_period_ms, evt.rotation_period_msgs, room_id + ) async def _verify_keys_are_on_server(self) -> None: self.log.debug("Making sure keys are still on server") @@ -289,6 +334,9 @@ async def _verify_keys_are_on_server(self) -> None: sys.exit(34) async def stop(self) -> None: + if self._key_delete_task: + self._key_delete_task.cancel() + self._key_delete_task = None self.client.stop() await self.crypto_store.close() if self.crypto_db: @@ -308,3 +356,12 @@ def _filter(self) -> Filter: ephemeral=RoomEventFilter(not_types=[all_events]), ), ) + + async def _periodically_delete_keys(self) -> None: + while True: + deleted = await self.crypto_store.redact_expired_group_sessions() + if deleted: + self.log.info(f"Deleted expired megolm sessions: {deleted}") + else: + self.log.debug("No expired megolm sessions found") + await asyncio.sleep(24 * 60 * 60) diff --git a/mautrix/bridge/portal.py b/mautrix/bridge/portal.py index 1a6a1343..05d67e3b 100644 --- a/mautrix/bridge/portal.py +++ b/mautrix/bridge/portal.py @@ -386,7 +386,7 @@ async def _disappear_event(self, msg: br.AbstractDisappearingMessage) -> None: await self._do_disappear(msg.event_id) self.log.debug(f"Expired event {msg.event_id} disappeared successfully") except Exception as e: - self.log.warning(f"Failed to make expired event {msg.event_id} disappear: {e}", e) + self.log.warning(f"Failed to make expired event {msg.event_id} disappear: {e}") async def _do_disappear(self, event_id: EventID) -> None: await self.main_intent.redact(self.mxid, event_id) diff --git a/mautrix/client/state_store/asyncpg/upgrade.py b/mautrix/client/state_store/asyncpg/upgrade.py index 0a489aae..88f115f2 100644 --- a/mautrix/client/state_store/asyncpg/upgrade.py +++ b/mautrix/client/state_store/asyncpg/upgrade.py @@ -14,8 +14,8 @@ ) -@upgrade_table.register(description="Latest revision", upgrades_to=2) -async def upgrade_blank_to_v2(conn: Connection, scheme: Scheme) -> None: +@upgrade_table.register(description="Latest revision", upgrades_to=3) +async def upgrade_blank_to_v3(conn: Connection, scheme: Scheme) -> None: await conn.execute( """CREATE TABLE mx_room_state ( room_id TEXT PRIMARY KEY, @@ -54,3 +54,18 @@ async def upgrade_v2(conn: Connection, scheme: Scheme) -> None: await conn.execute("ALTER TABLE mx_user_profile ALTER COLUMN user_id TYPE TEXT") await conn.execute("ALTER TABLE mx_user_profile ALTER COLUMN displayname TYPE TEXT") await conn.execute("ALTER TABLE mx_user_profile ALTER COLUMN avatar_url TYPE TEXT") + + +@upgrade_table.register(description="Mark rooms that need crypto state event resynced") +async def upgrade_v3(conn: Connection) -> None: + if await conn.table_exists("portal"): + await conn.execute( + """ + INSERT INTO mx_room_state (room_id, encryption) + SELECT portal.mxid, '{"resync":true}' FROM portal + WHERE portal.encrypted=true AND portal.mxid IS NOT NULL + ON CONFLICT (room_id) DO UPDATE + SET encryption=excluded.encryption + WHERE mx_room_state.encryption IS NULL + """ + ) diff --git a/mautrix/crypto/__init__.py b/mautrix/crypto/__init__.py index 39867225..743fc2c6 100644 --- a/mautrix/crypto/__init__.py +++ b/mautrix/crypto/__init__.py @@ -1,6 +1,6 @@ from .account import OlmAccount from .key_share import RejectKeyShare -from .sessions import InboundGroupSession, OutboundGroupSession, Session +from .sessions import InboundGroupSession, OutboundGroupSession, RatchetSafety, Session # These have to be last from .store import ( # isort: skip diff --git a/mautrix/crypto/base.py b/mautrix/crypto/base.py index 4d12e6cf..f403feee 100644 --- a/mautrix/crypto/base.py +++ b/mautrix/crypto/base.py @@ -46,6 +46,13 @@ class BaseOlmMachine: share_keys_min_trust: TrustState allow_key_share: Callable[[crypto.DeviceIdentity, RequestedKeyInfo], Awaitable[bool]] + delete_outbound_keys_on_ack: bool + dont_store_outbound_keys: bool + delete_previous_keys_on_receive: bool + ratchet_keys_on_decrypt: bool + delete_fully_used_keys_on_decrypt: bool + delete_keys_on_device_delete: bool + # Futures that wait for responses to a key request _key_request_waiters: dict[SessionID, asyncio.Future] # Futures that wait for a session to be received (either normally or through a key request) @@ -53,6 +60,7 @@ class BaseOlmMachine: _prev_unwedge: dict[IdentityKey, float] _fetch_keys_lock: asyncio.Lock + _megolm_decrypt_lock: asyncio.Lock _cs_fetch_attempted: set[UserID] async def wait_for_session( diff --git a/mautrix/crypto/decrypt_megolm.py b/mautrix/crypto/decrypt_megolm.py index 8edf5aaf..fd7a7eef 100644 --- a/mautrix/crypto/decrypt_megolm.py +++ b/mautrix/crypto/decrypt_megolm.py @@ -25,6 +25,7 @@ ) from .device_lists import DeviceListMachine +from .sessions import InboundGroupSession class MegolmDecryptionMachine(DeviceListMachine): @@ -45,18 +46,22 @@ async def decrypt_megolm_event(self, evt: EncryptedEvent) -> Event: raise DecryptionError("Unsupported event content class") elif evt.content.algorithm != EncryptionAlgorithm.MEGOLM_V1: raise DecryptionError("Unsupported event encryption algorithm") - session = await self.crypto_store.get_group_session(evt.room_id, evt.content.session_id) - if session is None: - # TODO check if olm session is wedged - raise SessionNotFound(evt.content.session_id, evt.content.sender_key) - try: - plaintext, index = session.decrypt(evt.content.ciphertext) - except olm.OlmGroupSessionError as e: - raise DecryptionError("Failed to decrypt megolm event") from e - if not await self.crypto_store.validate_message_index( - session.sender_key, SessionID(session.id), evt.event_id, index, evt.timestamp - ): - raise DuplicateMessageIndex() + async with self._megolm_decrypt_lock: + session = await self.crypto_store.get_group_session( + evt.room_id, evt.content.session_id + ) + if session is None: + # TODO check if olm session is wedged + raise SessionNotFound(evt.content.session_id, evt.content.sender_key) + try: + plaintext, index = session.decrypt(evt.content.ciphertext) + except olm.OlmGroupSessionError as e: + raise DecryptionError("Failed to decrypt megolm event") from e + if not await self.crypto_store.validate_message_index( + session.sender_key, SessionID(session.id), evt.event_id, index, evt.timestamp + ): + raise DuplicateMessageIndex() + await self._ratchet_session(session, index) forwarded_keys = False if ( @@ -133,3 +138,55 @@ async def decrypt_megolm_event(self, evt: EncryptedEvent) -> Event: "was_encrypted": True, } return result + + async def _ratchet_session(self, sess: InboundGroupSession, index: int) -> None: + expected_message_index = sess.ratchet_safety.next_index + did_modify = True + if index > expected_message_index: + sess.ratchet_safety.missed_indices += list(range(expected_message_index, index)) + sess.ratchet_safety.next_index = index + 1 + elif index == expected_message_index: + sess.ratchet_safety.next_index = index + 1 + else: + try: + sess.ratchet_safety.missed_indices.remove(index) + except ValueError: + did_modify = False + # Use presence of received_at as a sign that this is a recent megolm session, + # and therefore it's safe to drop missed indices entirely. + if ( + sess.received_at + and sess.ratchet_safety.missed_indices + and sess.ratchet_safety.missed_indices[0] < expected_message_index - 10 + ): + i = 0 + for i, lost_index in enumerate(sess.ratchet_safety.missed_indices): + if lost_index < expected_message_index - 10: + sess.ratchet_safety.lost_indices.append(lost_index) + else: + break + sess.ratchet_safety.missed_indices = sess.ratchet_safety.missed_indices[i + 1 :] + ratchet_target_index = sess.ratchet_safety.next_index + if len(sess.ratchet_safety.missed_indices) > 0: + ratchet_target_index = min(sess.ratchet_safety.missed_indices) + self.log.debug( + f"Ratchet safety info for {sess.id}: {sess.ratchet_safety}, {ratchet_target_index=}" + ) + sess_id = SessionID(sess.id) + if ( + sess.max_messages + and ratchet_target_index >= sess.max_messages + and not sess.ratchet_safety.missed_indices + and self.delete_fully_used_keys_on_decrypt + ): + self.log.info(f"Deleting fully used session {sess.id}") + await self.crypto_store.redact_group_session( + sess.room_id, sess_id, reason="maximum messages reached" + ) + return + elif sess.first_known_index < ratchet_target_index and self.ratchet_keys_on_decrypt: + self.log.info(f"Ratcheting session {sess.id} to {ratchet_target_index}") + sess = sess.ratchet_to(ratchet_target_index) + elif not did_modify: + return + await self.crypto_store.put_group_session(sess.room_id, sess.sender_key, sess_id, sess) diff --git a/mautrix/crypto/device_lists.py b/mautrix/crypto/device_lists.py index c0cea43f..9afe7b91 100644 --- a/mautrix/crypto/device_lists.py +++ b/mautrix/crypto/device_lists.py @@ -81,6 +81,16 @@ async def _fetch_keys( data[user_id] = new_devices if changed or len(new_devices) != len(existing_devices): + if self.delete_keys_on_device_delete: + for device_id in new_devices.keys() - existing_devices.keys(): + device = existing_devices[device_id] + removed_ids = await self.crypto_store.redact_group_sessions( + room_id=None, sender_key=device.identity_key, reason="device removed" + ) + self.log.info( + "Redacted megolm sessions sent by removed device " + f"{device.user_id}/{device.device_id}: {removed_ids}" + ) await self.on_devices_changed(user_id) for user_id in missing_users: diff --git a/mautrix/crypto/encrypt_megolm.py b/mautrix/crypto/encrypt_megolm.py index 9b37e486..227fd3fe 100644 --- a/mautrix/crypto/encrypt_megolm.py +++ b/mautrix/crypto/encrypt_megolm.py @@ -5,7 +5,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. from typing import Any, Dict, List, Tuple, Union from collections import defaultdict -from datetime import timedelta +from datetime import datetime, timedelta import asyncio import json import time @@ -173,21 +173,6 @@ async def _share_group_session(self, room_id: RoomID, users: List[UserID]) -> No session = await self._new_outbound_group_session(room_id) self.log.debug(f"Sharing group session {session.id} for room {room_id} with {users}") - encryption_info = await self.state_store.get_encryption_info(room_id) - if encryption_info: - if encryption_info.algorithm != EncryptionAlgorithm.MEGOLM_V1: - raise SessionShareError("Room encryption algorithm is not supported") - session.max_messages = encryption_info.rotation_period_msgs or session.max_messages - session.max_age = ( - timedelta(milliseconds=encryption_info.rotation_period_ms) - if encryption_info.rotation_period_ms - else session.max_age - ) - self.log.debug( - "Got stored encryption state event and configured session to rotate " - f"after {session.max_messages} messages or {session.max_age}" - ) - olm_sessions: DeviceMap = defaultdict(lambda: {}) withhold_key_msgs = defaultdict(lambda: {}) missing_sessions: Dict[UserID, Dict[DeviceID, DeviceIdentity]] = defaultdict(lambda: {}) @@ -253,13 +238,33 @@ async def _share_group_session(self, room_id: RoomID, users: List[UserID]) -> No async def _new_outbound_group_session(self, room_id: RoomID) -> OutboundGroupSession: session = OutboundGroupSession(room_id) - await self._create_group_session( - self.account.identity_key, - self.account.signing_key, - room_id, - SessionID(session.id), - session.session_key, - ) + + encryption_info = await self.state_store.get_encryption_info(room_id) + if encryption_info: + if encryption_info.algorithm != EncryptionAlgorithm.MEGOLM_V1: + raise SessionShareError("Room encryption algorithm is not supported") + session.max_messages = encryption_info.rotation_period_msgs or session.max_messages + session.max_age = ( + timedelta(milliseconds=encryption_info.rotation_period_ms) + if encryption_info.rotation_period_ms + else session.max_age + ) + self.log.debug( + "Got stored encryption state event and configured session to rotate " + f"after {session.max_messages} messages or {session.max_age}" + ) + + if not self.dont_store_outbound_keys: + await self._create_group_session( + self.account.identity_key, + self.account.signing_key, + room_id, + SessionID(session.id), + session.session_key, + max_messages=session.max_messages, + max_age=session.max_age, + is_scheduled=False, + ) return session async def _encrypt_and_share_group_session( @@ -286,6 +291,9 @@ async def _create_group_session( room_id: RoomID, session_id: SessionID, session_key: str, + max_age: Union[timedelta, int], + max_messages: int, + is_scheduled: bool = False, ) -> None: start = time.monotonic() session = InboundGroupSession( @@ -293,6 +301,10 @@ async def _create_group_session( signing_key=signing_key, sender_key=sender_key, room_id=room_id, + received_at=datetime.utcnow(), + max_age=max_age, + max_messages=max_messages, + is_scheduled=is_scheduled, ) olm_duration = time.monotonic() - start if olm_duration > 5: @@ -302,7 +314,10 @@ async def _create_group_session( session_id = session.id await self.crypto_store.put_group_session(room_id, sender_key, session_id, session) self._mark_session_received(session_id) - self.log.debug(f"Created inbound group session {room_id}/{sender_key}/{session_id}") + self.log.debug( + f"Created inbound group session {room_id}/{sender_key}/{session_id} " + f"(max {max_age} / {max_messages} messages, {is_scheduled=})" + ) async def _find_olm_sessions( self, diff --git a/mautrix/crypto/key_request.py b/mautrix/crypto/key_request.py index 9bbd2d67..600ecc75 100644 --- a/mautrix/crypto/key_request.py +++ b/mautrix/crypto/key_request.py @@ -4,6 +4,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. from typing import Dict, List, Optional, Union +from datetime import timedelta import asyncio import uuid @@ -120,9 +121,23 @@ async def _receive_forwarded_room_key(self, evt: DecryptedOlmEvent) -> None: f"{evt.sender_device}, as crypto store says we have it already" ) return + if not key.beeper_max_messages or not key.beeper_max_age_ms: + encryption_info = await self.state_store.get_encryption_info(key.room_id) + if encryption_info: + if not key.beeper_max_age_ms: + key.beeper_max_age_ms = encryption_info.rotation_period_ms + if not key.beeper_max_messages: + key.beeper_max_messages = encryption_info.rotation_period_msgs key.forwarding_key_chain.append(evt.sender_key) sess = InboundGroupSession.import_session( - key.session_key, key.signing_key, key.sender_key, key.room_id, key.forwarding_key_chain + key.session_key, + key.signing_key, + key.sender_key, + key.room_id, + key.forwarding_key_chain, + max_age=key.beeper_max_age_ms, + max_messages=key.beeper_max_messages, + is_scheduled=key.beeper_is_scheduled, ) if key.session_id != sess.id: self.log.warning( diff --git a/mautrix/crypto/machine.py b/mautrix/crypto/machine.py index f9884f7b..fcefc3e3 100644 --- a/mautrix/crypto/machine.py +++ b/mautrix/crypto/machine.py @@ -10,6 +10,7 @@ import logging from mautrix import client as cli +from mautrix.errors import GroupSessionWithheldError from mautrix.types import ( ASToDeviceEvent, DecryptedOlmEvent, @@ -76,7 +77,15 @@ def __init__( self.share_keys_min_trust = TrustState.CROSS_SIGNED_TOFU self.allow_key_share = self.default_allow_key_share + self.delete_outbound_keys_on_ack = False + self.dont_store_outbound_keys = False + self.delete_previous_keys_on_receive = False + self.ratchet_keys_on_decrypt = False + self.delete_fully_used_keys_on_decrypt = False + self.delete_keys_on_device_delete = False + self._fetch_keys_lock = asyncio.Lock() + self._megolm_decrypt_lock = asyncio.Lock() self._key_request_waiters = {} self._inbound_session_waiters = {} self._prev_unwedge = {} @@ -88,6 +97,7 @@ def __init__( self.client.add_event_handler(cli.InternalEventType.DEVICE_LISTS, self.handle_device_lists) self.client.add_event_handler(EventType.TO_DEVICE_ENCRYPTED, self.handle_to_device_event) self.client.add_event_handler(EventType.ROOM_KEY_REQUEST, self.handle_room_key_request) + self.client.add_event_handler(EventType.BEEPER_ROOM_KEY_ACK, self.handle_beep_room_key_ack) # self.client.add_event_handler(EventType.ROOM_KEY_WITHHELD, self.handle_room_key_withheld) # self.client.add_event_handler(EventType.ORG_MATRIX_ROOM_KEY_WITHHELD, # self.handle_room_key_withheld) @@ -123,6 +133,8 @@ async def handle_as_to_device_event(self, evt: ASToDeviceEvent) -> None: await self.handle_to_device_event(evt) elif evt.type == EventType.ROOM_KEY_REQUEST: await self.handle_room_key_request(evt) + elif evt.type == EventType.BEEPER_ROOM_KEY_ACK: + await self.handle_beep_room_key_ack(evt) else: self.log.debug(f"Got unknown to-device event {evt.type} from {evt.sender}") @@ -210,14 +222,55 @@ async def _receive_room_key(self, evt: DecryptedOlmEvent) -> None: # for the case where evt.Keys.Ed25519 is none? if evt.content.algorithm != EncryptionAlgorithm.MEGOLM_V1 or not evt.keys.ed25519: return + if not evt.content.beeper_max_messages or not evt.content.beeper_max_age_ms: + encryption_info = await self.state_store.get_encryption_info(evt.content.room_id) + if encryption_info: + if not evt.content.beeper_max_age_ms: + evt.content.beeper_max_age_ms = encryption_info.rotation_period_ms + if not evt.content.beeper_max_messages: + evt.content.beeper_max_messages = encryption_info.rotation_period_msgs + if self.delete_previous_keys_on_receive and not evt.content.beeper_is_scheduled: + removed_ids = await self.crypto_store.redact_group_sessions( + evt.content.room_id, evt.sender_key, reason="received new key from device" + ) + self.log.info(f"Redacted previous megolm sessions: {removed_ids}") await self._create_group_session( evt.sender_key, evt.keys.ed25519, evt.content.room_id, evt.content.session_id, evt.content.session_key, + max_age=evt.content.beeper_max_age_ms, + max_messages=evt.content.beeper_max_messages, + is_scheduled=evt.content.beeper_is_scheduled, ) + async def handle_beep_room_key_ack(self, evt: ToDeviceEvent) -> None: + try: + sess = await self.crypto_store.get_group_session( + evt.content.room_id, evt.content.session_id + ) + except GroupSessionWithheldError: + self.log.debug( + f"Ignoring room key ack for session {evt.content.session_id}" + " that was already redacted" + ) + return + if not sess: + self.log.debug(f"Ignoring room key ack for unknown session {evt.content.session_id}") + return + if ( + sess.sender_key == self.account.identity_key + and self.delete_outbound_keys_on_ack + and evt.content.first_message_index == 0 + ): + self.log.debug("Redacting inbound copy of outbound group session after ack") + await self.crypto_store.redact_group_session( + evt.content.room_id, evt.content.session_id, reason="outbound session acked" + ) + else: + self.log.debug(f"Received room key ack for {sess.id}") + async def share_keys(self, current_otk_count: int) -> None: """ Share any keys that need to be shared. This is automatically called from diff --git a/mautrix/crypto/sessions.py b/mautrix/crypto/sessions.py index c8164b27..b0b16a18 100644 --- a/mautrix/crypto/sessions.py +++ b/mautrix/crypto/sessions.py @@ -3,10 +3,11 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from typing import List, Optional, Set, Tuple, cast +from typing import List, Optional, Set, Tuple, Union, cast from datetime import datetime, timedelta from _libolm import ffi, lib +from attr import dataclass import olm from mautrix.errors import EncryptionError @@ -18,8 +19,10 @@ OlmMsgType, RoomID, RoomKeyEventContent, + SerializableAttrs, SigningKey, UserID, + field, ) @@ -93,12 +96,25 @@ def describe(self) -> str: return "describe not supported" +@dataclass +class RatchetSafety(SerializableAttrs): + next_index: int = 0 + missed_indices: List[int] = field(factory=lambda: []) + lost_indices: List[int] = field(factory=lambda: []) + + class InboundGroupSession(olm.InboundGroupSession): room_id: RoomID signing_key: SigningKey sender_key: IdentityKey forwarding_chain: List[IdentityKey] + ratchet_safety: RatchetSafety + received_at: datetime + max_age: timedelta + max_messages: int + is_scheduled: bool + def __init__( self, session_key: str, @@ -106,11 +122,23 @@ def __init__( sender_key: IdentityKey, room_id: RoomID, forwarding_chain: Optional[List[IdentityKey]] = None, + ratchet_safety: Optional[RatchetSafety] = None, + received_at: Optional[datetime] = None, + max_age: Union[timedelta, int, None] = None, + max_messages: Optional[int] = None, + is_scheduled: bool = False, ) -> None: self.signing_key = signing_key self.sender_key = sender_key self.room_id = room_id self.forwarding_chain = forwarding_chain or [] + self.ratchet_safety = ratchet_safety or RatchetSafety() + self.received_at = received_at or datetime.utcnow() + if isinstance(max_age, int): + max_age = timedelta(milliseconds=max_age) + self.max_age = max_age + self.max_messages = max_messages + self.is_scheduled = is_scheduled super().__init__(session_key) def __new__(cls, *args, **kwargs): @@ -125,12 +153,22 @@ def from_pickle( sender_key: IdentityKey, room_id: RoomID, forwarding_chain: Optional[List[IdentityKey]] = None, + ratchet_safety: Optional[RatchetSafety] = None, + received_at: Optional[datetime] = None, + max_age: Optional[timedelta] = None, + max_messages: Optional[int] = None, + is_scheduled: bool = False, ) -> "InboundGroupSession": session = super().from_pickle(pickle, passphrase) session.signing_key = signing_key session.sender_key = sender_key session.room_id = room_id session.forwarding_chain = forwarding_chain or [] + session.ratchet_safety = ratchet_safety or RatchetSafety() + session.received_at = received_at + session.max_age = max_age + session.max_messages = max_messages + session.is_scheduled = is_scheduled return session @classmethod @@ -141,14 +179,41 @@ def import_session( sender_key: IdentityKey, room_id: RoomID, forwarding_chain: Optional[List[str]] = None, + ratchet_safety: Optional[RatchetSafety] = None, + received_at: Optional[datetime] = None, + max_age: Union[timedelta, int, None] = None, + max_messages: Optional[int] = None, + is_scheduled: bool = False, ) -> "InboundGroupSession": session = super().import_session(session_key) session.signing_key = signing_key session.sender_key = sender_key session.room_id = room_id session.forwarding_chain = forwarding_chain or [] + session.ratchet_safety = ratchet_safety or RatchetSafety() + session.received_at = received_at or datetime.utcnow() + if isinstance(max_age, int): + max_age = timedelta(milliseconds=max_age) + session.max_age = max_age + session.max_messages = max_messages + session.is_scheduled = is_scheduled return session + def ratchet_to(self, index: int) -> "InboundGroupSession": + exported = self.export_session(index) + return self.import_session( + exported, + signing_key=self.signing_key, + sender_key=self.sender_key, + room_id=self.room_id, + forwarding_chain=self.forwarding_chain, + ratchet_safety=self.ratchet_safety, + received_at=self.received_at, + max_age=self.max_age, + max_messages=self.max_messages, + is_scheduled=self.is_scheduled, + ) + class OutboundGroupSession(olm.OutboundGroupSession): """Outbound group session aware of the users it is shared with. diff --git a/mautrix/crypto/store/abstract.py b/mautrix/crypto/store/abstract.py index 7828524d..4db5aa1d 100644 --- a/mautrix/crypto/store/abstract.py +++ b/mautrix/crypto/store/abstract.py @@ -197,6 +197,46 @@ async def get_group_session( The :class:`InboundGroupSession` object, or ``None`` if not found. """ + @abstractmethod + async def redact_group_session( + self, room_id: RoomID, session_id: SessionID, reason: str + ) -> None: + """ + Remove the keys for a specific Megolm group session. + + Args: + room_id: The room where the session is. + session_id: The session ID to remove. + reason: The reason the session is being removed. + """ + + @abstractmethod + async def redact_group_sessions( + self, room_id: RoomID | None, sender_key: IdentityKey | None, reason: str + ) -> list[SessionID]: + """ + Remove the keys for multiple Megolm group sessions, + based on the room ID and/or sender device. + + Args: + room_id: The room ID to delete keys from. + sender_key: The Olm identity key of the device to delete keys from. + reason: The reason why the keys are being deleted. + + Returns: + The list of session IDs that were deleted. + """ + + @abstractmethod + async def redact_expired_group_sessions(self) -> list[SessionID]: + """ + Remove all Megolm group sessions where at least twice the maximum age has passed since + receiving the keys. + + Returns: + The list of session IDs that were deleted. + """ + @abstractmethod async def has_group_session(self, room_id: RoomID, session_id: SessionID) -> bool: """ diff --git a/mautrix/crypto/store/asyncpg/store.py b/mautrix/crypto/store/asyncpg/store.py index cb15be77..69f378f2 100644 --- a/mautrix/crypto/store/asyncpg/store.py +++ b/mautrix/crypto/store/asyncpg/store.py @@ -21,6 +21,7 @@ EventID, IdentityKey, RoomID, + RoomKeyWithheldCode, SessionID, SigningKey, SyncToken, @@ -31,7 +32,7 @@ from mautrix.util.async_db import Database, Scheme from mautrix.util.logging import TraceLogger -from ... import InboundGroupSession, OlmAccount, OutboundGroupSession, Session +from ... import InboundGroupSession, OlmAccount, OutboundGroupSession, RatchetSafety, Session from ..abstract import CryptoStore, StateStore from .upgrade import upgrade_table @@ -235,12 +236,15 @@ async def put_group_session( forwarding_chains = ",".join(session.forwarding_chain) q = """ INSERT INTO crypto_megolm_inbound_session ( - session_id, sender_key, signing_key, room_id, session, forwarding_chains, account_id - ) VALUES ($1, $2, $3, $4, $5, $6, $7) + session_id, sender_key, signing_key, room_id, session, forwarding_chains, + ratchet_safety, received_at, max_age, max_messages, is_scheduled, account_id + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) ON CONFLICT (session_id, account_id) DO UPDATE SET withheld_code=NULL, withheld_reason=NULL, sender_key=excluded.sender_key, signing_key=excluded.signing_key, room_id=excluded.room_id, session=excluded.session, - forwarding_chains=excluded.forwarding_chains + forwarding_chains=excluded.forwarding_chains, ratchet_safety=excluded.ratchet_safety, + received_at=excluded.received_at, max_age=excluded.max_age, + max_messages=excluded.max_messages, is_scheduled=excluded.is_scheduled """ try: await self.db.execute( @@ -251,6 +255,11 @@ async def put_group_session( room_id, pickle, forwarding_chains, + session.ratchet_safety.json(), + session.received_at, + int(session.max_age.total_seconds() * 1000), + session.max_messages, + session.is_scheduled, self.account_id, ) except (IntegrityError, UniqueViolationError): @@ -260,7 +269,9 @@ async def get_group_session( self, room_id: RoomID, session_id: SessionID ) -> InboundGroupSession | None: q = """ - SELECT sender_key, signing_key, session, forwarding_chains, withheld_code + SELECT + sender_key, signing_key, session, forwarding_chains, withheld_code, + ratchet_safety, received_at, max_age, max_messages, is_scheduled FROM crypto_megolm_inbound_session WHERE room_id=$1 AND session_id=$2 AND account_id=$3 """ @@ -277,7 +288,78 @@ async def get_group_session( sender_key=row["sender_key"], room_id=room_id, forwarding_chain=forwarding_chain, + ratchet_safety=RatchetSafety.parse_json(row["ratchet_safety"] or "{}"), + received_at=row["received_at"], + max_age=timedelta(milliseconds=row["max_age"]) if row["max_age"] else None, + max_messages=row["max_messages"], + is_scheduled=row["is_scheduled"], + ) + + async def redact_group_session( + self, room_id: RoomID, session_id: SessionID, reason: str + ) -> None: + q = """ + UPDATE crypto_megolm_inbound_session + SET withheld_code=$1, withheld_reason=$2, session=NULL, forwarding_chains=NULL + WHERE session_id=$3 AND account_id=$4 AND session IS NOT NULL + """ + await self.db.execute( + q, + RoomKeyWithheldCode.BEEPER_REDACTED.value, + f"Session redacted: {reason}", + session_id, + self.account_id, + ) + + async def redact_group_sessions( + self, room_id: RoomID, sender_key: IdentityKey, reason: str + ) -> list[SessionID]: + if not room_id and not sender_key: + raise ValueError("Either room_id or sender_key must be provided") + q = """ + UPDATE crypto_megolm_inbound_session + SET withheld_code=$1, withheld_reason=$2, session=NULL, forwarding_chains=NULL + WHERE (room_id=$3 OR $3='') AND (sender_key=$4 OR $4='') AND session IS NOT NULL AND account_id=$5 AND is_scheduled=false + RETURNING session_id + """ + rows = await self.db.fetch( + q, + RoomKeyWithheldCode.BEEPER_REDACTED.value, + f"Session redacted: {reason}", + room_id, + sender_key, + self.account_id, + ) + return [row["session_id"] for row in rows] + + async def redact_expired_group_sessions(self) -> list[SessionID]: + if self.db.scheme == Scheme.SQLITE: + q = """ + UPDATE crypto_megolm_inbound_session + SET withheld_code=$1, withheld_reason=$2, session=NULL, forwarding_chains=NULL + WHERE account_id=$3 AND session IS NOT NULL AND is_scheduled=false + AND received_at IS NOT NULL and max_age IS NOT NULL + AND unixepoch(received_at) + (2 * max_age / 1000) < unixepoch(date('now')) + RETURNING session_id + """ + elif self.db.scheme in (Scheme.POSTGRES, Scheme.COCKROACH): + q = """ + UPDATE crypto_megolm_inbound_session + SET withheld_code=$1, withheld_reason=$2, session=NULL, forwarding_chains=NULL + WHERE account_id=$3 AND session IS NOT NULL AND is_scheduled=false + AND received_at IS NOT NULL and max_age IS NOT NULL + AND received_at + 2 * (max_age * interval '1 millisecond') < now() + RETURNING session_id + """ + else: + raise RuntimeError(f"Unsupported dialect {self.db.scheme}") + rows = await self.db.fetch( + q, + RoomKeyWithheldCode.BEEPER_REDACTED.value, + f"Session redacted: expired", + self.account_id, ) + return [row["session_id"] for row in rows] async def has_group_session(self, room_id: RoomID, session_id: SessionID) -> bool: q = """ diff --git a/mautrix/crypto/store/asyncpg/upgrade.py b/mautrix/crypto/store/asyncpg/upgrade.py index 914da23d..e097c5d9 100644 --- a/mautrix/crypto/store/asyncpg/upgrade.py +++ b/mautrix/crypto/store/asyncpg/upgrade.py @@ -16,7 +16,7 @@ ) -@upgrade_table.register(description="Latest revision", upgrades_to=9) +@upgrade_table.register(description="Latest revision", upgrades_to=10) async def upgrade_blank_to_latest(conn: Connection) -> None: await conn.execute( """CREATE TABLE IF NOT EXISTS crypto_account ( @@ -77,6 +77,11 @@ async def upgrade_blank_to_latest(conn: Connection) -> None: forwarding_chains TEXT, withheld_code TEXT, withheld_reason TEXT, + ratchet_safety jsonb, + received_at timestamp, + max_age BIGINT, + max_messages INTEGER, + is_scheduled BOOLEAN NOT NULL DEFAULT false, PRIMARY KEY (account_id, session_id) )""" ) @@ -420,3 +425,19 @@ async def upgrade_v9_sqlite(conn: Connection) -> None: await conn.execute("PRAGMA foreign_key_check") await conn.execute("PRAGMA foreign_keys = ON") + + +@upgrade_table.register( + description="Add metadata for detecting when megolm sessions are safe to delete" +) +async def upgrade_v10(conn: Connection) -> None: + await conn.execute("ALTER TABLE crypto_megolm_inbound_session ADD COLUMN ratchet_safety jsonb") + await conn.execute( + "ALTER TABLE crypto_megolm_inbound_session ADD COLUMN received_at timestamp" + ) + await conn.execute("ALTER TABLE crypto_megolm_inbound_session ADD COLUMN max_age BIGINT") + await conn.execute("ALTER TABLE crypto_megolm_inbound_session ADD COLUMN max_messages INTEGER") + await conn.execute( + "ALTER TABLE crypto_megolm_inbound_session " + "ADD COLUMN is_scheduled BOOLEAN NOT NULL DEFAULT false" + ) diff --git a/mautrix/crypto/store/memory.py b/mautrix/crypto/store/memory.py index 35dc26b5..cfc27125 100644 --- a/mautrix/crypto/store/memory.py +++ b/mautrix/crypto/store/memory.py @@ -110,6 +110,30 @@ async def get_group_session( ) -> InboundGroupSession: return self._inbound_sessions.get((room_id, session_id)) + async def redact_group_session( + self, room_id: RoomID, session_id: SessionID, reason: str + ) -> None: + self._inbound_sessions.pop((room_id, session_id), None) + + async def redact_group_sessions( + self, room_id: RoomID, sender_key: IdentityKey, reason: str + ) -> list[SessionID]: + if not room_id and not sender_key: + raise ValueError("Either room_id or sender_key must be provided") + deleted = [] + keys = list(self._inbound_sessions.keys()) + for key in keys: + item = self._inbound_sessions[key] + if (not room_id or item.room_id == room_id) and ( + not sender_key or item.sender_key == sender_key + ): + deleted.append(SessionID(item.id)) + del self._inbound_sessions[key] + return deleted + + async def redact_expired_group_sessions(self) -> list[SessionID]: + raise NotImplementedError() + async def has_group_session(self, room_id: RoomID, session_id: SessionID) -> bool: return (room_id, session_id) in self._inbound_sessions diff --git a/mautrix/types/event/beeper.py b/mautrix/types/event/beeper.py index 23a9f5cd..0ec16479 100644 --- a/mautrix/types/event/beeper.py +++ b/mautrix/types/event/beeper.py @@ -7,7 +7,7 @@ from attr import dataclass -from ..primitive import EventID +from ..primitive import EventID, RoomID, SessionID from ..util import SerializableAttrs, SerializableEnum, field from .base import BaseRoomEvent from .message import RelatesTo @@ -55,3 +55,10 @@ class BeeperMessageStatusEventContent(SerializableAttrs): @dataclass class BeeperMessageStatusEvent(BaseRoomEvent, SerializableAttrs): content: BeeperMessageStatusEventContent + + +@dataclass +class BeeperRoomKeyAckEventContent(SerializableAttrs): + room_id: RoomID + session_id: SessionID + first_message_index: int diff --git a/mautrix/types/event/to_device.py b/mautrix/types/event/to_device.py index 6a17e36e..d6392177 100644 --- a/mautrix/types/event/to_device.py +++ b/mautrix/types/event/to_device.py @@ -9,8 +9,9 @@ import attr from ..primitive import JSON, DeviceID, IdentityKey, RoomID, SessionID, SigningKey, UserID -from ..util import ExtensibleEnum, Obj, SerializableAttrs, deserializer +from ..util import ExtensibleEnum, Obj, SerializableAttrs, deserializer, field from .base import BaseEvent, EventType +from .beeper import BeeperRoomKeyAckEventContent from .encrypted import EncryptedOlmEventContent, EncryptionAlgorithm @@ -21,6 +22,8 @@ class RoomKeyWithheldCode(ExtensibleEnum): UNAVAILABLE: "RoomKeyWithheldCode" = "m.unavailable" NO_OLM_SESSION: "RoomKeyWithheldCode" = "m.no_olm" + BEEPER_REDACTED: "RoomKeyWithheldCode" = "com.beeper.redacted" + @dataclass class RoomKeyWithheldEventContent(SerializableAttrs): @@ -39,6 +42,10 @@ class RoomKeyEventContent(SerializableAttrs): session_id: SessionID session_key: str + beeper_max_age_ms: Optional[int] = field(json="com.beeper.max_age_ms", default=None) + beeper_max_messages: Optional[int] = field(json="com.beeper.max_messages", default=None) + beeper_is_scheduled: Optional[bool] = field(json="com.beeper.is_scheduled", default=False) + class KeyRequestAction(ExtensibleEnum): REQUEST: "KeyRequestAction" = "request" @@ -61,7 +68,7 @@ class RoomKeyRequestEventContent(SerializableAttrs): body: Optional[RequestedKeyInfo] = None -@dataclass +@dataclass(kw_only=True) class ForwardedRoomKeyEventContent(RoomKeyEventContent, SerializableAttrs): sender_key: IdentityKey signing_key: SigningKey = attr.ib(metadata={"json": "sender_claimed_ed25519_key"}) @@ -75,6 +82,7 @@ class ForwardedRoomKeyEventContent(RoomKeyEventContent, SerializableAttrs): RoomKeyEventContent, RoomKeyRequestEventContent, ForwardedRoomKeyEventContent, + BeeperRoomKeyAckEventContent, ] to_device_event_content_map = { EventType.TO_DEVICE_ENCRYPTED: EncryptedOlmEventContent, @@ -82,12 +90,10 @@ class ForwardedRoomKeyEventContent(RoomKeyEventContent, SerializableAttrs): EventType.ROOM_KEY_REQUEST: RoomKeyRequestEventContent, EventType.ROOM_KEY: RoomKeyEventContent, EventType.FORWARDED_ROOM_KEY: ForwardedRoomKeyEventContent, + EventType.BEEPER_ROOM_KEY_ACK: BeeperRoomKeyAckEventContent, } -# TODO remaining account data event types - - @dataclass class ToDeviceEvent(BaseEvent, SerializableAttrs): sender: UserID diff --git a/mautrix/types/event/type.py b/mautrix/types/event/type.py index 609f40ff..509d6857 100644 --- a/mautrix/types/event/type.py +++ b/mautrix/types/event/type.py @@ -216,6 +216,7 @@ def is_to_device(self) -> bool: "m.room_key_request": "ROOM_KEY_REQUEST", "m.forwarded_room_key": "FORWARDED_ROOM_KEY", "m.dummy": "TO_DEVICE_DUMMY", + "com.beeper.room_key.ack": "BEEPER_ROOM_KEY_ACK", }, EventType.Class.UNKNOWN: { "__ALL__": "ALL", # This is not a real event type diff --git a/mautrix/types/event/type.pyi b/mautrix/types/event/type.pyi index 6c4c6e65..bee0fde0 100644 --- a/mautrix/types/event/type.pyi +++ b/mautrix/types/event/type.pyi @@ -68,6 +68,7 @@ class EventType(Serializable): ORG_MATRIX_ROOM_KEY_WITHHELD: "EventType" ROOM_KEY_REQUEST: "EventType" FORWARDED_ROOM_KEY: "EventType" + BEEPER_ROOM_KEY_ACK: "EventType" ALL: "EventType" From b5b81108f6c4f45637a8501fda163a8a6d6622b6 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 13 Apr 2023 17:24:17 +0300 Subject: [PATCH 313/456] Don't redact old megolm sessions Old meaning received before message index tracking was implemented --- mautrix/crypto/store/asyncpg/store.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mautrix/crypto/store/asyncpg/store.py b/mautrix/crypto/store/asyncpg/store.py index 69f378f2..c070b949 100644 --- a/mautrix/crypto/store/asyncpg/store.py +++ b/mautrix/crypto/store/asyncpg/store.py @@ -319,7 +319,8 @@ async def redact_group_sessions( q = """ UPDATE crypto_megolm_inbound_session SET withheld_code=$1, withheld_reason=$2, session=NULL, forwarding_chains=NULL - WHERE (room_id=$3 OR $3='') AND (sender_key=$4 OR $4='') AND session IS NOT NULL AND account_id=$5 AND is_scheduled=false + WHERE (room_id=$3 OR $3='') AND (sender_key=$4 OR $4='') AND account_id=$5 + AND session IS NOT NULL AND is_scheduled=false AND received_at IS NOT NULL RETURNING session_id """ rows = await self.db.fetch( From e44579bb9000a04c78e8a2b25cce4e81926f8952 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 13 Apr 2023 21:20:28 +0300 Subject: [PATCH 314/456] Bump version to 0.19.10 --- CHANGELOG.md | 5 +++++ mautrix/__init__.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e7cabdd6..74c384f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +## v0.19.10 (2023-04-13) + +* *(crypto, bridge)* Added options to automatically ratchet/delete megolm + sessions to minimize access to old messages. + ## v0.19.9 (2023-04-12) * *(crypto)* Fixed bug in crypto store migration when using outbound sessions diff --git a/mautrix/__init__.py b/mautrix/__init__.py index de48a11a..2cbb2011 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.19.9" +__version__ = "0.19.10" __author__ = "Tulir Asokan " __all__ = [ "api", From 57a7a5c03697178e5cadc8477a2297840617168b Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 13 Apr 2023 23:30:04 +0300 Subject: [PATCH 315/456] Fix putting group sessions with no max_age Fixes mautrix/signal#355 --- mautrix/crypto/store/asyncpg/store.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/crypto/store/asyncpg/store.py b/mautrix/crypto/store/asyncpg/store.py index c070b949..0642d210 100644 --- a/mautrix/crypto/store/asyncpg/store.py +++ b/mautrix/crypto/store/asyncpg/store.py @@ -257,7 +257,7 @@ async def put_group_session( forwarding_chains, session.ratchet_safety.json(), session.received_at, - int(session.max_age.total_seconds() * 1000), + int(session.max_age.total_seconds() * 1000) if session.max_age else None, session.max_messages, session.is_scheduled, self.account_id, From 1e464f8e8b0722215c82b04e2902bf5e1fb1b335 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 14 Apr 2023 12:07:05 +0300 Subject: [PATCH 316/456] Fetch encryption info from server if not cached --- mautrix/crypto/base.py | 34 +++++++++++++++++++++++++++++++++- mautrix/crypto/key_request.py | 10 ++-------- mautrix/crypto/machine.py | 9 ++------- 3 files changed, 37 insertions(+), 16 deletions(-) diff --git a/mautrix/crypto/base.py b/mautrix/crypto/base.py index f403feee..5af3ec0a 100644 --- a/mautrix/crypto/base.py +++ b/mautrix/crypto/base.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022 Tulir Asokan +# Copyright (c) 2023 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this @@ -12,13 +12,17 @@ import olm +from mautrix.errors import MForbidden, MNotFound from mautrix.types import ( DeviceID, EncryptionKeyAlgorithm, + EventType, IdentityKey, KeyID, RequestedKeyInfo, + RoomEncryptionStateEventContent, RoomID, + RoomKeyEventContent, SessionID, SigningKey, TrustState, @@ -82,6 +86,34 @@ def _mark_session_received(self, session_id: SessionID) -> None: except KeyError: return + async def _fill_encryption_info(self, evt: RoomKeyEventContent) -> None: + encryption_info = await self.state_store.get_encryption_info(evt.room_id) + if not encryption_info: + self.log.warning( + f"Encryption info for {evt.room_id} not found in state store, fetching from server" + ) + try: + encryption_info = await self.client.get_state_event( + evt.room_id, EventType.ROOM_ENCRYPTION + ) + except (MNotFound, MForbidden) as e: + self.log.warning( + f"Failed to get encryption info for {evt.room_id} from server: {e}," + " using defaults" + ) + encryption_info = RoomEncryptionStateEventContent() + if not encryption_info: + self.log.warning( + f"Didn't find encryption info for {evt.room_id} on server either," + " using defaults" + ) + encryption_info = RoomEncryptionStateEventContent() + + if not evt.beeper_max_age_ms: + evt.beeper_max_age_ms = encryption_info.rotation_period_ms + if not evt.beeper_max_messages: + evt.beeper_max_messages = encryption_info.rotation_period_msgs + canonical_json = functools.partial( json.dumps, ensure_ascii=False, separators=(",", ":"), sort_keys=True diff --git a/mautrix/crypto/key_request.py b/mautrix/crypto/key_request.py index 600ecc75..ecaf994d 100644 --- a/mautrix/crypto/key_request.py +++ b/mautrix/crypto/key_request.py @@ -1,10 +1,9 @@ -# Copyright (c) 2022 Tulir Asokan +# Copyright (c) 2023 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. from typing import Dict, List, Optional, Union -from datetime import timedelta import asyncio import uuid @@ -122,12 +121,7 @@ async def _receive_forwarded_room_key(self, evt: DecryptedOlmEvent) -> None: ) return if not key.beeper_max_messages or not key.beeper_max_age_ms: - encryption_info = await self.state_store.get_encryption_info(key.room_id) - if encryption_info: - if not key.beeper_max_age_ms: - key.beeper_max_age_ms = encryption_info.rotation_period_ms - if not key.beeper_max_messages: - key.beeper_max_messages = encryption_info.rotation_period_msgs + await self._fill_encryption_info(key) key.forwarding_key_chain.append(evt.sender_key) sess = InboundGroupSession.import_session( key.session_key, diff --git a/mautrix/crypto/machine.py b/mautrix/crypto/machine.py index fcefc3e3..9cf53312 100644 --- a/mautrix/crypto/machine.py +++ b/mautrix/crypto/machine.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022 Tulir Asokan +# Copyright (c) 2023 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this @@ -223,12 +223,7 @@ async def _receive_room_key(self, evt: DecryptedOlmEvent) -> None: if evt.content.algorithm != EncryptionAlgorithm.MEGOLM_V1 or not evt.keys.ed25519: return if not evt.content.beeper_max_messages or not evt.content.beeper_max_age_ms: - encryption_info = await self.state_store.get_encryption_info(evt.content.room_id) - if encryption_info: - if not evt.content.beeper_max_age_ms: - evt.content.beeper_max_age_ms = encryption_info.rotation_period_ms - if not evt.content.beeper_max_messages: - evt.content.beeper_max_messages = encryption_info.rotation_period_msgs + await self._fill_encryption_info(evt.content) if self.delete_previous_keys_on_receive and not evt.content.beeper_is_scheduled: removed_ids = await self.crypto_store.redact_group_sessions( evt.content.room_id, evt.sender_key, reason="received new key from device" From 2f57efa20d2af76a1cb8acd931f6f67d6c156acd Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 14 Apr 2023 12:10:30 +0300 Subject: [PATCH 317/456] Bump version to 0.19.11 --- CHANGELOG.md | 8 ++++++++ mautrix/__init__.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 74c384f5..55ecc068 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,11 @@ +## v0.19.11 (2023-04-14) + +* *(crypto)* Fixed bug in previous release which caused errors if the `max_age` + of a megolm session was not known. +* *(crypto)* Changed key receiving handler to fetch encryption config from + server if it's not cached locally (to find `max_age` and `max_messages` more + reliably). + ## v0.19.10 (2023-04-13) * *(crypto, bridge)* Added options to automatically ratchet/delete megolm diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 2cbb2011..e9e88943 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.19.10" +__version__ = "0.19.11" __author__ = "Tulir Asokan " __all__ = [ "api", From a1443a7542eca7d82148dfa9ea6a25340dccf38e Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 17 Apr 2023 17:16:08 +0300 Subject: [PATCH 318/456] Fix backwards compatibility with old bridges --- mautrix/bridge/e2ee.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/mautrix/bridge/e2ee.py b/mautrix/bridge/e2ee.py index 95b419f7..805efd4f 100644 --- a/mautrix/bridge/e2ee.py +++ b/mautrix/bridge/e2ee.py @@ -119,15 +119,17 @@ def __init__( self.az.device_list_handler = self.crypto.handle_as_device_lists self.az.to_device_handler = self.crypto.handle_as_to_device_event - delete_cfg = bridge.config["bridge.encryption.delete_keys"] - self.crypto.delete_outbound_keys_on_ack = delete_cfg["delete_outbound_on_ack"] - self.crypto.dont_store_outbound_keys = delete_cfg["dont_store_outbound"] - self.crypto.delete_previous_keys_on_receive = delete_cfg["delete_prev_on_new_session"] - self.crypto.ratchet_keys_on_decrypt = delete_cfg["ratchet_on_decrypt"] - self.crypto.delete_fully_used_keys_on_decrypt = delete_cfg["delete_fully_used_on_decrypt"] - self.crypto.delete_keys_on_device_delete = delete_cfg["delete_on_device_delete"] - self.periodically_delete_expired_keys = delete_cfg["periodically_delete_expired"] + self.periodically_delete_expired_keys = False self._key_delete_task = None + del_cfg = bridge.config["bridge.encryption.delete_keys"] + if del_cfg: + self.crypto.delete_outbound_keys_on_ack = del_cfg["delete_outbound_on_ack"] + self.crypto.dont_store_outbound_keys = del_cfg["dont_store_outbound"] + self.crypto.delete_previous_keys_on_receive = del_cfg["delete_prev_on_new_session"] + self.crypto.ratchet_keys_on_decrypt = del_cfg["ratchet_on_decrypt"] + self.crypto.delete_fully_used_keys_on_decrypt = del_cfg["delete_fully_used_on_decrypt"] + self.crypto.delete_keys_on_device_delete = del_cfg["delete_on_device_delete"] + self.periodically_delete_expired_keys = del_cfg["periodically_delete_expired"] async def _exit_on_sync_fail(self, data) -> None: if data["error"]: From 1bf463552890833db01301648def311d2d7e0dd5 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 18 Apr 2023 19:03:36 +0300 Subject: [PATCH 319/456] Remove is_bot flag for bridge bots --- mautrix/bridge/matrix.py | 1 - 1 file changed, 1 deletion(-) diff --git a/mautrix/bridge/matrix.py b/mautrix/bridge/matrix.py index 6ce3bfde..b60706cd 100644 --- a/mautrix/bridge/matrix.py +++ b/mautrix/bridge/matrix.py @@ -279,7 +279,6 @@ async def init_as_bot(self) -> None: "com.beeper.bridge.service": self.bridge.beeper_service_name, "com.beeper.bridge.network": self.bridge.beeper_network_name, "com.beeper.bridge.is_bridge_bot": True, - "com.beeper.bridge.is_bot": True, } ) From aa8d1c89d581c9aa99e60c91205388dd1a820deb Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 18 Apr 2023 19:04:38 +0300 Subject: [PATCH 320/456] Bump version to 0.19.12 --- CHANGELOG.md | 4 ++++ mautrix/__init__.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 55ecc068..96a206c9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## v0.19.12 (2023-04-18) + +* *(bridge)* Fixed backwards-compatibility with new key deletion config options. + ## v0.19.11 (2023-04-14) * *(crypto)* Fixed bug in previous release which caused errors if the `max_age` diff --git a/mautrix/__init__.py b/mautrix/__init__.py index e9e88943..1f48593d 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.19.11" +__version__ = "0.19.12" __author__ = "Tulir Asokan " __all__ = [ "api", From 0b69dd6af0c2a3126dd235a1c762ac77336e2e10 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 24 Apr 2023 18:33:26 +0300 Subject: [PATCH 321/456] Fix deleting megolm sessions on device removal --- CHANGELOG.md | 4 ++++ mautrix/__init__.py | 2 +- mautrix/crypto/device_lists.py | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 96a206c9..730f2a2d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## v0.19.13 (2023-04-24) + +* *(crypto)* Fixed bug with redacting megolm sessions when device is deleted. + ## v0.19.12 (2023-04-18) * *(bridge)* Fixed backwards-compatibility with new key deletion config options. diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 1f48593d..440745cf 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.19.12" +__version__ = "0.19.13" __author__ = "Tulir Asokan " __all__ = [ "api", diff --git a/mautrix/crypto/device_lists.py b/mautrix/crypto/device_lists.py index 9afe7b91..d49c7e19 100644 --- a/mautrix/crypto/device_lists.py +++ b/mautrix/crypto/device_lists.py @@ -82,7 +82,7 @@ async def _fetch_keys( if changed or len(new_devices) != len(existing_devices): if self.delete_keys_on_device_delete: - for device_id in new_devices.keys() - existing_devices.keys(): + for device_id in existing_devices.keys() - new_devices.keys(): device = existing_devices[device_id] removed_ids = await self.crypto_store.redact_group_sessions( room_id=None, sender_key=device.identity_key, reason="device removed" From 714d6eb450a28c398ff29355ec2718971781cc1d Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 1 May 2023 10:55:16 +0300 Subject: [PATCH 322/456] Implement MSC2659 --- mautrix/appservice/api/intent.py | 8 ++++++ mautrix/appservice/appservice.py | 3 +++ mautrix/appservice/as_handler.py | 19 +++++++++++++ mautrix/bridge/bridge.py | 3 +++ mautrix/bridge/matrix.py | 46 +++++++++++++++++++++----------- 5 files changed, 63 insertions(+), 16 deletions(-) diff --git a/mautrix/appservice/api/intent.py b/mautrix/appservice/api/intent.py index 8b5ca16f..e09db0d0 100644 --- a/mautrix/appservice/api/intent.py +++ b/mautrix/appservice/api/intent.py @@ -489,6 +489,14 @@ async def mark_read( ) self.state_store.set_read(room_id, self.mxid, event_id) + async def appservice_ping(self, appservice_id: str, txn_id: str | None = None) -> int: + resp = await self.api.request( + Method.POST, + Path.unstable["fi.mau.msc2659"].appservice[appservice_id].ping, + content={"transaction_id": txn_id} if txn_id is not None else {}, + ) + return resp.get("duration_ms") or resp.get("duration") or -1 + async def batch_send( self, room_id: RoomID, diff --git a/mautrix/appservice/appservice.py b/mautrix/appservice/appservice.py index 551eab5c..892db195 100644 --- a/mautrix/appservice/appservice.py +++ b/mautrix/appservice/appservice.py @@ -194,3 +194,6 @@ async def _liveness_probe(self, _: web.Request) -> web.Response: async def _readiness_probe(self, _: web.Request) -> web.Response: return web.Response(status=200 if self.ready else 500, text="{}") + + async def ping_self(self, txn_id: str | None = None) -> int: + return await self.intent.appservice_ping(self.id, txn_id=txn_id) diff --git a/mautrix/appservice/as_handler.py b/mautrix/appservice/as_handler.py index 78cd4307..2adcb2de 100644 --- a/mautrix/appservice/as_handler.py +++ b/mautrix/appservice/as_handler.py @@ -75,6 +75,8 @@ def register_routes(self, app: web.Application) -> None: ) app.router.add_route("GET", "/_matrix/app/v1/rooms/{alias}", self._http_query_alias) app.router.add_route("GET", "/_matrix/app/v1/users/{user_id}", self._http_query_user) + app.router.add_route("POST", "/_matrix/app/v1/ping", self._http_ping) + app.router.add_route("POST", "/_matrix/app/unstable/fi.mau.msc2659/ping", self._http_ping) def _check_token(self, request: web.Request) -> bool: try: @@ -128,6 +130,23 @@ async def _http_query_alias(self, request: web.Request) -> web.Response: return web.json_response({}, status=404) return web.json_response(response) + async def _http_ping(self, request: web.Request) -> web.Response: + if not self._check_token(request): + raise web.HTTPUnauthorized( + content_type="application/json", + text=json.dumps({"error": "Invalid auth token", "errcode": "M_UNKNOWN_TOKEN"}), + ) + try: + body = await request.json() + except JSONDecodeError: + raise web.HTTPBadRequest( + content_type="application/json", + text=json.dumps({"error": "Body is not JSON", "errcode": "M_NOT_JSON"}), + ) + txn_id = body.get("transaction_id") + self.log.info(f"Received ping from homeserver with transaction ID {txn_id}") + return web.json_response({}) + @staticmethod def _get_with_fallback( json: dict[str, Any], field: str, unstable_prefix: str, default: Any = None diff --git a/mautrix/bridge/bridge.py b/mautrix/bridge/bridge.py index d8241038..c474109b 100644 --- a/mautrix/bridge/bridge.py +++ b/mautrix/bridge/bridge.py @@ -246,6 +246,9 @@ async def start(self) -> None: "correct, and do they match the values in the registration?" ) sys.exit(16) + except Exception: + self.log.critical("Failed to check connection to homeserver", exc_info=True) + sys.exit(16) await self.matrix.init_encryption() self.add_startup_actions(self.matrix.init_as_bot()) diff --git a/mautrix/bridge/matrix.py b/mautrix/bridge/matrix.py index b60706cd..e6b280a5 100644 --- a/mautrix/bridge/matrix.py +++ b/mautrix/bridge/matrix.py @@ -222,33 +222,47 @@ async def check_versions(self) -> None: async def wait_for_connection(self) -> None: self.log.info("Ensuring connectivity to homeserver") errors = 0 - tried_to_register = False while True: try: self.versions = await self.az.intent.versions() - await self.check_versions() - await self.az.intent.whoami() break - except (MUnknownToken, MExclusive): - # These are probably not going to resolve themselves by waiting - raise - except MForbidden: - if not tried_to_register: - self.log.debug( - "Whoami endpoint returned M_FORBIDDEN, " - "trying to register bridge bot before retrying..." - ) - await self.az.intent.ensure_registered() - tried_to_register = True - else: - raise except Exception: errors += 1 if errors <= 6: self.log.exception("Connection to homeserver failed, retrying in 10 seconds") await asyncio.sleep(10) + continue else: raise + await self.check_versions() + try: + await self.az.intent.whoami() + except (MUnknownToken, MExclusive): + raise + except MForbidden: + self.log.debug( + "Whoami endpoint returned M_FORBIDDEN, " + "trying to register bridge bot before retrying..." + ) + await self.az.intent.ensure_registered() + await self.az.intent.whoami() + if self.versions.supports("fi.mau.msc2659"): + try: + txn_id = self.az.intent.api.get_txn_id() + duration = await self.az.ping_self(txn_id) + self.log.debug( + "Homeserver->bridge connection works, " + f"roundtrip time is {duration} ms (txn ID: {txn_id})" + ) + except Exception: + self.log.exception( + "Error checking homeserver -> bridge connection, retrying in 10 seconds" + ) + sys.exit(16) + else: + self.log.debug( + "Homeserver does not support checking status of homeserver -> bridge connection" + ) try: self.media_config = await self.az.intent.get_media_repo_config() except Exception: From 7330fb17b84879203ec97e88c0d4cab7ddf6530f Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 1 May 2023 21:50:23 +0300 Subject: [PATCH 323/456] Add beeper_update_profile to methods that call ensure_registered --- mautrix/appservice/api/intent.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mautrix/appservice/api/intent.py b/mautrix/appservice/api/intent.py index e09db0d0..22a93354 100644 --- a/mautrix/appservice/api/intent.py +++ b/mautrix/appservice/api/intent.py @@ -71,6 +71,7 @@ def quote(*args, **kwargs): ClientAPI.search_users, ClientAPI.set_displayname, ClientAPI.set_avatar_url, + ClientAPI.beeper_update_profile, ClientAPI.unstable_create_mxc, ClientAPI.upload_media, ClientAPI.send_receipt, From e1b0dfdeb33953561d796f01a25f858550be259c Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 2 May 2023 15:11:01 +0300 Subject: [PATCH 324/456] Stabilize MSC2659 --- mautrix/appservice/api/intent.py | 4 ++-- mautrix/bridge/matrix.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mautrix/appservice/api/intent.py b/mautrix/appservice/api/intent.py index 22a93354..e52d8016 100644 --- a/mautrix/appservice/api/intent.py +++ b/mautrix/appservice/api/intent.py @@ -493,10 +493,10 @@ async def mark_read( async def appservice_ping(self, appservice_id: str, txn_id: str | None = None) -> int: resp = await self.api.request( Method.POST, - Path.unstable["fi.mau.msc2659"].appservice[appservice_id].ping, + Path.v1.appservice[appservice_id].ping, content={"transaction_id": txn_id} if txn_id is not None else {}, ) - return resp.get("duration_ms") or resp.get("duration") or -1 + return resp.get("duration_ms") or -1 async def batch_send( self, diff --git a/mautrix/bridge/matrix.py b/mautrix/bridge/matrix.py index e6b280a5..45d087f2 100644 --- a/mautrix/bridge/matrix.py +++ b/mautrix/bridge/matrix.py @@ -246,7 +246,7 @@ async def wait_for_connection(self) -> None: ) await self.az.intent.ensure_registered() await self.az.intent.whoami() - if self.versions.supports("fi.mau.msc2659"): + if self.versions.supports("fi.mau.msc2659.stable"): try: txn_id = self.az.intent.api.get_txn_id() duration = await self.az.ping_self(txn_id) From 3d90a2e89e2472834c240d034e68b5d6ce4470bc Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 2 May 2023 15:11:08 +0300 Subject: [PATCH 325/456] Add spec version constants --- mautrix/types/versions.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mautrix/types/versions.py b/mautrix/types/versions.py index 8b42de39..388927e8 100644 --- a/mautrix/types/versions.py +++ b/mautrix/types/versions.py @@ -70,6 +70,9 @@ class SpecVersions: V11 = Version.deserialize("v1.1") V12 = Version.deserialize("v1.2") V13 = Version.deserialize("v1.3") + V14 = Version.deserialize("v1.4") + V15 = Version.deserialize("v1.5") + V16 = Version.deserialize("v1.6") @dataclass From 7b6c727c6d2a0e29bc957609cc0bea1de1f7ac3f Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 5 May 2023 15:00:02 +0300 Subject: [PATCH 326/456] Retry /whoami forever in bridges like mautrix-go --- mautrix/bridge/matrix.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/mautrix/bridge/matrix.py b/mautrix/bridge/matrix.py index 45d087f2..a525b9ac 100644 --- a/mautrix/bridge/matrix.py +++ b/mautrix/bridge/matrix.py @@ -221,19 +221,13 @@ async def check_versions(self) -> None: async def wait_for_connection(self) -> None: self.log.info("Ensuring connectivity to homeserver") - errors = 0 while True: try: self.versions = await self.az.intent.versions() break except Exception: - errors += 1 - if errors <= 6: - self.log.exception("Connection to homeserver failed, retrying in 10 seconds") - await asyncio.sleep(10) - continue - else: - raise + self.log.exception("Connection to homeserver failed, retrying in 10 seconds") + await asyncio.sleep(10) await self.check_versions() try: await self.az.intent.whoami() From 3a339a2f91f93b1eec2b4f16eabb7d6655455281 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 5 May 2023 15:00:52 +0300 Subject: [PATCH 327/456] Enable AS ping if server advertises spec v1.7 --- mautrix/bridge/matrix.py | 4 +++- mautrix/types/versions.py | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/mautrix/bridge/matrix.py b/mautrix/bridge/matrix.py index a525b9ac..204fbbeb 100644 --- a/mautrix/bridge/matrix.py +++ b/mautrix/bridge/matrix.py @@ -240,7 +240,9 @@ async def wait_for_connection(self) -> None: ) await self.az.intent.ensure_registered() await self.az.intent.whoami() - if self.versions.supports("fi.mau.msc2659.stable"): + if self.versions.supports("fi.mau.msc2659.stable") or self.versions.supports_at_least( + SpecVersions.V17 + ): try: txn_id = self.az.intent.api.get_txn_id() duration = await self.az.ping_self(txn_id) diff --git a/mautrix/types/versions.py b/mautrix/types/versions.py index 388927e8..9ad81710 100644 --- a/mautrix/types/versions.py +++ b/mautrix/types/versions.py @@ -73,6 +73,7 @@ class SpecVersions: V14 = Version.deserialize("v1.4") V15 = Version.deserialize("v1.5") V16 = Version.deserialize("v1.6") + V17 = Version.deserialize("v1.7") @dataclass From 858f98c567eb196e58d16313d9bf6340df96c8c8 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 5 May 2023 15:01:35 +0300 Subject: [PATCH 328/456] Remove redundant except --- mautrix/bridge/matrix.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/mautrix/bridge/matrix.py b/mautrix/bridge/matrix.py index 204fbbeb..1fa1e589 100644 --- a/mautrix/bridge/matrix.py +++ b/mautrix/bridge/matrix.py @@ -231,8 +231,6 @@ async def wait_for_connection(self) -> None: await self.check_versions() try: await self.az.intent.whoami() - except (MUnknownToken, MExclusive): - raise except MForbidden: self.log.debug( "Whoami endpoint returned M_FORBIDDEN, " From fda258a0f124b72dcf140a44cc7bf6fec3a89737 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 5 May 2023 15:02:13 +0300 Subject: [PATCH 329/456] Fix error message --- mautrix/bridge/matrix.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/mautrix/bridge/matrix.py b/mautrix/bridge/matrix.py index 1fa1e589..957aa9be 100644 --- a/mautrix/bridge/matrix.py +++ b/mautrix/bridge/matrix.py @@ -249,9 +249,7 @@ async def wait_for_connection(self) -> None: f"roundtrip time is {duration} ms (txn ID: {txn_id})" ) except Exception: - self.log.exception( - "Error checking homeserver -> bridge connection, retrying in 10 seconds" - ) + self.log.exception("Error checking homeserver -> bridge connection") sys.exit(16) else: self.log.debug( From cd23618415e1fde197df955ab3bafd428eae391a Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 5 May 2023 15:11:27 +0300 Subject: [PATCH 330/456] Send MSS events for commands --- mautrix/bridge/matrix.py | 56 +++++++++++++++++++++++++++++++--------- 1 file changed, 44 insertions(+), 12 deletions(-) diff --git a/mautrix/bridge/matrix.py b/mautrix/bridge/matrix.py index 957aa9be..dc7d9fcf 100644 --- a/mautrix/bridge/matrix.py +++ b/mautrix/bridge/matrix.py @@ -557,6 +557,28 @@ def is_command(self, message: MessageEventContent) -> tuple[bool, str]: text = text[len(prefix) + 1 :].lstrip() return is_command, text + async def _send_mss( + self, + evt: Event, + status: MessageStatus, + reason: MessageStatusReason | None = None, + error: str | None = None, + message: str | None = None, + ) -> None: + if not self.config.get("bridge.message_status_events", False): + return + status_content = BeeperMessageStatusEventContent( + network="", # TODO set network properly + relates_to=RelatesTo(rel_type=RelationType.REFERENCE, event_id=evt.event_id), + status=status, + reason=reason, + error=error, + message=message, + ) + await self.az.intent.send_message_event( + evt.room_id, EventType.BEEPER_MESSAGE_STATUS, status_content + ) + async def _send_crypto_status_error( self, evt: Event, @@ -594,18 +616,13 @@ async def _send_crypto_status_error( "it by default on the bridge (bridge -> encryption -> default)." ) - if self.config.get("bridge.message_status_events", False): - status_content = BeeperMessageStatusEventContent( - network="", # TODO set network properly - relates_to=RelatesTo(rel_type=RelationType.REFERENCE, event_id=evt.event_id), - status=MessageStatus.RETRIABLE if is_final else MessageStatus.PENDING, - reason=MessageStatusReason.UNDECRYPTABLE, - error=msg, - message=err.human_message if err else None, - ) - await self.az.intent.send_message_event( - evt.room_id, EventType.BEEPER_MESSAGE_STATUS, status_content - ) + await self._send_mss( + evt, + status=MessageStatus.RETRIABLE if is_final else MessageStatus.PENDING, + reason=MessageStatusReason.UNDECRYPTABLE, + error=msg, + message=err.human_message if err else None, + ) return event_id @@ -697,6 +714,13 @@ async def handle_message(self, evt: MessageEvent, was_encrypted: bool = False) - except Exception as e: self.log.debug(f"Error handling command {command} from {sender}: {e}") self._send_message_checkpoint(evt, MessageSendCheckpointStep.COMMAND, e) + await self._send_mss( + evt, + status=MessageStatus.FAIL, + reason=MessageStatusReason.GENERIC_ERROR, + error="", + message="Command execution failed", + ) else: await MessageSendCheckpoint( event_id=event_id, @@ -712,6 +736,7 @@ async def handle_message(self, evt: MessageEvent, was_encrypted: bool = False) - self.az.as_token, self.log, ) + await self._send_mss(evt, status=MessageStatus.SUCCESS) else: self.log.debug( f"Ignoring event {event_id} from {sender.mxid}:" @@ -720,6 +745,13 @@ async def handle_message(self, evt: MessageEvent, was_encrypted: bool = False) - self._send_message_checkpoint( evt, MessageSendCheckpointStep.COMMAND, "not a command and not a portal room" ) + await self._send_mss( + evt, + status=MessageStatus.FAIL, + reason=MessageStatusReason.UNSUPPORTED, + error="Unknown room", + message="Unknown room", + ) async def _is_direct_chat(self, room_id: RoomID) -> tuple[bool, bool]: try: From 4ebc1a3ec56783869f6a104d60b577426b4dacda Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 5 May 2023 18:44:45 +0300 Subject: [PATCH 331/456] Reuse aiosqlite connection pool for crypto db Otherwise stopping bridges gets stuck because the crypto pool isn't closed. asyncpg was already reusing the pool. --- mautrix/bridge/bridge.py | 5 ++++- mautrix/util/async_db/aiosqlite.py | 20 ++++++++++++++++++-- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/mautrix/bridge/bridge.py b/mautrix/bridge/bridge.py index c474109b..1dd9c0f9 100644 --- a/mautrix/bridge/bridge.py +++ b/mautrix/bridge/bridge.py @@ -262,8 +262,11 @@ async def start(self) -> None: async def system_exit(self) -> None: if hasattr(self, "db") and isinstance(self.db, Database): - self.log.trace("Stopping database due to SystemExit") + self.log.debug("Stopping database due to SystemExit") await self.db.stop() + self.log.debug("Database stopped") + elif getattr(self, "db", None): + self.log.trace("Database not started at SystemExit") async def stop(self) -> None: if self.manhole: diff --git a/mautrix/util/async_db/aiosqlite.py b/mautrix/util/async_db/aiosqlite.py index 9b7fa16a..22c2ffa6 100644 --- a/mautrix/util/async_db/aiosqlite.py +++ b/mautrix/util/async_db/aiosqlite.py @@ -5,7 +5,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import annotations -from typing import Any +from typing import Any, AsyncContextManager from contextlib import asynccontextmanager import asyncio import logging @@ -81,6 +81,7 @@ async def fetchval( class SQLiteDatabase(Database): scheme = Scheme.SQLITE + _parent: SQLiteDatabase | None _pool: asyncio.Queue[TxnConnection] _stopped: bool _conns: int @@ -103,6 +104,7 @@ def __init__( owner_name=owner_name, ignore_foreign_tables=ignore_foreign_tables, ) + self._parent = None self._path = url.path if self._path.startswith("/"): self._path = self._path[1:] @@ -134,7 +136,14 @@ def _add_missing_pragmas(init_commands: list[str]) -> list[str]: init_commands.append("PRAGMA busy_timeout = 5000") return init_commands + def override_pool(self, db: Database) -> None: + assert isinstance(db, SQLiteDatabase) + self._parent = db + async def start(self) -> None: + if self._parent: + await super().start() + return if self._conns: raise RuntimeError("database pool has already been started") elif self._stopped: @@ -155,14 +164,21 @@ async def start(self) -> None: await super().start() async def stop(self) -> None: + if self._parent: + return self._stopped = True while self._conns > 0: conn = await self._pool.get() self._conns -= 1 await conn.close() + def acquire(self) -> AsyncContextManager[LoggingConnection]: + if self._parent: + return self._parent.acquire() + return self._acquire() + @asynccontextmanager - async def acquire(self) -> LoggingConnection: + async def _acquire(self) -> LoggingConnection: if self._stopped: raise RuntimeError("database pool has been stopped") conn = await self._pool.get() From 488ce5cb0582b68f5b9558f3e30900aed79656b4 Mon Sep 17 00:00:00 2001 From: Nick Barrett Date: Tue, 16 May 2023 10:38:11 +0100 Subject: [PATCH 332/456] Use the current proxy as fallback when failing to fetch a new one Prevents accidentally resetting the proxy! --- mautrix/util/proxy.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mautrix/util/proxy.py b/mautrix/util/proxy.py index 5edd6531..4b5a4153 100644 --- a/mautrix/util/proxy.py +++ b/mautrix/util/proxy.py @@ -51,11 +51,10 @@ def get_proxy_url_from_api(self, reason: str | None = None) -> str | None: response = json.loads(f.read().decode()) except Exception: self.log.exception("Failed to retrieve proxy from API") + return self.current_proxy_url else: return response["proxy_url"] - return None - def update_proxy_url(self, reason: str | None = None) -> bool: old_proxy = self.current_proxy_url new_proxy = None From 27e3e0a7a587824b5672bda40fdfa3e6aee4d417 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 16 May 2023 12:44:21 +0300 Subject: [PATCH 333/456] Bump version to 0.19.14 --- CHANGELOG.md | 7 +++++++ mautrix/__init__.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 730f2a2d..4dda779a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## v0.19.14 (2023-05-16) + +* *(bridge)* Implemented appservice pinging using MSC2659. +* *(bridge)* Started reusing aiosqlite connection pool for crypto db. + * This fixes the crypto pool getting stuck if the bridge exits unexpectedly + (the default pool is closed automatically at any type of exit). + ## v0.19.13 (2023-04-24) * *(crypto)* Fixed bug with redacting megolm sessions when device is deleted. diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 440745cf..f3419a7b 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.19.13" +__version__ = "0.19.14" __author__ = "Tulir Asokan " __all__ = [ "api", From 21b97d7e3116b648a4f0c6cca34aaad9a2c32925 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 17 May 2023 20:52:20 +0300 Subject: [PATCH 334/456] Add trace logs for incorrect tokens in AS requests --- mautrix/appservice/as_handler.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mautrix/appservice/as_handler.py b/mautrix/appservice/as_handler.py index 2adcb2de..b49a8583 100644 --- a/mautrix/appservice/as_handler.py +++ b/mautrix/appservice/as_handler.py @@ -85,9 +85,11 @@ def _check_token(self, request: web.Request) -> bool: try: token = request.headers["Authorization"].removeprefix("Bearer ") except (KeyError, AttributeError): + self.log.trace("No access_token nor Authorization header in request") return False if token != self.hs_token: + self.log.trace(f"Incorrect hs_token in request ({token!r} != {self.hs_token!r})") return False return True From 204342da0a57311742e656dd8e5dbe01b3796647 Mon Sep 17 00:00:00 2001 From: Nick Mills-Barrett Date: Thu, 18 May 2023 17:40:01 +0100 Subject: [PATCH 335/456] Add `reset_after_seconds` to `proxy_with_retry` helper This is handy where wrapping long running tasks that may fail from time to time (hours) and we don't want them to raise an exception if only failing rarely (and presumably running fine most of the time). --- mautrix/util/proxy.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/mautrix/util/proxy.py b/mautrix/util/proxy.py index 4b5a4153..3041d164 100644 --- a/mautrix/util/proxy.py +++ b/mautrix/util/proxy.py @@ -4,6 +4,7 @@ import asyncio import json import logging +import time import urllib.request from aiohttp import ClientConnectionError @@ -93,8 +94,10 @@ async def proxy_with_retry( max_wait_seconds: int = 60, multiply_wait_seconds: int = 10, retryable_exceptions: tuple[Exception] = RETRYABLE_PROXY_EXCEPTIONS, + reset_after_seconds: int | None = None, ) -> T: errors = 0 + last_error = 0 while True: try: @@ -116,3 +119,11 @@ async def proxy_with_retry( f"{e.__class__.__name__} while trying to {name}" ): await on_proxy_change() + + # If sufficient time has passed since the previous error, reset the + # error count. Useful for long running tasks with rare failures. + if reset_after_seconds is not None: + now = time.time() + if now - last_error > reset_after_seconds: + errors = 0 + last_error = now From 50fa320617dd471fe74a54a98e3ec5d4fc066c26 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 22 May 2023 19:13:42 +0300 Subject: [PATCH 336/456] Handle ephemeral events inside rooms in sync --- mautrix/client/syncer.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/mautrix/client/syncer.py b/mautrix/client/syncer.py index 9f942665..62ae97a1 100644 --- a/mautrix/client/syncer.py +++ b/mautrix/client/syncer.py @@ -340,6 +340,13 @@ def handle_sync(self, data: JSON) -> list[asyncio.Task]: self._try_deserialize(Event, raw_event), source=SyncStream.JOINED_ROOM | SyncStream.TIMELINE, ) + + for raw_event in room_data.get("ephemeral", {}).get("events", []): + raw_event["room_id"] = room_id + tasks += self.dispatch_event( + self._try_deserialize(EphemeralEvent, raw_event), + source=SyncStream.JOINED_ROOM | SyncStream.EPHEMERAL, + ) for room_id, room_data in rooms.get("invite", {}).items(): events: list[dict[str, JSON]] = room_data.get("invite_state", {}).get("events", []) for raw_event in events: From 437e6e585fc26be682d080f736fd4802fbbf2213 Mon Sep 17 00:00:00 2001 From: Nick Mills-Barrett Date: Wed, 24 May 2023 10:39:41 +0100 Subject: [PATCH 337/456] Fix check for last error when using `reset_after_seconds` --- mautrix/util/proxy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/util/proxy.py b/mautrix/util/proxy.py index 3041d164..f36da73d 100644 --- a/mautrix/util/proxy.py +++ b/mautrix/util/proxy.py @@ -124,6 +124,6 @@ async def proxy_with_retry( # error count. Useful for long running tasks with rare failures. if reset_after_seconds is not None: now = time.time() - if now - last_error > reset_after_seconds: + if last_error and now - last_error > reset_after_seconds: errors = 0 last_error = now From 0da6f45c9921a1e6b7cccd97113f2a31c670feba Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 24 May 2023 12:45:25 +0300 Subject: [PATCH 338/456] Bump version to 0.19.15 --- CHANGELOG.md | 4 ++++ mautrix/__init__.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4dda779a..d43a914d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## v0.19.15 (2023-05-24) + +* *(client)* Fixed dispatching room ephemeral events (i.e. typing notifications) in syncer. + ## v0.19.14 (2023-05-16) * *(bridge)* Implemented appservice pinging using MSC2659. diff --git a/mautrix/__init__.py b/mautrix/__init__.py index f3419a7b..ce7df73a 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.19.14" +__version__ = "0.19.15" __author__ = "Tulir Asokan " __all__ = [ "api", From fbbe039b85974b993079fcd92d5a4aa354b53855 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 24 May 2023 19:58:45 +0300 Subject: [PATCH 339/456] Use path as-is for SQLite database Also add bridge config migration to remove extra slashes --- mautrix/bridge/config.py | 7 ++++++- mautrix/util/async_db/aiosqlite.py | 2 -- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/mautrix/bridge/config.py b/mautrix/bridge/config.py index f3e7720c..2674256b 100644 --- a/mautrix/bridge/config.py +++ b/mautrix/bridge/config.py @@ -115,7 +115,12 @@ def do_update(self, helper: ConfigUpdateHelper) -> None: copy("appservice.tls_cert") copy("appservice.tls_key") - copy("appservice.database") + if "appservice.database" in self and self["appservice.database"].startswith("sqlite:///"): + helper.base["appservice.database"] = self["appservice.database"].replace( + "sqlite:///", "sqlite:" + ) + else: + copy("appservice.database") copy("appservice.database_opts") copy("appservice.id") diff --git a/mautrix/util/async_db/aiosqlite.py b/mautrix/util/async_db/aiosqlite.py index 22c2ffa6..d9857b8c 100644 --- a/mautrix/util/async_db/aiosqlite.py +++ b/mautrix/util/async_db/aiosqlite.py @@ -106,8 +106,6 @@ def __init__( ) self._parent = None self._path = url.path - if self._path.startswith("/"): - self._path = self._path[1:] self._pool = asyncio.Queue(self._db_args.pop("min_size", 1)) self._db_args.pop("max_size", None) self._stopped = False From 7c60ee9175ad7d9edb64e8095c181e60485a6990 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 18 May 2023 12:37:26 +0300 Subject: [PATCH 340/456] Remove SQLAlchemy state store stuff. Fixes #147 --- docs/api/mautrix.client.state_store/index.rst | 1 - .../mautrix.client.state_store/sqlalchemy.rst | 5 - mautrix/appservice/state_store/__init__.py | 2 +- mautrix/appservice/state_store/sqlalchemy.py | 14 -- mautrix/bridge/crypto_state_store.py | 23 --- mautrix/bridge/e2ee.py | 7 - mautrix/bridge/state_store/__init__.py | 2 +- mautrix/bridge/state_store/sqlalchemy.py | 41 ---- mautrix/client/state_store/__init__.py | 1 - .../client/state_store/sqlalchemy/__init__.py | 3 - .../state_store/sqlalchemy/mx_room_state.py | 66 ------- .../state_store/sqlalchemy/mx_user_profile.py | 94 --------- .../state_store/sqlalchemy/sqlstatestore.py | 183 ------------------ .../client/state_store/tests/store_test.py | 15 +- setup.py | 2 +- 15 files changed, 4 insertions(+), 455 deletions(-) delete mode 100644 docs/api/mautrix.client.state_store/sqlalchemy.rst delete mode 100644 mautrix/appservice/state_store/sqlalchemy.py delete mode 100644 mautrix/bridge/state_store/sqlalchemy.py delete mode 100644 mautrix/client/state_store/sqlalchemy/__init__.py delete mode 100644 mautrix/client/state_store/sqlalchemy/mx_room_state.py delete mode 100644 mautrix/client/state_store/sqlalchemy/mx_user_profile.py delete mode 100644 mautrix/client/state_store/sqlalchemy/sqlstatestore.py diff --git a/docs/api/mautrix.client.state_store/index.rst b/docs/api/mautrix.client.state_store/index.rst index 91f2ba42..0a934308 100644 --- a/docs/api/mautrix.client.state_store/index.rst +++ b/docs/api/mautrix.client.state_store/index.rst @@ -13,5 +13,4 @@ Implementations In-memory Async database (asyncpg/aiosqlite) - Legacy database (SQLAlchemy) Flat file diff --git a/docs/api/mautrix.client.state_store/sqlalchemy.rst b/docs/api/mautrix.client.state_store/sqlalchemy.rst deleted file mode 100644 index 767e0595..00000000 --- a/docs/api/mautrix.client.state_store/sqlalchemy.rst +++ /dev/null @@ -1,5 +0,0 @@ -mautrix.client.state\_store.sqlalchemy -====================================== - -.. autoclass:: mautrix.client.state_store.sqlalchemy.SQLStateStore - :no-undoc-members: diff --git a/mautrix/appservice/state_store/__init__.py b/mautrix/appservice/state_store/__init__.py index bc23b5f5..771ac252 100644 --- a/mautrix/appservice/state_store/__init__.py +++ b/mautrix/appservice/state_store/__init__.py @@ -1,4 +1,4 @@ from .file import FileASStateStore from .memory import ASStateStore -__all__ = ["FileASStateStore", "ASStateStore", "sqlalchemy", "asyncpg"] +__all__ = ["FileASStateStore", "ASStateStore", "asyncpg"] diff --git a/mautrix/appservice/state_store/sqlalchemy.py b/mautrix/appservice/state_store/sqlalchemy.py deleted file mode 100644 index 30c0b1fc..00000000 --- a/mautrix/appservice/state_store/sqlalchemy.py +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright (c) 2022 Tulir Asokan -# -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. -from mautrix.client.state_store.sqlalchemy import SQLStateStore as SQLClientStateStore - -from .memory import ASStateStore - - -class SQLASStateStore(SQLClientStateStore, ASStateStore): - def __init__(self) -> None: - SQLClientStateStore.__init__(self) - ASStateStore.__init__(self) diff --git a/mautrix/bridge/crypto_state_store.py b/mautrix/bridge/crypto_state_store.py index 84169e2e..f08dec2c 100644 --- a/mautrix/bridge/crypto_state_store.py +++ b/mautrix/bridge/crypto_state_store.py @@ -28,29 +28,6 @@ async def is_encrypted(self, room_id: RoomID) -> bool: return portal.encrypted if portal else False -try: - from mautrix.client.state_store.sqlalchemy import RoomState, UserProfile - - class SQLCryptoStateStore(BaseCryptoStateStore): - @staticmethod - async def find_shared_rooms(user_id: UserID) -> list[RoomID]: - return [profile.room_id for profile in UserProfile.find_rooms_with_user(user_id)] - - @staticmethod - async def get_encryption_info(room_id: RoomID) -> RoomEncryptionStateEventContent | None: - state = RoomState.get(room_id) - if not state: - return None - return state.encryption - -except ImportError: - if __optional_imports__: - raise - UserProfile = None - RoomState = None - SQLCryptoStateStore = None - - class PgCryptoStateStore(BaseCryptoStateStore): db: Database diff --git a/mautrix/bridge/e2ee.py b/mautrix/bridge/e2ee.py index 805efd4f..3961be58 100644 --- a/mautrix/bridge/e2ee.py +++ b/mautrix/bridge/e2ee.py @@ -41,13 +41,6 @@ from .. import bridge as br from .crypto_state_store import PgCryptoStateStore -try: - from mautrix.client.state_store.sqlalchemy import UserProfile -except ImportError: - if __optional_imports__: - raise - UserProfile = None - class EncryptionManager: loop: asyncio.AbstractEventLoop diff --git a/mautrix/bridge/state_store/__init__.py b/mautrix/bridge/state_store/__init__.py index 0e8137a5..b990ac86 100644 --- a/mautrix/bridge/state_store/__init__.py +++ b/mautrix/bridge/state_store/__init__.py @@ -1 +1 @@ -__all__ = ["asyncpg", "sqlalchemy"] +__all__ = ["asyncpg"] diff --git a/mautrix/bridge/state_store/sqlalchemy.py b/mautrix/bridge/state_store/sqlalchemy.py deleted file mode 100644 index 2fa7353e..00000000 --- a/mautrix/bridge/state_store/sqlalchemy.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright (c) 2022 Tulir Asokan -# -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import annotations - -from typing import Awaitable, Callable, Union - -from mautrix.appservice.state_store.sqlalchemy import SQLASStateStore -from mautrix.types import UserID - -from ..puppet import BasePuppet - -GetPuppetFunc = Union[ - Callable[[UserID], Awaitable[BasePuppet]], Callable[[UserID, bool], Awaitable[BasePuppet]] -] - - -class SQLBridgeStateStore(SQLASStateStore): - def __init__(self, get_puppet: GetPuppetFunc, get_double_puppet: GetPuppetFunc) -> None: - super().__init__() - self.get_puppet = get_puppet - self.get_double_puppet = get_double_puppet - - async def is_registered(self, user_id: UserID) -> bool: - puppet = await self.get_puppet(user_id) - if puppet: - return puppet.is_registered - custom_puppet = await self.get_double_puppet(user_id) - if custom_puppet: - return True - return await super().is_registered(user_id) - - async def registered(self, user_id: UserID) -> None: - puppet = await self.get_puppet(user_id, True) - if puppet: - puppet.is_registered = True - await puppet.save() - else: - await super().registered(user_id) diff --git a/mautrix/client/state_store/__init__.py b/mautrix/client/state_store/__init__.py index 5f76c7dc..98150ca4 100644 --- a/mautrix/client/state_store/__init__.py +++ b/mautrix/client/state_store/__init__.py @@ -10,5 +10,4 @@ "MemorySyncStore", "SyncStore", "asyncpg", - "sqlalchemy", ] diff --git a/mautrix/client/state_store/sqlalchemy/__init__.py b/mautrix/client/state_store/sqlalchemy/__init__.py deleted file mode 100644 index 6ce78f46..00000000 --- a/mautrix/client/state_store/sqlalchemy/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .mx_room_state import RoomState, SerializableType -from .mx_user_profile import UserProfile -from .sqlstatestore import SQLStateStore diff --git a/mautrix/client/state_store/sqlalchemy/mx_room_state.py b/mautrix/client/state_store/sqlalchemy/mx_room_state.py deleted file mode 100644 index 9f97056f..00000000 --- a/mautrix/client/state_store/sqlalchemy/mx_room_state.py +++ /dev/null @@ -1,66 +0,0 @@ -# Copyright (c) 2022 Tulir Asokan -# -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import annotations - -from typing import Type -import json - -from sqlalchemy import Boolean, Column, Text, types - -from mautrix.types import ( - PowerLevelStateEventContent as PowerLevels, - RoomEncryptionStateEventContent as EncryptionInfo, - RoomID, - Serializable, -) -from mautrix.util.db import Base - - -class SerializableType(types.TypeDecorator): - impl = types.Text - - def __init__(self, python_type: Type[Serializable], *args, **kwargs) -> None: - super().__init__(*args, **kwargs) - self._python_type = python_type - - @property - def python_type(self) -> Type[Serializable]: - return self._python_type - - def process_bind_param(self, value: Serializable, dialect) -> str | None: - if value is not None: - return json.dumps(value.serialize() if isinstance(value, Serializable) else value) - return None - - def process_result_value(self, value: str, dialect) -> Serializable | None: - if value is not None: - return self.python_type.deserialize(json.loads(value)) - return None - - def process_literal_param(self, value, dialect): - return value - - -class RoomState(Base): - __tablename__ = "mx_room_state" - - room_id: RoomID = Column(Text, primary_key=True) - is_encrypted: bool = Column(Boolean, nullable=True) - has_full_member_list: bool = Column(Boolean, nullable=True) - encryption: EncryptionInfo = Column(SerializableType(EncryptionInfo), nullable=True) - power_levels: PowerLevels = Column(SerializableType(PowerLevels), nullable=True) - - @property - def has_power_levels(self) -> bool: - return bool(self.power_levels) - - @property - def has_encryption_info(self) -> bool: - return self.is_encrypted is not None - - @classmethod - def get(cls, room_id: RoomID) -> RoomState | None: - return cls._select_one_or_none(cls.c.room_id == room_id) diff --git a/mautrix/client/state_store/sqlalchemy/mx_user_profile.py b/mautrix/client/state_store/sqlalchemy/mx_user_profile.py deleted file mode 100644 index 0bb6b766..00000000 --- a/mautrix/client/state_store/sqlalchemy/mx_user_profile.py +++ /dev/null @@ -1,94 +0,0 @@ -# Copyright (c) 2022 Tulir Asokan -# -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import annotations - -from typing import Iterable - -from sqlalchemy import Column, Enum, Text - -from mautrix.types import ContentURI, Member, Membership, RoomID, UserID -from mautrix.util.db import Base - -from .mx_room_state import RoomState - - -class UserProfile(Base): - __tablename__ = "mx_user_profile" - - room_id: RoomID = Column(Text, primary_key=True) - user_id: UserID = Column(Text, primary_key=True) - membership: Membership = Column(Enum(Membership), nullable=False, default=Membership.LEAVE) - displayname: str = Column(Text, nullable=True) - avatar_url: ContentURI = Column(Text, nullable=True) - - def member(self) -> Member: - return Member( - membership=self.membership, displayname=self.displayname, avatar_url=self.avatar_url - ) - - @classmethod - def get(cls, room_id: RoomID, user_id: UserID) -> UserProfile | None: - return cls._select_one_or_none((cls.c.room_id == room_id) & (cls.c.user_id == user_id)) - - @classmethod - def all_in_room( - cls, - room_id: RoomID, - memberships: tuple[Membership, ...], - prefix: str = None, - suffix: str = None, - bot: str = None, - ) -> Iterable[UserProfile]: - clause = cls.c.membership == memberships[0] - for membership in memberships[1:]: - clause |= cls.c.membership == membership - clause &= cls.c.room_id == room_id - if bot: - clause &= cls.c.user_id != bot - if prefix: - clause &= ~cls.c.user_id.startswith(prefix, autoescape=True) - if suffix: - clause &= ~cls.c.user_id.startswith(suffix, autoescape=True) - return cls._select_all(clause) - - @classmethod - def find_rooms_with_user(cls, user_id: UserID) -> Iterable[UserProfile]: - return cls._select_all( - (cls.c.user_id == user_id) - & (cls.c.room_id == RoomState.c.room_id) - & (RoomState.c.is_encrypted == True) - ) - - @classmethod - def delete_all(cls, room_id: RoomID) -> None: - with cls.db.begin() as conn: - conn.execute(cls.t.delete().where(cls.c.room_id == room_id)) - - @classmethod - def bulk_replace( - cls, - room_id: RoomID, - members: dict[UserID, Member], - only_membership: Membership | None = None, - ) -> None: - with cls.db.begin() as conn: - delete_condition = cls.c.room_id == room_id - if only_membership is not None: - delete_condition &= cls.c.membership == only_membership - cls.db.execute(cls.t.delete().where(delete_condition)) - conn.execute( - cls.t.insert(), - [ - dict( - room_id=room_id, - user_id=user_id, - membership=member.membership, - displayname=member.displayname, - avatar_url=member.avatar_url, - ) - for user_id, member in members.items() - ], - ) diff --git a/mautrix/client/state_store/sqlalchemy/sqlstatestore.py b/mautrix/client/state_store/sqlalchemy/sqlstatestore.py deleted file mode 100644 index 1145dec7..00000000 --- a/mautrix/client/state_store/sqlalchemy/sqlstatestore.py +++ /dev/null @@ -1,183 +0,0 @@ -# Copyright (c) 2022 Tulir Asokan -# -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import annotations - -from typing import Any - -from mautrix.types import ( - Member, - Membership, - PowerLevelStateEventContent, - RoomEncryptionStateEventContent, - RoomID, - UserID, -) - -from ..abstract import StateStore -from .mx_room_state import RoomState -from .mx_user_profile import UserProfile - - -class SQLStateStore(StateStore): - _profile_cache: dict[RoomID, dict[UserID, UserProfile]] - _room_state_cache: dict[RoomID, RoomState] - - def __init__(self) -> None: - super().__init__() - self._profile_cache = {} - self._room_state_cache = {} - - def _get_user_profile( - self, room_id: RoomID, user_id: UserID, create: bool = False - ) -> UserProfile: - if not room_id: - raise ValueError("room_id is empty") - elif not user_id: - raise ValueError("user_id is empty") - try: - return self._profile_cache[room_id][user_id] - except KeyError: - pass - if room_id not in self._profile_cache: - self._profile_cache[room_id] = {} - - profile = UserProfile.get(room_id, user_id) - if profile: - self._profile_cache[room_id][user_id] = profile - elif create: - profile = UserProfile(room_id=room_id, user_id=user_id, membership=Membership.LEAVE) - profile.insert() - self._profile_cache[room_id][user_id] = profile - return profile - - async def get_member(self, room_id: RoomID, user_id: UserID) -> Member | None: - profile = self._get_user_profile(room_id, user_id) - if not profile: - return None - return profile.member() - - async def set_member(self, room_id: RoomID, user_id: UserID, member: Member) -> None: - if not member: - raise ValueError("member info is empty") - profile = self._get_user_profile(room_id, user_id, create=True) - profile.edit( - membership=member.membership, - displayname=member.displayname or profile.displayname, - avatar_url=member.avatar_url or profile.avatar_url, - ) - - async def set_membership( - self, room_id: RoomID, user_id: UserID, membership: Membership - ) -> None: - await self.set_member(room_id, user_id, Member(membership=membership)) - - async def get_member_profiles( - self, - room_id: RoomID, - memberships: tuple[Membership, ...] = (Membership.JOIN, Membership.INVITE), - ) -> dict[UserID, Member]: - self._profile_cache[room_id] = {} - for profile in UserProfile.all_in_room(room_id, memberships): - self._profile_cache[room_id][profile.user_id] = profile - return { - profile.user_id: profile.member() for profile in self._profile_cache[room_id].values() - } - - async def get_members_filtered( - self, - room_id: RoomID, - not_prefix: str, - not_suffix: str, - not_id: str, - memberships: tuple[Membership, ...] = (Membership.JOIN, Membership.INVITE), - ) -> list[UserID]: - return [ - profile.user_id - for profile in UserProfile.all_in_room( - room_id, memberships, not_suffix, not_prefix, not_id - ) - ] - - async def set_members( - self, - room_id: RoomID, - members: dict[UserID, Member], - only_membership: Membership | None = None, - ) -> None: - UserProfile.bulk_replace(room_id, members, only_membership=only_membership) - self._get_room_state(room_id, create=True).edit(has_full_member_list=True) - try: - del self._profile_cache[room_id] - except KeyError: - pass - - async def has_full_member_list(self, room_id: RoomID) -> bool: - room = self._get_room_state(room_id) - if not room: - return False - return room.has_full_member_list - - async def find_shared_rooms(self, user_id: UserID) -> list[RoomID]: - return [profile.room_id for profile in UserProfile.find_rooms_with_user(user_id)] - - def _get_room_state(self, room_id: RoomID, create: bool = False) -> RoomState: - if not room_id: - raise ValueError("room_id is empty") - try: - return self._room_state_cache[room_id] - except KeyError: - pass - - room = RoomState.get(room_id) - if room: - self._room_state_cache[room_id] = room - elif create: - room = RoomState(room_id=room_id) - room.insert() - self._room_state_cache[room_id] = room - return room - - async def has_power_levels_cached(self, room_id: RoomID) -> bool: - room = self._get_room_state(room_id) - if not room: - return False - return room.has_power_levels - - async def get_power_levels(self, room_id: RoomID) -> PowerLevelStateEventContent | None: - room = self._get_room_state(room_id) - if not room: - return None - return room.power_levels - - async def set_power_levels( - self, room_id: RoomID, content: PowerLevelStateEventContent - ) -> None: - if not content: - raise ValueError("content is empty") - self._get_room_state(room_id, create=True).edit(power_levels=content) - - async def is_encrypted(self, room_id: RoomID) -> bool | None: - room = self._get_room_state(room_id) - if not room: - return None - return room.is_encrypted - - async def has_encryption_info_cached(self, room_id: RoomID) -> bool: - room = self._get_room_state(room_id) - return room and room.has_encryption_info - - async def get_encryption_info(self, room_id: RoomID) -> RoomEncryptionStateEventContent | None: - room = self._get_room_state(room_id) - if not room: - return None - return room.encryption - - async def set_encryption_info( - self, room_id: RoomID, content: RoomEncryptionStateEventContent | dict[str, Any] - ) -> None: - if not content: - raise ValueError("content is empty") - self._get_room_state(room_id, create=True).edit(encryption=content, is_encrypted=True) diff --git a/mautrix/client/state_store/tests/store_test.py b/mautrix/client/state_store/tests/store_test.py index dbbd376a..f81ff76d 100644 --- a/mautrix/client/state_store/tests/store_test.py +++ b/mautrix/client/state_store/tests/store_test.py @@ -16,7 +16,6 @@ import asyncpg import pytest -import sqlalchemy as sql from mautrix.types import EncryptionAlgorithm, Member, Membership, RoomID, StateEvent, UserID from mautrix.util.async_db import Database @@ -24,7 +23,6 @@ from .. import MemoryStateStore, StateStore from ..asyncpg import PgStateStore -from ..sqlalchemy import RoomState, SQLStateStore, UserProfile @asynccontextmanager @@ -62,23 +60,12 @@ async def async_sqlite_store() -> AsyncIterator[PgStateStore]: await db.stop() -@asynccontextmanager -async def alchemy_store() -> AsyncIterator[SQLStateStore]: - db = sql.create_engine("sqlite:///:memory:") - Base.metadata.bind = db - for table in (RoomState, UserProfile): - table.bind(db) - Base.metadata.create_all() - yield SQLStateStore() - db.dispose() - - @asynccontextmanager async def memory_store() -> AsyncIterator[MemoryStateStore]: yield MemoryStateStore() -@pytest.fixture(params=[async_postgres_store, async_sqlite_store, alchemy_store, memory_store]) +@pytest.fixture(params=[async_postgres_store, async_sqlite_store, memory_store]) async def store(request) -> AsyncIterator[StateStore]: param: Callable[[], AsyncContextManager[StateStore]] = request.param async with param() as state_store: diff --git a/setup.py b/setup.py index cabdf15a..a345d37e 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ from mautrix import __version__ encryption_dependencies = ["python-olm", "unpaddedbase64", "pycryptodome"] -test_dependencies = ["aiosqlite", "sqlalchemy<2", "asyncpg", *encryption_dependencies] +test_dependencies = ["aiosqlite", "asyncpg", *encryption_dependencies] setuptools.setup( name="mautrix", From 1af59f960bd4aba4afef09ea720078bf3e5e86bd Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 24 May 2023 20:08:30 +0300 Subject: [PATCH 341/456] Update changelog --- CHANGELOG.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d43a914d..9531d6f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,14 @@ +## v0.20.0 (unreleased) + +* **Breaking change *(.state_store)*** Removed legacy SQLAlchemy state store + implementations. +* **Mildly breaking change *(util.async_db)*** Changed `SQLiteDatabase` to not + remove prefix slashes from database paths. + * Library users should use `sqlite:path.db` instead of `sqlite:///path.db` + for relative paths, and `sqlite:/path.db` instead of `sqlite:////path.db` + for absolute paths. + * Bridge configs do this migration automatically. + ## v0.19.15 (2023-05-24) * *(client)* Fixed dispatching room ephemeral events (i.e. typing notifications) in syncer. From 6faf041f0a285423ddb1ace48a2cab6dceed8393 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 24 May 2023 20:31:00 +0300 Subject: [PATCH 342/456] Remove extra import --- mautrix/client/state_store/tests/store_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/mautrix/client/state_store/tests/store_test.py b/mautrix/client/state_store/tests/store_test.py index f81ff76d..750b87a9 100644 --- a/mautrix/client/state_store/tests/store_test.py +++ b/mautrix/client/state_store/tests/store_test.py @@ -19,7 +19,6 @@ from mautrix.types import EncryptionAlgorithm, Member, Membership, RoomID, StateEvent, UserID from mautrix.util.async_db import Database -from mautrix.util.db import Base from .. import MemoryStateStore, StateStore from ..asyncpg import PgStateStore From 1d49625f87c86f82da394ac40c8249eddaaac844 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 24 May 2023 20:31:51 +0300 Subject: [PATCH 343/456] Fix SQLite paths in tests --- mautrix/client/state_store/tests/store_test.py | 2 +- mautrix/crypto/store/tests/store_test.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mautrix/client/state_store/tests/store_test.py b/mautrix/client/state_store/tests/store_test.py index 750b87a9..46eee750 100644 --- a/mautrix/client/state_store/tests/store_test.py +++ b/mautrix/client/state_store/tests/store_test.py @@ -51,7 +51,7 @@ async def async_postgres_store() -> AsyncIterator[PgStateStore]: @asynccontextmanager async def async_sqlite_store() -> AsyncIterator[PgStateStore]: db = Database.create( - "sqlite:///:memory:", upgrade_table=PgStateStore.upgrade_table, db_args={"min_size": 1} + "sqlite::memory:", upgrade_table=PgStateStore.upgrade_table, db_args={"min_size": 1} ) store = PgStateStore(db) await db.start() diff --git a/mautrix/crypto/store/tests/store_test.py b/mautrix/crypto/store/tests/store_test.py index 8d3fc851..949949b8 100644 --- a/mautrix/crypto/store/tests/store_test.py +++ b/mautrix/crypto/store/tests/store_test.py @@ -50,7 +50,7 @@ async def async_postgres_store() -> AsyncIterator[PgCryptoStore]: @asynccontextmanager async def async_sqlite_store() -> AsyncIterator[PgCryptoStore]: db = Database.create( - "sqlite:///:memory:", upgrade_table=PgCryptoStore.upgrade_table, db_args={"min_size": 1} + "sqlite::memory:", upgrade_table=PgCryptoStore.upgrade_table, db_args={"min_size": 1} ) store = PgCryptoStore("", "test", db) await db.start() From 40328415b9d2a18c74d12351f51577d5b1ea90d5 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 26 May 2023 21:04:30 +0300 Subject: [PATCH 344/456] Drop Python 3.8 compatibility --- .github/workflows/python-package.yml | 2 +- CHANGELOG.md | 5 +++++ setup.py | 3 +-- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 96b55773..8c6c7606 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -9,7 +9,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.8", "3.9", "3.10", "3.11"] + python-version: ["3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v3 diff --git a/CHANGELOG.md b/CHANGELOG.md index 9531d6f7..da1a8d5a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ ## v0.20.0 (unreleased) +* Dropped Python 3.8 support. * **Breaking change *(.state_store)*** Removed legacy SQLAlchemy state store implementations. * **Mildly breaking change *(util.async_db)*** Changed `SQLiteDatabase` to not @@ -9,6 +10,10 @@ for absolute paths. * Bridge configs do this migration automatically. +## v0.19.16 (2023-05-26) + +* *(appservice)* Fixed Python 3.8 compatibility. + ## v0.19.15 (2023-05-24) * *(client)* Fixed dispatching room ephemeral events (i.e. typing notifications) in syncer. diff --git a/setup.py b/setup.py index a345d37e..04a2a8c3 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,7 @@ "encryption": encryption_dependencies, }, tests_require=test_dependencies, - python_requires="~=3.8", + python_requires="~=3.9", classifiers=[ "Development Status :: 4 - Beta", @@ -42,7 +42,6 @@ "Framework :: AsyncIO", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", From c1e3a66465b814c4c6319b2ad94b8b9983817357 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 26 May 2023 21:08:11 +0300 Subject: [PATCH 345/456] Only catch KeyError in HS token checker --- mautrix/appservice/as_handler.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mautrix/appservice/as_handler.py b/mautrix/appservice/as_handler.py index b49a8583..c0a41f73 100644 --- a/mautrix/appservice/as_handler.py +++ b/mautrix/appservice/as_handler.py @@ -84,12 +84,12 @@ def _check_token(self, request: web.Request) -> bool: except KeyError: try: token = request.headers["Authorization"].removeprefix("Bearer ") - except (KeyError, AttributeError): - self.log.trace("No access_token nor Authorization header in request") + except KeyError: + self.log.debug("No access_token nor Authorization header in request") return False if token != self.hs_token: - self.log.trace(f"Incorrect hs_token in request ({token!r} != {self.hs_token!r})") + self.log.debug(f"Incorrect hs_token in request") return False return True From cedf2aa89d341f8be7fe3814405c9af80950ac30 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 31 May 2023 14:40:16 +0300 Subject: [PATCH 346/456] Retry sending unconfigured bridge state --- mautrix/bridge/bridge.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mautrix/bridge/bridge.py b/mautrix/bridge/bridge.py index 1dd9c0f9..37e1646d 100644 --- a/mautrix/bridge/bridge.py +++ b/mautrix/bridge/bridge.py @@ -8,6 +8,7 @@ from typing import Any from abc import ABC, abstractmethod from enum import Enum +import asyncio import sys from aiohttp import web @@ -258,7 +259,8 @@ async def start(self) -> None: status_endpoint = self.config["homeserver.status_endpoint"] if status_endpoint and await self.count_logged_in_users() == 0: state = BridgeState(state_event=BridgeStateEvent.UNCONFIGURED).fill() - await state.send(status_endpoint, self.az.as_token, self.log) + while not await state.send(status_endpoint, self.az.as_token, self.log): + await asyncio.sleep(5) async def system_exit(self) -> None: if hasattr(self, "db") and isinstance(self.db, Database): From dd81640904af047692a5f307eef3763f23f48a17 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 10 Jun 2023 16:25:41 +0300 Subject: [PATCH 347/456] Stabilize async uploads (#149) --- CHANGELOG.md | 3 + .../client/api/modules/media_repository.py | 68 +++++++++---------- mautrix/types/media.py | 2 +- 3 files changed, 35 insertions(+), 38 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index da1a8d5a..7b92b469 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,9 @@ for relative paths, and `sqlite:/path.db` instead of `sqlite:////path.db` for absolute paths. * Bridge configs do this migration automatically. +* *(client)* Stabilized support for asynchronous uploads. + * `unstable_create_msc` was renamed to `create_mxc`, and the `max_stall_ms` + parameters for downloading were renamed to `timeout_ms`.. ## v0.19.16 (2023-05-26) diff --git a/mautrix/client/api/modules/media_repository.py b/mautrix/client/api/modules/media_repository.py index edf5e701..0d51d9f6 100644 --- a/mautrix/client/api/modules/media_repository.py +++ b/mautrix/client/api/modules/media_repository.py @@ -46,22 +46,19 @@ class MediaRepositoryMethods(BaseClientAPI): downloading content from the media repository and for getting URL previews without leaking client IPs. - See also: `API reference `__ - - There are also methods for supporting `MSC2246 - `__ which allows asynchronous - uploads of media. + See also: `API reference `__ """ - async def unstable_create_mxc(self) -> MediaCreateResponse: + async def create_mxc(self) -> MediaCreateResponse: """ - Create a media ID for uploading media to the homeserver. Requires the homeserver to have - `MSC2246 `__ support. + Create a media ID for uploading media to the homeserver. + + See also: `API reference `__ Returns: - MediaCreateResponse Containing the MXC URI that can be used to upload a file to later, as well as an optional upload URL + MediaCreateResponse Containing the MXC URI that can be used to upload a file to later """ - resp = await self.api.request(Method.POST, MediaPath.unstable["fi.mau.msc2246"].create) + resp = await self.api.request(Method.POST, MediaPath.v1.create) return MediaCreateResponse.deserialize(resp) @contextmanager @@ -87,21 +84,18 @@ async def upload_media( """ Upload a file to the content repository. - See also: `API reference `__ + See also: `API reference `__ Args: data: The data to upload. mime_type: The MIME type to send with the upload request. filename: The filename to send with the upload request. size: The file size to send with the upload request. - mxc: An existing MXC URI which doesn't have content yet to upload into. Requires the - homeserver to have MSC2246_ support. - async_upload: Should the media be uploaded in the background (using MSC2246_)? - If ``True``, this will create a MXC URI, start uploading in the background and then - immediately return the created URI. This is mutually exclusive with manually - passing the ``mxc`` parameter. - - .. _MSC2246: https://github.com/matrix-org/matrix-spec-proposals/pull/2246 + mxc: An existing MXC URI which doesn't have content yet to upload into. + async_upload: Should the media be uploaded in the background? + If ``True``, this will create a MXC URI using :meth:`create_mxc`, start uploading + in the background, and then immediately return the created URI. This is mutually + exclusive with manually passing the ``mxc`` parameter. Returns: The MXC URI to the uploaded file. @@ -128,19 +122,21 @@ async def upload_media( if async_upload: if mxc: raise ValueError("async_upload and mxc can't be provided simultaneously") - create_response = await self.unstable_create_mxc() + create_response = await self.create_mxc() mxc = create_response.content_uri - upload_url = create_response.upload_url + upload_url = create_response.unstable_upload_url path = MediaPath.v3.upload method = Method.POST if mxc: server_name, media_id = self.api.parse_mxc_uri(mxc) if upload_url is None: - path = MediaPath.unstable["fi.mau.msc2246"].upload[server_name][media_id] + path = MediaPath.v3.upload[server_name][media_id] method = Method.PUT else: - path = MediaPath.unstable["fi.mau.msc2246"].upload[server_name][media_id].complete + path = ( + MediaPath.unstable["com.beeper.msc3870"].upload[server_name][media_id].complete + ) if upload_url is not None: task = self._upload_to_url(upload_url, path, headers, data, post_upload_query=query) @@ -168,25 +164,24 @@ async def _try_upload(): except KeyError: raise MatrixResponseError("`content_uri` not in response.") - async def download_media(self, url: ContentURI, max_stall_ms: int | None = None) -> bytes: + async def download_media(self, url: ContentURI, timeout_ms: int | None = None) -> bytes: """ Download a file from the content repository. - See also: `API reference `__ + See also: `API reference `__ Args: url: The MXC URI to download. - max_stall_ms: The maximum number of milliseconds that the client is willing to wait to - start receiving data. Used for MSC2246 Asynchronous Uploads. + timeout_ms: The maximum number of milliseconds that the client is willing to wait to + start receiving data. Used for asynchronous uploads. Returns: The raw downloaded data. """ url = self.api.get_download_url(url) query_params: dict[str, Any] = {"allow_redirect": "true"} - if max_stall_ms is not None: - query_params["max_stall_ms"] = max_stall_ms - query_params["fi.mau.msc2246.max_stall_ms"] = max_stall_ms + if timeout_ms is not None: + query_params["timeout_ms"] = timeout_ms async with self.api.session.get(url, params=query_params) as response: return await response.read() @@ -197,12 +192,12 @@ async def download_thumbnail( height: int | None = None, resize_method: Literal["crop", "scale"] = None, allow_remote: bool = True, - max_stall_ms: int | None = None, + timeout_ms: int | None = None, ): """ Download a thumbnail for a file in the content repository. - See also: `API reference `__ + See also: `API reference `__ Args: url: The MXC URI to download. @@ -214,8 +209,8 @@ async def download_thumbnail( allow_remote: Indicates to the server that it should not attempt to fetch the media if it is deemed remote. This is to prevent routing loops where the server contacts itself. - max_stall_ms: The maximum number of milliseconds that the client is willing to wait to - start receiving data. Used for MSC2246 Asynchronous Uploads. + timeout_ms: The maximum number of milliseconds that the client is willing to wait to + start receiving data. Used for asynchronous Uploads. Returns: The raw downloaded data. @@ -230,9 +225,8 @@ async def download_thumbnail( query_params["method"] = resize_method if allow_remote is not None: query_params["allow_remote"] = allow_remote - if max_stall_ms is not None: - query_params["max_stall_ms"] = max_stall_ms - query_params["fi.mau.msc2246.max_stall_ms"] = max_stall_ms + if timeout_ms is not None: + query_params["timeout_ms"] = timeout_ms async with self.api.session.get(url, params=query_params) as response: return await response.read() diff --git a/mautrix/types/media.py b/mautrix/types/media.py index 0b277382..1d0ee66a 100644 --- a/mautrix/types/media.py +++ b/mautrix/types/media.py @@ -71,4 +71,4 @@ class MediaCreateResponse(SerializableAttrs): content_uri: ContentURI unused_expired_at: Optional[int] = None - upload_url: Optional[str] = None + unstable_upload_url: Optional[str] = field(default=None, json="com.beeper.msc3870.upload_url") From 7cdebe553b2c9700e1a6c2b34613001d628628c4 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 4 Jun 2023 18:01:12 +0300 Subject: [PATCH 348/456] Log warning if db file doesn't seem writable --- mautrix/util/async_db/aiosqlite.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/mautrix/util/async_db/aiosqlite.py b/mautrix/util/async_db/aiosqlite.py index d9857b8c..99d8fa44 100644 --- a/mautrix/util/async_db/aiosqlite.py +++ b/mautrix/util/async_db/aiosqlite.py @@ -9,6 +9,7 @@ from contextlib import asynccontextmanager import asyncio import logging +import os import re import sqlite3 @@ -148,6 +149,11 @@ async def start(self) -> None: raise RuntimeError("database pool can't be restarted") self.log.debug(f"Connecting to {self.url}") self.log.debug(f"Database connection init commands: {self._init_commands}") + if os.path.exists(self._path): + if not os.access(self._path, os.W_OK): + self.log.warning("Database file doesn't seem writable") + elif not os.access(os.path.dirname(os.path.abspath(self._path)), os.W_OK): + self.log.warning("Database file doesn't exist and directory doesn't seem writable") for _ in range(self._pool.maxsize): conn = await TxnConnection(self._path, **self._db_args) if self._init_commands: From afa904edce4911add55a0d69f54f43d99ee36d83 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 31 May 2023 14:39:57 +0300 Subject: [PATCH 349/456] Fix ensure registered methods --- mautrix/appservice/api/intent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/appservice/api/intent.py b/mautrix/appservice/api/intent.py index e52d8016..ab9e1e82 100644 --- a/mautrix/appservice/api/intent.py +++ b/mautrix/appservice/api/intent.py @@ -72,7 +72,7 @@ def quote(*args, **kwargs): ClientAPI.set_displayname, ClientAPI.set_avatar_url, ClientAPI.beeper_update_profile, - ClientAPI.unstable_create_mxc, + ClientAPI.create_mxc, ClientAPI.upload_media, ClientAPI.send_receipt, ClientAPI.set_fully_read_marker, From 6c58b51461c3134fec04b823f4d0a89469cf18ed Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 13 Jun 2023 13:51:28 +0300 Subject: [PATCH 350/456] Add option to not rotate keys when devices change --- mautrix/bridge/config.py | 1 + mautrix/bridge/e2ee.py | 3 +++ mautrix/crypto/base.py | 1 + mautrix/crypto/device_lists.py | 2 ++ mautrix/crypto/machine.py | 1 + 5 files changed, 8 insertions(+) diff --git a/mautrix/bridge/config.py b/mautrix/bridge/config.py index 2674256b..0dc536e6 100644 --- a/mautrix/bridge/config.py +++ b/mautrix/bridge/config.py @@ -166,6 +166,7 @@ def do_update(self, helper: ConfigUpdateHelper) -> None: copy("bridge.encryption.rotation.enable_custom") copy("bridge.encryption.rotation.milliseconds") copy("bridge.encryption.rotation.messages") + copy("bridge.encryption.rotation.disable_device_change_key_rotation") copy("bridge.relay.enabled") copy_dict("bridge.relay.message_formats", override_existing_map=False) diff --git a/mautrix/bridge/e2ee.py b/mautrix/bridge/e2ee.py index 3961be58..b1a75b2c 100644 --- a/mautrix/bridge/e2ee.py +++ b/mautrix/bridge/e2ee.py @@ -123,6 +123,9 @@ def __init__( self.crypto.delete_fully_used_keys_on_decrypt = del_cfg["delete_fully_used_on_decrypt"] self.crypto.delete_keys_on_device_delete = del_cfg["delete_on_device_delete"] self.periodically_delete_expired_keys = del_cfg["periodically_delete_expired"] + self.crypto.disable_device_change_key_rotation = bridge.config[ + "bridge.encryption.rotation.disable_device_change_key_rotation" + ] async def _exit_on_sync_fail(self, data) -> None: if data["error"]: diff --git a/mautrix/crypto/base.py b/mautrix/crypto/base.py index 5af3ec0a..e1386834 100644 --- a/mautrix/crypto/base.py +++ b/mautrix/crypto/base.py @@ -56,6 +56,7 @@ class BaseOlmMachine: ratchet_keys_on_decrypt: bool delete_fully_used_keys_on_decrypt: bool delete_keys_on_device_delete: bool + disable_device_change_key_rotation: bool # Futures that wait for responses to a key request _key_request_waiters: dict[SessionID, asyncio.Future] diff --git a/mautrix/crypto/device_lists.py b/mautrix/crypto/device_lists.py index d49c7e19..1b5e0dbd 100644 --- a/mautrix/crypto/device_lists.py +++ b/mautrix/crypto/device_lists.py @@ -244,6 +244,8 @@ async def get_or_fetch_device_by_key( return None async def on_devices_changed(self, user_id: UserID) -> None: + if self.disable_device_change_key_rotation: + return shared_rooms = await self.state_store.find_shared_rooms(user_id) self.log.debug( f"Devices of {user_id} changed, invalidating group session in {shared_rooms}" diff --git a/mautrix/crypto/machine.py b/mautrix/crypto/machine.py index 9cf53312..c68335af 100644 --- a/mautrix/crypto/machine.py +++ b/mautrix/crypto/machine.py @@ -83,6 +83,7 @@ def __init__( self.ratchet_keys_on_decrypt = False self.delete_fully_used_keys_on_decrypt = False self.delete_keys_on_device_delete = False + self.disable_device_change_key_rotation = False self._fetch_keys_lock = asyncio.Lock() self._megolm_decrypt_lock = asyncio.Lock() From 69328fa4d7a524a24699c39742235c0d85bf52b0 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 13 Jun 2023 14:01:48 +0300 Subject: [PATCH 351/456] Update changelog --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b92b469..74214d56 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,9 +9,12 @@ for relative paths, and `sqlite:/path.db` instead of `sqlite:////path.db` for absolute paths. * Bridge configs do this migration automatically. +* *(util.async_db)* Added warning log if using SQLite database path that isn't + writable. * *(client)* Stabilized support for asynchronous uploads. * `unstable_create_msc` was renamed to `create_mxc`, and the `max_stall_ms` parameters for downloading were renamed to `timeout_ms`.. +* *(crypto)* Added option to not rotate keys when devices change. ## v0.19.16 (2023-05-26) From a28b7576978f263354f433ef20b8943d8829f81b Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 14 Jun 2023 16:11:33 +0300 Subject: [PATCH 352/456] Simplify SimpleLock check --- mautrix/util/simple_lock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/util/simple_lock.py b/mautrix/util/simple_lock.py index ad979aec..c6bd08ce 100644 --- a/mautrix/util/simple_lock.py +++ b/mautrix/util/simple_lock.py @@ -47,7 +47,7 @@ def locked(self) -> bool: return not self.noop_mode and not self._event.is_set() async def wait(self, task: str | None = None) -> None: - if not self.noop_mode and not self._event.is_set(): + if self.locked: if self.log and self.message: self.log.debug(self.message, task) await self._event.wait() From 80c5a0fdb2768bcb883bd86711cd472c57141d3f Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 14 Jun 2023 16:12:43 +0300 Subject: [PATCH 353/456] Bump version to 0.20.0rc1 --- mautrix/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/__init__.py b/mautrix/__init__.py index ce7df73a..59187608 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.19.15" +__version__ = "0.20.0rc1" __author__ = "Tulir Asokan " __all__ = [ "api", From ad42ba80605ff257ef7d490e1e7bdcfedaa9a320 Mon Sep 17 00:00:00 2001 From: Max Sandholm Date: Thu, 22 Jun 2023 09:47:15 +0300 Subject: [PATCH 354/456] Add option to delete outdated inbound keys (keys which lack the metadata about when they're safe to delete) --- mautrix/bridge/config.py | 1 + mautrix/bridge/e2ee.py | 9 +++++++++ mautrix/crypto/store/abstract.py | 10 ++++++++++ mautrix/crypto/store/asyncpg/store.py | 15 +++++++++++++++ mautrix/crypto/store/memory.py | 3 +++ 5 files changed, 38 insertions(+) diff --git a/mautrix/bridge/config.py b/mautrix/bridge/config.py index 0dc536e6..1705e914 100644 --- a/mautrix/bridge/config.py +++ b/mautrix/bridge/config.py @@ -150,6 +150,7 @@ def do_update(self, helper: ConfigUpdateHelper) -> None: copy("bridge.encryption.delete_keys.delete_prev_on_new_session") copy("bridge.encryption.delete_keys.delete_on_device_delete") copy("bridge.encryption.delete_keys.periodically_delete_expired") + copy("bridge.encryption.delete_keys.delete_outdated_inbound") copy("bridge.encryption.verification_levels.receive") copy("bridge.encryption.verification_levels.send") copy("bridge.encryption.verification_levels.share") diff --git a/mautrix/bridge/e2ee.py b/mautrix/bridge/e2ee.py index b1a75b2c..7ae66abf 100644 --- a/mautrix/bridge/e2ee.py +++ b/mautrix/bridge/e2ee.py @@ -56,6 +56,7 @@ class EncryptionManager: key_sharing_enabled: bool appservice_mode: bool periodically_delete_expired_keys: bool + delete_outdated_inbound: bool bridge: br.Bridge az: AppService @@ -113,6 +114,7 @@ def __init__( self.az.to_device_handler = self.crypto.handle_as_to_device_event self.periodically_delete_expired_keys = False + self.delete_outdated_inbound = False self._key_delete_task = None del_cfg = bridge.config["bridge.encryption.delete_keys"] if del_cfg: @@ -123,6 +125,7 @@ def __init__( self.crypto.delete_fully_used_keys_on_decrypt = del_cfg["delete_fully_used_on_decrypt"] self.crypto.delete_keys_on_device_delete = del_cfg["delete_on_device_delete"] self.periodically_delete_expired_keys = del_cfg["periodically_delete_expired"] + self.delete_outdated_inbound = del_cfg["delete_outdated_inbound"] self.crypto.disable_device_change_key_rotation = bridge.config[ "bridge.encryption.rotation.disable_device_change_key_rotation" ] @@ -279,6 +282,12 @@ async def start(self) -> None: else: _ = self.client.start(self._filter) self.log.info("End-to-bridge encryption support is enabled (sync mode)") + if self.delete_outdated_inbound: + deleted = await self.crypto_store.redact_outdated_group_sessions() + if len(deleted) > 0: + self.log.debug( + f"Deleted {len(deleted)} inbound keys which lacked expiration metadata" + ) if self.periodically_delete_expired_keys: self._key_delete_task = background_task.create(self._periodically_delete_keys()) background_task.create(self._resync_encryption_info()) diff --git a/mautrix/crypto/store/abstract.py b/mautrix/crypto/store/abstract.py index 4db5aa1d..787d5225 100644 --- a/mautrix/crypto/store/abstract.py +++ b/mautrix/crypto/store/abstract.py @@ -237,6 +237,16 @@ async def redact_expired_group_sessions(self) -> list[SessionID]: The list of session IDs that were deleted. """ + @abstractmethod + async def redact_outdated_group_sessions(self) -> list[SessionID]: + """ + Remove all Megolm group sessions which lack the metadata to determine when they should + expire. + + Returns: + The list of session IDs that were deleted. + """ + @abstractmethod async def has_group_session(self, room_id: RoomID, session_id: SessionID) -> bool: """ diff --git a/mautrix/crypto/store/asyncpg/store.py b/mautrix/crypto/store/asyncpg/store.py index 0642d210..a29f7737 100644 --- a/mautrix/crypto/store/asyncpg/store.py +++ b/mautrix/crypto/store/asyncpg/store.py @@ -362,6 +362,21 @@ async def redact_expired_group_sessions(self) -> list[SessionID]: ) return [row["session_id"] for row in rows] + async def redact_outdated_group_sessions(self) -> list[SessionID]: + q = """ + UPDATE crypto_megolm_inbound_session + SET withheld_code=$1, withheld_reason=$2, session=NULL, forwarding_chains=NULL + WHERE account_id=$3 AND session IS NOT NULL AND received_at IS NULL + RETURNING session_id + """ + rows = await self.db.fetch( + q, + RoomKeyWithheldCode.BEEPER_REDACTED.value, + f"Session redacted: outdated", + self.account_id, + ) + return [row["session_id"] for row in rows] + async def has_group_session(self, room_id: RoomID, session_id: SessionID) -> bool: q = """ SELECT COUNT(session) FROM crypto_megolm_inbound_session diff --git a/mautrix/crypto/store/memory.py b/mautrix/crypto/store/memory.py index cfc27125..c26f86bc 100644 --- a/mautrix/crypto/store/memory.py +++ b/mautrix/crypto/store/memory.py @@ -134,6 +134,9 @@ async def redact_group_sessions( async def redact_expired_group_sessions(self) -> list[SessionID]: raise NotImplementedError() + async def redact_outdated_group_sessions(self) -> list[SessionID]: + raise NotImplementedError() + async def has_group_session(self, room_id: RoomID, session_id: SessionID) -> bool: return (room_id, session_id) in self._inbound_sessions From 2f9654c97db1bde2373b8adf2e13e5a9b25fd06e Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 23 Jun 2023 12:18:01 +0300 Subject: [PATCH 355/456] Match mautrix-go behavior for reply fallback removal --- mautrix/types/event/message.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/mautrix/types/event/message.py b/mautrix/types/event/message.py index 021e91f9..faaf0809 100644 --- a/mautrix/types/event/message.py +++ b/mautrix/types/event/message.py @@ -350,14 +350,12 @@ def trim_reply_fallback(self) -> None: setattr(self, "__reply_fallback_trimmed", True) def _trim_reply_fallback_text(self) -> None: - if not self.body.startswith("> ") or "\n" not in self.body: + if not self.body.startswith("> <") or "\n" not in self.body: return lines = self.body.split("\n") while len(lines) > 0 and lines[0].startswith("> "): lines.pop(0) - # Pop extra newline at end of fallback - lines.pop(0) - self.body = "\n".join(lines) + self.body = "\n".join(lines).strip() def _trim_reply_fallback_html(self) -> None: if self.formatted_body and self.format == Format.HTML: From 0634700d07540b8d6928712f45136d4ac3032546 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 23 Jun 2023 12:29:43 +0300 Subject: [PATCH 356/456] Handle m.emote fallbacks in reply fallback trimming safety --- mautrix/types/event/message.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mautrix/types/event/message.py b/mautrix/types/event/message.py index faaf0809..8d5f1bba 100644 --- a/mautrix/types/event/message.py +++ b/mautrix/types/event/message.py @@ -350,7 +350,9 @@ def trim_reply_fallback(self) -> None: setattr(self, "__reply_fallback_trimmed", True) def _trim_reply_fallback_text(self) -> None: - if not self.body.startswith("> <") or "\n" not in self.body: + if ( + not self.body.startswith("> <") and not self.body.startswith("> * <") + ) or "\n" not in self.body: return lines = self.body.split("\n") while len(lines) > 0 and lines[0].startswith("> "): From 654b215a9fbad3b506859e7a00f17d21c7f66253 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 25 Jun 2023 13:31:48 +0300 Subject: [PATCH 357/456] Fix using manual_stop while startup task is running --- mautrix/util/program.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/util/program.py b/mautrix/util/program.py index 80d2298b..f7a7b448 100644 --- a/mautrix/util/program.py +++ b/mautrix/util/program.py @@ -214,6 +214,7 @@ def _run(self) -> None: signal.signal(signal.SIGINT, signal.default_int_handler) signal.signal(signal.SIGTERM, signal.default_int_handler) + self._stop_task = self.loop.create_future() exit_code = 0 try: self.log.debug("Running startup actions...") @@ -224,7 +225,6 @@ def _run(self) -> None: f"Startup actions complete in {round(end_ts - start_ts, 2)} seconds, " "now running forever" ) - self._stop_task = self.loop.create_future() exit_code = self.loop.run_until_complete(self._stop_task) self.log.debug("manual_stop() called, stopping...") except KeyboardInterrupt: From 2ab1cdccf79a49f45189d246421223690d487cf5 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 25 Jun 2023 13:44:48 +0300 Subject: [PATCH 358/456] Bump version to 0.20.0 --- CHANGELOG.md | 9 +++++++-- mautrix/__init__.py | 2 +- setup.py | 2 +- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 74214d56..40771515 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -## v0.20.0 (unreleased) +## v0.20.0 (2023-06-25) * Dropped Python 3.8 support. * **Breaking change *(.state_store)*** Removed legacy SQLAlchemy state store @@ -11,10 +11,15 @@ * Bridge configs do this migration automatically. * *(util.async_db)* Added warning log if using SQLite database path that isn't writable. +* *(util.program)* Fixed `manual_stop` not working if it's called during startup. * *(client)* Stabilized support for asynchronous uploads. * `unstable_create_msc` was renamed to `create_mxc`, and the `max_stall_ms` - parameters for downloading were renamed to `timeout_ms`.. + parameters for downloading were renamed to `timeout_ms`. * *(crypto)* Added option to not rotate keys when devices change. +* *(crypto)* Added option to remove all keys that were received before the + automatic ratcheting was implemented (in v0.19.10). +* *(types)* Improved reply fallback removal to have a smaller chance of false + positives for messages that don't use reply fallbacks. ## v0.19.16 (2023-05-26) diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 59187608..16693197 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.20.0rc1" +__version__ = "0.20.0" __author__ = "Tulir Asokan " __all__ = [ "api", diff --git a/setup.py b/setup.py index 04a2a8c3..966e9f49 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ ], extras_require={ "detect_mimetype": ["python-magic>=0.4.15,<0.5"], - "lint": ["black~=22.1", "isort"], + "lint": ["black~=23.1", "isort"], "test": ["pytest", "pytest-asyncio", *test_dependencies], "encryption": encryption_dependencies, }, From 9b67b17ac4927f63cc9a71dc450b9296c849b3ab Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 1 Jul 2023 13:58:59 +0300 Subject: [PATCH 359/456] Remove --base-config flag from program utility --- CHANGELOG.md | 6 ++++++ mautrix/util/program.py | 23 ++++++----------------- 2 files changed, 12 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 40771515..7ddf6b9c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## unreleased + +* *(util.program)* Removed `--base-config` flag in bridges, as there are no + valid use cases (package data should always work) and it's easy to cause + issues by pointing the flag at the wrong file. + ## v0.20.0 (2023-06-25) * Dropped Python 3.8 support. diff --git a/mautrix/util/program.py b/mautrix/util/program.py index f7a7b448..2fb6b141 100644 --- a/mautrix/util/program.py +++ b/mautrix/util/program.py @@ -119,7 +119,7 @@ def preinit(self) -> None: self.check_config() @property - def _default_base_config(self) -> str: + def base_config_path(self) -> str: return f"pkg://{self.module}/example-config.yaml" def prepare_arg_parser(self) -> None: @@ -133,21 +133,13 @@ def prepare_arg_parser(self) -> None: metavar="", help="the path to your config file", ) - self.parser.add_argument( - "-b", - "--base-config", - type=str, - default=self._default_base_config, - metavar="", - help="the path to the example config (for automatic config updates)", - ) self.parser.add_argument( "-n", "--no-update", action="store_true", help="Don't save updated config to disk" ) def prepare_config(self) -> None: """Pre-init lifecycle method. Extend this if you want to customize config loading.""" - self.config = self.config_class(self.args.config, self.args.base_config) + self.config = self.config_class(self.args.config, self.base_config_path) self.load_and_update_config() def load_and_update_config(self) -> None: @@ -155,13 +147,10 @@ def load_and_update_config(self) -> None: try: self.config.update(save=not self.args.no_update) except BaseMissingError: - if self.args.base_config != self._default_base_config: - print(f"Failed to read base config from {self.args.base_config}") - else: - print( - "Failed to read base config from the default path " - f"({self._default_base_config}). Maybe your installation is corrupted?" - ) + print( + "Failed to read base config from the default path " + f"({self.base_config_path}). Maybe your installation is corrupted?" + ) sys.exit(12) def check_config(self) -> None: From 782f568d7873972e0b5ef7ba698df4243e3a4151 Mon Sep 17 00:00:00 2001 From: Sumner Evans Date: Fri, 28 Jul 2023 17:38:44 -0600 Subject: [PATCH 360/456] bridge/prepare_config: replace last instance of args.base_config with self.base_config_path Signed-off-by: Sumner Evans --- mautrix/bridge/bridge.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/bridge/bridge.py b/mautrix/bridge/bridge.py index 37e1646d..9ce9360e 100644 --- a/mautrix/bridge/bridge.py +++ b/mautrix/bridge/bridge.py @@ -136,7 +136,7 @@ def prepare_config(self) -> None: self.config = self.config_class( self.args.config, self.args.registration, - self.args.base_config, + self.base_config_path, env_prefix=self.module.upper(), ) if self.args.generate_registration: From 37c791b8fb877c283180259c2d16864302d00f96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gabriel=20Rodr=C3=ADguez?= Date: Sun, 30 Jul 2023 15:08:54 -0400 Subject: [PATCH 361/456] fix typo: seconds -> milliseconds --- mautrix/client/api/modules/misc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/client/api/modules/misc.py b/mautrix/client/api/modules/misc.py index 443877c1..8ff9c85a 100644 --- a/mautrix/client/api/modules/misc.py +++ b/mautrix/client/api/modules/misc.py @@ -50,7 +50,7 @@ async def set_typing(self, room_id: RoomID, timeout: int = 0) -> None: Args: room_id: The ID of the room in which the user is typing. - timeout: The length of time in seconds to mark this user as typing. + timeout: The length of time in milliseconds to mark this user as typing. """ if timeout > 0: content = {"typing": True, "timeout": timeout} From 2d598551f7e795840e3c22e66511f05ddbf5be3a Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 4 Aug 2023 21:08:19 +0300 Subject: [PATCH 362/456] Add support for `com.devture.shared_secret_auth` for double puppeting --- mautrix/bridge/custom_puppet.py | 50 +++++++++++++++++---------------- mautrix/types/auth.py | 2 ++ 2 files changed, 28 insertions(+), 24 deletions(-) diff --git a/mautrix/bridge/custom_puppet.py b/mautrix/bridge/custom_puppet.py index e2759513..c09095ab 100644 --- a/mautrix/bridge/custom_puppet.py +++ b/mautrix/bridge/custom_puppet.py @@ -17,8 +17,8 @@ from aiohttp import ClientConnectionError from yarl import URL -from mautrix.api import Path from mautrix.appservice import AppService, IntentAPI +from mautrix.client import ClientAPI from mautrix.errors import ( IntentError, MatrixError, @@ -33,6 +33,7 @@ Filter, FilterID, LoginType, + MatrixUserIdentifier, PresenceState, RoomEventFilter, RoomFilter, @@ -182,31 +183,32 @@ async def _login_with_shared_secret(cls, mxid: UserID) -> str: base_url = cls.az.intent.api.base_url else: raise AutologinError(f"No homeserver URL configured for {server}") - url = base_url / str(Path.v3.login) - headers = {"Content-Type": "application/json"} - login_req = { - "initial_device_display_name": cls.login_device_name, - "device_id": cls.login_device_name, - "identifier": { - "type": "m.id.user", - "user": mxid, - }, - } + client = ClientAPI(base_url=base_url) + login_args = {} if secret == b"appservice": - login_req["type"] = str(LoginType.APPSERVICE) - headers["Authorization"] = f"Bearer {cls.az.as_token}" + login_type = LoginType.APPSERVICE + client.api.token = cls.az.as_token else: - login_req["type"] = str(LoginType.PASSWORD) - login_req["password"] = hmac.new( - secret, mxid.encode("utf-8"), hashlib.sha512 - ).hexdigest() - resp = await cls.az.http_session.post(url, data=json.dumps(login_req), headers=headers) - data = await resp.json() - try: - return data["access_token"] - except KeyError: - error_msg = data.get("error", data.get("errcode", f"HTTP {resp.status}")) - raise AutologinError(f"Didn't get an access token: {error_msg}") from None + flows = await client.get_login_flows() + flow = flows.get_first_of_type(LoginType.DEVTURE_SHARED_SECRET, LoginType.PASSWORD) + if not flow: + raise AutologinError("No supported shared secret auth login flows") + login_type = flow.type + token = hmac.new(secret, mxid.encode("utf-8"), hashlib.sha512).hexdigest() + if login_type == LoginType.DEVTURE_SHARED_SECRET: + login_args["token"] = token + elif login_type == LoginType.PASSWORD: + login_args["password"] = token + resp = await client.login( + identifier=MatrixUserIdentifier(user=mxid), + device_id=cls.login_device_name, + initial_device_display_name=cls.login_device_name, + login_type=login_type, + **login_args, + store_access_token=False, + update_hs_url=False, + ) + return resp.access_token async def switch_mxid( self, access_token: str | None, mxid: UserID | None, start_sync_task: bool = True diff --git a/mautrix/types/auth.py b/mautrix/types/auth.py index 198d2f53..ad582118 100644 --- a/mautrix/types/auth.py +++ b/mautrix/types/auth.py @@ -26,6 +26,8 @@ class LoginType(ExtensibleEnum): UNSTABLE_JWT: "LoginType" = "org.matrix.login.jwt" + DEVTURE_SHARED_SECRET: "LoginType" = "com.devture.shared_secret_auth" + @dataclass class LoginFlow(SerializableAttrs): From c4ffe9be4ecd55624a79a5161a40fb84ee20f072 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 7 Aug 2023 17:23:06 +0300 Subject: [PATCH 363/456] Fix error code in crypto-related MSS events --- mautrix/bridge/matrix.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/bridge/matrix.py b/mautrix/bridge/matrix.py index dc7d9fcf..da42a95e 100644 --- a/mautrix/bridge/matrix.py +++ b/mautrix/bridge/matrix.py @@ -620,7 +620,7 @@ async def _send_crypto_status_error( evt, status=MessageStatus.RETRIABLE if is_final else MessageStatus.PENDING, reason=MessageStatusReason.UNDECRYPTABLE, - error=msg, + error=str(err), message=err.human_message if err else None, ) From e0aad7111253c41669814df3fbfab96a6a31b4df Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 27 Aug 2023 00:23:09 +0300 Subject: [PATCH 364/456] Drop support for syncing with double puppets --- mautrix/bridge/commands/login_matrix.py | 21 ---- mautrix/bridge/custom_puppet.py | 136 ++---------------------- 2 files changed, 8 insertions(+), 149 deletions(-) diff --git a/mautrix/bridge/commands/login_matrix.py b/mautrix/bridge/commands/login_matrix.py index 2fd89d2d..6cc6f7aa 100644 --- a/mautrix/bridge/commands/login_matrix.py +++ b/mautrix/bridge/commands/login_matrix.py @@ -70,24 +70,3 @@ async def ping_matrix(evt: CommandEvent) -> EventID: except InvalidAccessToken: return await evt.reply("Your access token is invalid.") return await evt.reply("Your Matrix login is working.") - - -@command_handler( - needs_auth=True, - help_section=SECTION_AUTH, - help_text="Clear the Matrix sync token stored for your double puppet.", -) -async def clear_cache_matrix(evt: CommandEvent) -> EventID: - try: - puppet = await evt.sender.get_puppet() - except NotImplementedError: - return await evt.reply("This bridge has not implemented the clear-cache-matrix command") - if not puppet.is_real_user: - return await evt.reply("You are not logged in with your Matrix account.") - try: - puppet.stop() - puppet.next_batch = None - await puppet.start() - except InvalidAccessToken: - return await evt.reply("Your access token is invalid.") - return await evt.reply("Cleared cache successfully.") diff --git a/mautrix/bridge/custom_puppet.py b/mautrix/bridge/custom_puppet.py index c09095ab..2605e003 100644 --- a/mautrix/bridge/custom_puppet.py +++ b/mautrix/bridge/custom_puppet.py @@ -116,7 +116,6 @@ class CustomPuppetMixin(ABC): intent: The primary IntentAPI. """ - sync_with_custom_puppets: bool = True allow_discover_url: bool = False homeserver_url_map: dict[str, URL] = {} only_handle_own_synced_events: bool = True @@ -135,12 +134,9 @@ class CustomPuppetMixin(ABC): custom_mxid: UserID | None access_token: str | None base_url: URL | None - next_batch: SyncToken | None intent: IntentAPI - _sync_task: asyncio.Task | None = None - @abstractmethod async def save(self) -> None: """Save the information of this puppet. Called from :meth:`switch_mxid`""" @@ -221,7 +217,6 @@ async def switch_mxid( the appservice-owned ID. mxid: The expected Matrix user ID of the custom account, or ``None`` when ``access_token`` is None. - start_sync_task: Whether or not syncing should be started after logging in. """ if access_token == "auto": access_token = await self._login_with_shared_secret(mxid) @@ -251,7 +246,7 @@ async def switch_mxid( self.base_url = base_url self.intent = self._fresh_intent() - await self.start(start_sync_task=start_sync_task, check_e2ee_keys=True) + await self.start(check_e2ee_keys=True) try: del self.by_custom_mxid[prev_mxid] @@ -276,7 +271,6 @@ async def _invalidate_double_puppet(self) -> None: del self.by_custom_mxid[self.custom_mxid] self.custom_mxid = None self.access_token = None - self.next_batch = None await self.save() self.intent = self._fresh_intent() @@ -296,7 +290,7 @@ async def start( except MatrixInvalidToken as e: if retry_auto_login and self.custom_mxid and self.can_auto_login(self.custom_mxid): self.log.debug(f"Got {e.errcode} while trying to initialize custom mxid") - await self.switch_mxid("auto", self.custom_mxid, start_sync_task=start_sync_task) + await self.switch_mxid("auto", self.custom_mxid) return self.log.warning(f"Got {e.errcode} while trying to initialize custom mxid") whoami = None @@ -319,19 +313,14 @@ async def start( if device_keys and len(device_keys.keys) > 0: await self._invalidate_double_puppet() raise EncryptionKeysFound() - if self.sync_with_custom_puppets and start_sync_task: - if self._sync_task: - self._sync_task.cancel() - self.log.info(f"Initialized custom mxid: {whoami.user_id}. Starting sync task") - self._sync_task = asyncio.create_task(self._try_sync()) - else: - self.log.info(f"Initialized custom mxid: {whoami.user_id}. Not starting sync task") + self.log.info(f"Initialized custom mxid: {whoami.user_id}") def stop(self) -> None: - """Cancel the sync task.""" - if self._sync_task: - self._sync_task.cancel() - self._sync_task = None + """ + No-op + + .. deprecated:: 0.20.1 + """ async def default_puppet_should_leave_room(self, room_id: RoomID) -> bool: """ @@ -354,112 +343,3 @@ async def _leave_rooms_with_default_user(self) -> None: await self.intent.ensure_joined(room_id) except (IntentError, MatrixRequestError): pass - - def _create_sync_filter(self) -> Awaitable[FilterID]: - all_events = EventType.find("*") - return self.intent.create_filter( - Filter( - account_data=EventFilter(types=[all_events]), - presence=EventFilter( - types=[EventType.PRESENCE], - senders=[self.custom_mxid] if self.only_handle_own_synced_events else None, - ), - room=RoomFilter( - include_leave=False, - state=StateFilter(not_types=[all_events]), - timeline=RoomEventFilter(not_types=[all_events]), - account_data=RoomEventFilter(not_types=[all_events]), - ephemeral=RoomEventFilter( - types=[ - EventType.TYPING, - EventType.RECEIPT, - ] - ), - ), - ) - ) - - def _filter_events(self, room_id: RoomID, events: list[dict]) -> Iterator[Event]: - for event in events: - event["room_id"] = room_id - if self.only_handle_own_synced_events: - # We only want events about the custom puppet user, but we can't use - # filters for typing and read receipt events. - evt_type = EventType.find(event.get("type", None)) - event.setdefault("content", {}) - if evt_type == EventType.TYPING: - is_typing = self.custom_mxid in event["content"].get("user_ids", []) - event["content"]["user_ids"] = [self.custom_mxid] if is_typing else [] - elif evt_type == EventType.RECEIPT: - try: - event_id, receipt = event["content"].popitem() - data = receipt["m.read"][self.custom_mxid] - event["content"] = {event_id: {"m.read": {self.custom_mxid: data}}} - except KeyError: - continue - yield event - - def _handle_sync(self, sync_resp: dict) -> None: - # Get events from rooms -> join -> [room_id] -> ephemeral -> events (array) - ephemeral_events = ( - event - for room_id, data in sync_resp.get("rooms", {}).get("join", {}).items() - for event in self._filter_events(room_id, data.get("ephemeral", {}).get("events", [])) - ) - - # Get events from presence -> events (array) - presence_events = sync_resp.get("presence", {}).get("events", []) - - # Deserialize and handle all events - for event in chain(ephemeral_events, presence_events): - background_task.create(self.mx.try_handle_sync_event(Event.deserialize(event))) - - async def _try_sync(self) -> None: - try: - await self._sync() - except asyncio.CancelledError: - self.log.info(f"Syncing for {self.custom_mxid} cancelled") - except Exception: - self.log.critical(f"Fatal error syncing {self.custom_mxid}", exc_info=True) - - async def _sync(self) -> None: - if not self.is_real_user: - self.log.warning("Called sync() for non-custom puppet.") - return - custom_mxid: UserID = self.custom_mxid - access_token_at_start: str = self.access_token - errors: int = 0 - filter_id: FilterID = await self._create_sync_filter() - self.log.debug(f"Starting syncer for {custom_mxid} with sync filter {filter_id}.") - while access_token_at_start == self.access_token: - try: - cur_batch = self.next_batch - sync_resp = await self.intent.sync( - filter_id=filter_id, since=cur_batch, set_presence=PresenceState.OFFLINE - ) - try: - self.next_batch = sync_resp.get("next_batch", None) - except Exception: - self.log.warning("Failed to store next batch", exc_info=True) - errors = 0 - if cur_batch is not None: - self._handle_sync(sync_resp) - except MatrixInvalidToken: - # TODO when not using syncing, we should still check this occasionally and relogin - self.log.warning(f"Access token for {custom_mxid} got invalidated, restarting...") - await self.start(retry_auto_login=True, start_sync_task=False) - if self.is_real_user: - self.log.info("Successfully relogined custom puppet, continuing sync") - filter_id = await self._create_sync_filter() - access_token_at_start = self.access_token - else: - self.log.warning("Something went wrong during relogin") - raise - except (MatrixError, ClientConnectionError, asyncio.TimeoutError) as e: - errors += 1 - wait = min(errors, 11) ** 2 - self.log.warning( - f"Syncer for {custom_mxid} errored: {e}. Waiting for {wait} seconds..." - ) - await asyncio.sleep(wait) - self.log.debug(f"Syncer for custom puppet {custom_mxid} stopped.") From af04ca12389e344fbda10463085ed5df4b62db69 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 27 Aug 2023 00:37:50 +0300 Subject: [PATCH 365/456] Add support for double puppeting with another as_token --- mautrix/appservice/api/appservice.py | 26 ++++++++++++++++++++++---- mautrix/appservice/api/intent.py | 10 +++++++--- mautrix/bridge/custom_puppet.py | 18 +++++++++++++++++- 3 files changed, 46 insertions(+), 8 deletions(-) diff --git a/mautrix/appservice/api/appservice.py b/mautrix/appservice/api/appservice.py index af6ae48d..6654ec20 100644 --- a/mautrix/appservice/api/appservice.py +++ b/mautrix/appservice/api/appservice.py @@ -51,6 +51,7 @@ def __init__( client_session: ClientSession = None, child: bool = False, real_user: bool = False, + real_user_as_token: bool = False, bridge_name: str | None = None, default_retry_count: int = None, loop: asyncio.AbstractEventLoop | None = None, @@ -66,6 +67,7 @@ def __init__( client_session: The aiohttp ClientSession to use. child: Whether or not this is instance is a child of another AppServiceAPI. real_user: Whether or not this is a real (non-appservice-managed) user. + real_user_as_token: Whether this real user is actually using another ``as_token``. bridge_name: The name of the bridge to put in the ``fi.mau.double_puppet_source`` field in outgoing message events sent through real users. """ @@ -85,6 +87,7 @@ def __init__( self._bot_intent = None self.state_store = state_store self.is_real_user = real_user + self.is_real_user_as_token = real_user_as_token self.bridge_name = bridge_name if not child: @@ -113,7 +116,9 @@ def user(self, user: UserID) -> ChildAppServiceAPI: self.children[user] = child return child - def real_user(self, mxid: UserID, token: str, base_url: URL | None = None) -> AppServiceAPI: + def real_user( + self, mxid: UserID, token: str, base_url: URL | None = None, as_token: bool = False + ) -> AppServiceAPI: """ Get the AppServiceAPI for a real (non-appservice-managed) Matrix user. @@ -122,6 +127,8 @@ def real_user(self, mxid: UserID, token: str, base_url: URL | None = None) -> Ap token: The access token for the user. base_url: The base URL of the homeserver client-server API to use. Defaults to the appservice homeserver URL. + as_token: Whether the token is actually an as_token + (meaning the ``user_id`` query parameter needs to be used). Returns: The AppServiceAPI object for the user. @@ -136,6 +143,7 @@ def real_user(self, mxid: UserID, token: str, base_url: URL | None = None) -> Ap child = self.real_users[mxid] child.base_url = base_url or child.base_url child.token = token or child.token + child.is_real_user_as_token = as_token except KeyError: child = type(self)( base_url=base_url or self.base_url, @@ -145,6 +153,7 @@ def real_user(self, mxid: UserID, token: str, base_url: URL | None = None) -> Ap state_store=self.state_store, client_session=self.session, real_user=True, + real_user_as_token=as_token, bridge_name=self.bridge_name, default_retry_count=self.default_retry_count, ) @@ -163,7 +172,11 @@ def bot_intent(self) -> as_api.IntentAPI: return self._bot_intent def intent( - self, user: UserID = None, token: str | None = None, base_url: str | None = None + self, + user: UserID = None, + token: str | None = None, + base_url: str | None = None, + real_user_as_token: bool = False, ) -> as_api.IntentAPI: """ Get the intent API of a child user. @@ -173,6 +186,8 @@ def intent( token: The access token to use. Only applicable for non-appservice-managed users. base_url: The base URL of the homeserver client-server API to use. Only applicable for non-appservice users. Defaults to the appservice homeserver URL. + real_user_as_token: When providing a token, whether it's actually another as_token + (meaning the ``user_id`` query parameter needs to be used). Returns: The IntentAPI object for the given user. @@ -184,7 +199,10 @@ def intent( raise ValueError("Can't get child intent of real user") if token: return as_api.IntentAPI( - user, self.real_user(user, token, base_url), self.bot_intent(), self.state_store + user, + self.real_user(user, token, base_url, as_token=real_user_as_token), + self.bot_intent(), + self.state_store, ) return as_api.IntentAPI(user, self.user(user), self.bot_intent(), self.state_store) @@ -229,7 +247,7 @@ def request( if isinstance(timestamp, datetime): timestamp = int(timestamp.replace(tzinfo=timezone.utc).timestamp() * 1000) query_params["ts"] = timestamp - if not self.is_real_user: + if not self.is_real_user or self.is_real_user_as_token: query_params["user_id"] = self.identity or self.bot_mxid return super().request( diff --git a/mautrix/appservice/api/intent.py b/mautrix/appservice/api/intent.py index ab9e1e82..1892743e 100644 --- a/mautrix/appservice/api/intent.py +++ b/mautrix/appservice/api/intent.py @@ -144,7 +144,11 @@ async def wrapper(*args, __self=self, __method=method, **kwargs): setattr(self, method.__name__, wrapper) def user( - self, user_id: UserID, token: str | None = None, base_url: str | None = None + self, + user_id: UserID, + token: str | None = None, + base_url: str | None = None, + as_token: bool = False, ) -> IntentAPI: """ Get the intent API for a specific user. @@ -162,10 +166,10 @@ def user( The IntentAPI for the given user. """ if not self.bot: - return self.api.intent(user_id, token, base_url) + return self.api.intent(user_id, token, base_url, real_user_as_token=as_token) else: self.log.warning("Called IntentAPI#user() of child intent object.") - return self.bot.api.intent(user_id, token, base_url) + return self.bot.api.intent(user_id, token, base_url, real_user_as_token=as_token) # region User actions diff --git a/mautrix/bridge/custom_puppet.py b/mautrix/bridge/custom_puppet.py index 2605e003..a702ea25 100644 --- a/mautrix/bridge/custom_puppet.py +++ b/mautrix/bridge/custom_puppet.py @@ -152,6 +152,19 @@ def is_real_user(self) -> bool: return bool(self.custom_mxid and self.access_token) def _fresh_intent(self) -> IntentAPI: + if self.access_token == "appservice-config" and self.custom_mxid: + _, server = self.az.intent.parse_user_id(self.custom_mxid) + try: + secret = self.login_shared_secret_map[server] + except KeyError: + raise AutologinError(f"No shared secret configured for {server}") + self.log.debug(f"Using as_token for double puppeting {self.custom_mxid}") + return self.az.intent.user( + self.custom_mxid, + secret.decode("utf-8").removeprefix("as_token:"), + self.base_url, + as_token=True, + ) return ( self.az.intent.user(self.custom_mxid, self.access_token, self.base_url) if self.is_real_user @@ -172,6 +185,8 @@ async def _login_with_shared_secret(cls, mxid: UserID) -> str: secret = cls.login_shared_secret_map[server] except KeyError: raise AutologinError(f"No shared secret configured for {server}") + if secret.startswith(b"as_token:"): + return "appservice-config" try: base_url = cls.homeserver_url_map[server] except KeyError: @@ -220,7 +235,8 @@ async def switch_mxid( """ if access_token == "auto": access_token = await self._login_with_shared_secret(mxid) - self.log.debug(f"Logged in for {mxid} using shared secret") + if access_token != "appservice-config": + self.log.debug(f"Logged in for {mxid} using shared secret") if mxid is not None: _, mxid_domain = self.az.intent.parse_user_id(mxid) From 6e4c86de6865fee68c6a3c83eb9868e7ab2ecdce Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 27 Aug 2023 00:49:32 +0300 Subject: [PATCH 366/456] Remove unused imports --- mautrix/bridge/custom_puppet.py | 22 +--------------------- 1 file changed, 1 insertion(+), 21 deletions(-) diff --git a/mautrix/bridge/custom_puppet.py b/mautrix/bridge/custom_puppet.py index a702ea25..9057877c 100644 --- a/mautrix/bridge/custom_puppet.py +++ b/mautrix/bridge/custom_puppet.py @@ -5,16 +5,12 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import annotations -from typing import Awaitable, Iterator from abc import ABC, abstractmethod -from itertools import chain import asyncio import hashlib import hmac -import json import logging -from aiohttp import ClientConnectionError from yarl import URL from mautrix.appservice import AppService, IntentAPI @@ -26,23 +22,7 @@ MatrixRequestError, WellKnownError, ) -from mautrix.types import ( - Event, - EventFilter, - EventType, - Filter, - FilterID, - LoginType, - MatrixUserIdentifier, - PresenceState, - RoomEventFilter, - RoomFilter, - RoomID, - StateFilter, - SyncToken, - UserID, -) -from mautrix.util import background_task +from mautrix.types import LoginType, MatrixUserIdentifier, RoomID, UserID from .. import bridge as br From a48ef9f39139cc01286efd46fa8c70d3f3ab19fe Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 27 Aug 2023 00:49:36 +0300 Subject: [PATCH 367/456] Update changelog --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ddf6b9c..4adf9eb2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,11 @@ * *(util.program)* Removed `--base-config` flag in bridges, as there are no valid use cases (package data should always work) and it's easy to cause issues by pointing the flag at the wrong file. +* *(bridge)* Added support for the `com.devture.shared_secret_auth` login type + for automatic double puppeting. +* *(bridge)* Dropped support for syncing with double puppets. MSC2409 is now + the only way to receive ephemeral events. +* *(bridge)* Added support for double puppeting with arbitrary `as_token`s. ## v0.20.0 (2023-06-25) From 958e198ef21c14e08cd4335c3afb0bb1e2f315d7 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 29 Aug 2023 20:49:17 +0300 Subject: [PATCH 368/456] Bump version to 0.20.1 --- CHANGELOG.md | 2 +- mautrix/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4adf9eb2..6278822a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -## unreleased +## v0.20.1 (2023-08-29) * *(util.program)* Removed `--base-config` flag in bridges, as there are no valid use cases (package data should always work) and it's easy to cause diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 16693197..19015ee4 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.20.0" +__version__ = "0.20.1" __author__ = "Tulir Asokan " __all__ = [ "api", From cef7b5af66e50bb440712cd94c5979b80d852685 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 6 Sep 2023 21:23:26 +0300 Subject: [PATCH 369/456] Add option to run appservice handlers synchronously --- mautrix/appservice/as_handler.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/mautrix/appservice/as_handler.py b/mautrix/appservice/as_handler.py index c0a41f73..fc44ac66 100644 --- a/mautrix/appservice/as_handler.py +++ b/mautrix/appservice/as_handler.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022 Tulir Asokan +# Copyright (c) 2023 Tulir Asokan # # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this @@ -8,7 +8,6 @@ from typing import Any, Awaitable, Callable from json import JSONDecodeError -import asyncio import json import logging @@ -33,12 +32,12 @@ class AppServiceServerMixin: - loop: asyncio.AbstractEventLoop log: logging.Logger hs_token: str ephemeral_events: bool encryption_events: bool + synchronous_handlers: bool query_user: Callable[[UserID], JSON] query_alias: Callable[[RoomAlias], JSON] @@ -57,6 +56,7 @@ def __init__(self, ephemeral_events: bool = False, encryption_events: bool = Fal self.device_list_handler = None self.ephemeral_events = ephemeral_events self.encryption_events = encryption_events + self.synchronous_handlers = False async def default_query_handler(_): return None @@ -309,7 +309,7 @@ async def handle_transaction( except SerializerError: self.log.exception("Failed to deserialize ephemeral event %s", raw_edu) else: - self.handle_matrix_event(edu, ephemeral=True) + await self.handle_matrix_event(edu, ephemeral=True) for raw_event in events: try: self._fix_prev_content(raw_event) @@ -317,10 +317,10 @@ async def handle_transaction( except SerializerError: self.log.exception("Failed to deserialize event %s", raw_event) else: - self.handle_matrix_event(event) + await self.handle_matrix_event(event) return {} - def handle_matrix_event(self, event: Event, ephemeral: bool = False) -> None: + async def handle_matrix_event(self, event: Event, ephemeral: bool = False) -> None: if ephemeral: event.type = event.type.with_class(EventType.Class.EPHEMERAL) elif getattr(event, "state_key", None) is not None: @@ -334,9 +334,12 @@ async def try_handle(handler_func: HandlerFunc): except Exception: self.log.exception("Exception in Matrix event handler") - for handler in self.event_handlers: - # TODO add option to handle events synchronously - background_task.create(try_handle(handler)) + if self.synchronous_handlers: + for handler in self.event_handlers: + await handler(event) + else: + for handler in self.event_handlers: + background_task.create(try_handle(handler)) def matrix_event_handler(self, func: HandlerFunc) -> HandlerFunc: self.event_handlers.append(func) From cb26030cc16b1c2541c77453366718c5328a46c4 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 6 Sep 2023 21:23:58 +0300 Subject: [PATCH 370/456] Make AppServiceServerMixin usable standalone --- mautrix/appservice/as_handler.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/mautrix/appservice/as_handler.py b/mautrix/appservice/as_handler.py index fc44ac66..33bf4bf7 100644 --- a/mautrix/appservice/as_handler.py +++ b/mautrix/appservice/as_handler.py @@ -48,7 +48,17 @@ class AppServiceServerMixin: otk_handler: Callable[[dict[UserID, dict[DeviceID, DeviceOTKCount]]], Awaitable] | None device_list_handler: Callable[[DeviceLists], Awaitable] | None - def __init__(self, ephemeral_events: bool = False, encryption_events: bool = False) -> None: + def __init__( + self, + ephemeral_events: bool = False, + encryption_events: bool = False, + log: logging.Logger | None = None, + hs_token: str | None = None, + ) -> None: + if log is not None: + self.log = log + if hs_token is not None: + self.hs_token = hs_token self.transactions = set() self.event_handlers = [] self.to_device_handler = None From c6979e877f94a4849872e5c66984947ff643006f Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 6 Sep 2023 21:24:26 +0300 Subject: [PATCH 371/456] Clean up AppServiceServerMixin --- mautrix/appservice/as_handler.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mautrix/appservice/as_handler.py b/mautrix/appservice/as_handler.py index 33bf4bf7..ec7e339f 100644 --- a/mautrix/appservice/as_handler.py +++ b/mautrix/appservice/as_handler.py @@ -86,7 +86,6 @@ def register_routes(self, app: web.Application) -> None: app.router.add_route("GET", "/_matrix/app/v1/rooms/{alias}", self._http_query_alias) app.router.add_route("GET", "/_matrix/app/v1/users/{user_id}", self._http_query_user) app.router.add_route("POST", "/_matrix/app/v1/ping", self._http_ping) - app.router.add_route("POST", "/_matrix/app/unstable/fi.mau.msc2659/ping", self._http_ping) def _check_token(self, request: web.Request) -> bool: try: @@ -301,7 +300,7 @@ async def handle_transaction( else: try: await self.to_device_handler(td) - except: + except Exception: self.log.exception("Exception in Matrix to-device event handler") if device_lists and self.device_list_handler: try: From e1b28d39ce7d2e0e1ada8afea3d6a07415fdc24e Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 6 Sep 2023 21:24:47 +0300 Subject: [PATCH 372/456] Allow using OlmMachine.share_keys without OTK count --- mautrix/crypto/base.py | 2 ++ mautrix/crypto/machine.py | 26 +++++++++++++++++++++++--- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/mautrix/crypto/base.py b/mautrix/crypto/base.py index e1386834..1f8e9a62 100644 --- a/mautrix/crypto/base.py +++ b/mautrix/crypto/base.py @@ -66,6 +66,8 @@ class BaseOlmMachine: _prev_unwedge: dict[IdentityKey, float] _fetch_keys_lock: asyncio.Lock _megolm_decrypt_lock: asyncio.Lock + _share_keys_lock: asyncio.Lock + _last_key_share: float _cs_fetch_attempted: set[UserID] async def wait_for_session( diff --git a/mautrix/crypto/machine.py b/mautrix/crypto/machine.py index c68335af..0cfe6ea3 100644 --- a/mautrix/crypto/machine.py +++ b/mautrix/crypto/machine.py @@ -8,6 +8,7 @@ from typing import Optional import asyncio import logging +import time from mautrix import client as cli from mautrix.errors import GroupSessionWithheldError @@ -18,6 +19,7 @@ DeviceLists, DeviceOTKCount, EncryptionAlgorithm, + EncryptionKeyAlgorithm, EventType, Member, Membership, @@ -87,6 +89,8 @@ def __init__( self._fetch_keys_lock = asyncio.Lock() self._megolm_decrypt_lock = asyncio.Lock() + self._share_keys_lock = asyncio.Lock() + self._last_key_share = time.monotonic() - 60 self._key_request_waiters = {} self._inbound_session_waiters = {} self._prev_unwedge = {} @@ -267,14 +271,29 @@ async def handle_beep_room_key_ack(self, evt: ToDeviceEvent) -> None: else: self.log.debug(f"Received room key ack for {sess.id}") - async def share_keys(self, current_otk_count: int) -> None: + async def share_keys(self, current_otk_count: int | None = None) -> None: """ Share any keys that need to be shared. This is automatically called from :meth:`handle_otk_count`, so you should not need to call this yourself. Args: current_otk_count: The current number of signed curve25519 keys present on the server. + If omitted, the count will be fetched from the server. """ + async with self._share_keys_lock: + await self._share_keys(current_otk_count) + + async def _share_keys(self, current_otk_count: int | None) -> None: + if current_otk_count is None or ( + # If the last key share was recent and the new count is very low, re-check the count + # from the server to avoid any race conditions. + self._last_key_share + 60 > time.monotonic() + and current_otk_count < 10 + ): + self.log.debug("Checking OTK count on server") + current_otk_count = (await self.client.upload_keys()).get( + EncryptionKeyAlgorithm.SIGNED_CURVE25519 + ) device_keys = ( self.account.get_device_keys(self.client.mxid, self.client.device_id) if not self.account.shared @@ -289,7 +308,8 @@ async def share_keys(self, current_otk_count: int) -> None: if device_keys: self.log.debug("Going to upload initial account keys") self.log.debug(f"Uploading {len(one_time_keys)} one-time keys") - await self.client.upload_keys(one_time_keys=one_time_keys, device_keys=device_keys) + resp = await self.client.upload_keys(one_time_keys=one_time_keys, device_keys=device_keys) self.account.shared = True + self._last_key_share = time.monotonic() await self.crypto_store.put_account(self.account) - self.log.debug("Shared keys and saved account") + self.log.debug(f"Shared keys and saved account, new keys: {resp}") From e9f8bf8189f04755a586bf929fb00126af3296f7 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 6 Sep 2023 21:24:56 +0300 Subject: [PATCH 373/456] Add missing item in errors.__all__ --- mautrix/errors/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mautrix/errors/__init__.py b/mautrix/errors/__init__.py index d3646a37..afe68dc9 100644 --- a/mautrix/errors/__init__.py +++ b/mautrix/errors/__init__.py @@ -73,6 +73,7 @@ "DeviceValidationError", "DuplicateMessageIndex", "EncryptionError", + "GroupSessionWithheldError", "MatchingSessionDecryptionError", "MismatchingRoomError", "SessionNotFound", From 980d8e4350f7efe7519a50db8d6a80b3393ce70b Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 6 Sep 2023 23:08:02 +0300 Subject: [PATCH 374/456] Use proper background task for async event handlers --- mautrix/client/syncer.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mautrix/client/syncer.py b/mautrix/client/syncer.py index 62ae97a1..8e115bbb 100644 --- a/mautrix/client/syncer.py +++ b/mautrix/client/syncer.py @@ -34,6 +34,7 @@ ToDeviceEvent, UserID, ) +from mautrix.util import background_task from mautrix.util.logging import TraceLogger from . import dispatcher @@ -248,9 +249,10 @@ def dispatch_manual_event( handlers = self.global_event_handlers + handlers tasks = [] for handler, wait_sync in handlers: - task = asyncio.create_task(self._catch_errors(handler, data)) if force_synchronous or wait_sync: - tasks.append(task) + tasks.append(asyncio.create_task(self._catch_errors(handler, data))) + else: + background_task.create(self._catch_errors(handler, data)) return tasks async def run_internal_event( From 92e6091760c7e49498de4134be16998f11b4db82 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 6 Sep 2023 23:08:09 +0300 Subject: [PATCH 375/456] Update changelog --- CHANGELOG.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6278822a..d761b89c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,12 @@ +## v0.20.2 (unreleased) + +* *(crypto)* Changed `OlmMachine.share_keys` to make the OTK count parameter + optional. When omitted, the count is fetched from the server. +* *(appservice)* Added option to run appservice transaction event handlers + synchronously. +* *(appservice)* Added `log` and `hs_token` parameters to `AppServiceServerMixin` + to allow using it as a standalone class without extending. + ## v0.20.1 (2023-08-29) * *(util.program)* Removed `--base-config` flag in bridges, as there are no From 8ff8d07128325dc2894b9be47d8fdce548bef3fb Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 6 Sep 2023 23:18:38 +0300 Subject: [PATCH 376/456] Add appservice user/device masquerading support to base HTTPAPI --- mautrix/api.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/mautrix/api.py b/mautrix/api.py index f6c9f475..d034149b 100644 --- a/mautrix/api.py +++ b/mautrix/api.py @@ -5,7 +5,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import annotations -from typing import AsyncGenerator, ClassVar, Literal, Mapping, Union +from typing import ClassVar, Literal, Mapping from enum import Enum from json.decoder import JSONDecodeError from urllib.parse import quote as urllib_quote, urljoin as urllib_join @@ -28,7 +28,7 @@ if __optional_imports__: # Safe to import, but it's not actually needed, so don't force-import the whole types module. - from mautrix.types import JSON + from mautrix.types import JSON, DeviceID, UserID API_CALLS = Counter( name="bridge_matrix_api_calls", @@ -193,6 +193,13 @@ class HTTPAPI: default_retry_count: int """The default retry count to use if a custom value is not passed to :meth:`request`""" + as_user_id: UserID | None + """An optional user ID to set as the user_id query parameter for appservice requests.""" + as_device_id: DeviceID | None + """ + An optional device ID to set as the user_id query parameter for appservice requests (MSC3202). + """ + def __init__( self, base_url: URL | str, @@ -203,6 +210,8 @@ def __init__( txn_id: int = 0, log: TraceLogger | None = None, loop: asyncio.AbstractEventLoop | None = None, + as_user_id: UserID | None = None, + as_device_id: UserID | None = None, ) -> None: """ Args: @@ -212,6 +221,10 @@ def __init__( txn_id: The outgoing transaction ID to start with. log: The :class:`logging.Logger` instance to log requests with. default_retry_count: Default number of retries to do when encountering network errors. + as_user_id: An optional user ID to set as the user_id query parameter for + appservice requests. + as_device_id: An optional device ID to set as the user_id query parameter for + appservice requests (MSC3202). """ self.base_url = URL(base_url) self.token = token @@ -219,6 +232,8 @@ def __init__( self.session = client_session or ClientSession( loop=loop, headers={"User-Agent": self.default_ua} ) + self.as_user_id = as_user_id + self.as_device_id = as_device_id if txn_id is not None: self.txn_id = txn_id if default_retry_count is not None: @@ -360,6 +375,11 @@ async def request( query_params = query_params or {} if isinstance(query_params, dict): query_params = {k: v for k, v in query_params.items() if v is not None} + if self.as_user_id: + query_params["user_id"] = self.as_user_id + if self.as_device_id: + query_params["org.matrix.msc3202.device_id"] = self.as_device_id + query_params["device_id"] = self.as_device_id if method != Method.GET: content = content or {} From 3471a8feb8546bc144d588cbf0a63a9ada5c0191 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 9 Sep 2023 15:40:01 +0300 Subject: [PATCH 377/456] Bump version to 0.20.2 --- CHANGELOG.md | 4 +++- mautrix/__init__.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d761b89c..9599f6cc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -## v0.20.2 (unreleased) +## v0.20.2 (2023-09-09) * *(crypto)* Changed `OlmMachine.share_keys` to make the OTK count parameter optional. When omitted, the count is fetched from the server. @@ -6,6 +6,8 @@ synchronously. * *(appservice)* Added `log` and `hs_token` parameters to `AppServiceServerMixin` to allow using it as a standalone class without extending. +* *(api)* Added support for setting appservice `user_id` and `device_id` query + parameters manually without using `AppServiceAPI`. ## v0.20.1 (2023-08-29) diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 19015ee4..d52cd2af 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.20.1" +__version__ = "0.20.2" __author__ = "Tulir Asokan " __all__ = [ "api", From 193acb7660e7650487906ec95fb1fb1ed44218da Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 1 Nov 2023 01:02:50 +0200 Subject: [PATCH 378/456] Add wrapper for beeper batch send endpoint --- mautrix/appservice/api/intent.py | 55 ++++++++++++++++++++++++++++++-- mautrix/types/__init__.py | 1 + mautrix/types/misc.py | 5 +++ 3 files changed, 59 insertions(+), 2 deletions(-) diff --git a/mautrix/appservice/api/intent.py b/mautrix/appservice/api/intent.py index 1892743e..00990671 100644 --- a/mautrix/appservice/api/intent.py +++ b/mautrix/appservice/api/intent.py @@ -25,6 +25,7 @@ BatchSendEvent, BatchSendResponse, BatchSendStateEvent, + BeeperBatchSendResponse, ContentURI, EventContent, EventID, @@ -40,7 +41,6 @@ RoomNameStateEventContent, RoomPinnedEventsStateEventContent, RoomTopicStateEventContent, - SerializableAttrs, StateEventContent, UserID, ) @@ -161,6 +161,8 @@ def user( user_id: The Matrix ID of the user whose intent API to get. token: The access token to use for the Matrix ID. base_url: An optional URL to use for API requests. + as_token: Whether the provided token is actually another as_token + (meaning the ``user_id`` query parameter needs to be used). Returns: The IntentAPI for the given user. @@ -187,7 +189,7 @@ async def set_presence( Args: presence: The online status of the user. status: The status message. - ignore_cache: Whether or not to set presence even if the cache says the presence is + ignore_cache: Whether to set presence even if the cache says the presence is already set to that value. """ await self.ensure_registered() @@ -520,6 +522,9 @@ async def batch_send( .. versionadded:: v0.12.5 + .. deprecated:: v0.20.3 + MSC2716 was abandoned by upstream and Beeper has forked the endpoint. + Args: room_id: The room ID to send the events to. prev_event_id: The anchor event. The batch will be inserted immediately after this event. @@ -554,6 +559,52 @@ async def batch_send( ) return BatchSendResponse.deserialize(resp) + async def beeper_batch_send( + self, + room_id: RoomID, + events: Iterable[BatchSendEvent], + *, + forward: bool = False, + forward_if_no_messages: bool = False, + send_notification: bool = False, + mark_read_by: UserID | None = None, + ) -> BeeperBatchSendResponse: + """ + Send a batch of events into a room. Only for Beeper/hungryserv. + + .. versionadded:: v0.20.3 + + Args: + room_id: The room ID to send the events to. + events: The events to send. + forward: Send events to the end of the room instead of the beginning + forward_if_no_messages: Send events to the end of the room, but only if there are no + messages in the room. If there are messages, send the new messages to the beginning. + send_notification: Send a push notification for the new messages. + Only applies when sending to the end of the room. + mark_read_by: Send a read receipt from the given user ID atomically. + + Returns: + All the event IDs generated. + """ + body = { + "events": [evt.serialize() for evt in events], + } + if forward: + body["forward"] = forward + elif forward_if_no_messages: + body["forward_if_no_messages"] = forward_if_no_messages + if send_notification: + body["send_notification"] = send_notification + if mark_read_by: + body["mark_read_by"] = mark_read_by + resp = await self.api.request( + Method.POST, + Path.unstable["com.beeper.backfill"].rooms[room_id].batch_send, + content=body, + ) + return BeeperBatchSendResponse.deserialize(resp) + async def beeper_delete_room(self, room_id: RoomID) -> None: versions = await self.versions() if not versions.supports("com.beeper.room_yeeting"): diff --git a/mautrix/types/__init__.py b/mautrix/types/__init__.py index 6d2ffa7c..42b9068c 100644 --- a/mautrix/types/__init__.py +++ b/mautrix/types/__init__.py @@ -149,6 +149,7 @@ ) from .misc import ( BatchSendResponse, + BeeperBatchSendResponse, DeviceLists, DeviceOTKCount, DirectoryPaginationToken, diff --git a/mautrix/types/misc.py b/mautrix/types/misc.py index 7a978bd1..699fc67e 100644 --- a/mautrix/types/misc.py +++ b/mautrix/types/misc.py @@ -129,3 +129,8 @@ class BatchSendResponse(SerializableAttrs): batch_event_id: EventID next_batch_id: BatchID base_insertion_event_id: Optional[EventID] = None + + +@dataclass +class BeeperBatchSendResponse(SerializableAttrs): + event_ids: List[EventID] From 1d8c2f44588e7c5b6e79a27b7df9061e86e5ca68 Mon Sep 17 00:00:00 2001 From: Nick Mills-Barrett Date: Thu, 31 Aug 2023 09:59:38 +0100 Subject: [PATCH 379/456] Default synchronous = NORMAL for sqlite --- mautrix/util/async_db/aiosqlite.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/mautrix/util/async_db/aiosqlite.py b/mautrix/util/async_db/aiosqlite.py index 99d8fa44..18594c9c 100644 --- a/mautrix/util/async_db/aiosqlite.py +++ b/mautrix/util/async_db/aiosqlite.py @@ -117,6 +117,7 @@ def __init__( def _add_missing_pragmas(init_commands: list[str]) -> list[str]: has_foreign_keys = False has_journal_mode = False + has_synchronous = False has_busy_timeout = False for cmd in init_commands: if "PRAGMA" not in cmd: @@ -125,12 +126,16 @@ def _add_missing_pragmas(init_commands: list[str]) -> list[str]: has_foreign_keys = True elif "journal_mode" in cmd: has_journal_mode = True + elif "synchronous" in cmd: + has_synchronous = True elif "busy_timeout" in cmd: has_busy_timeout = True if not has_foreign_keys: init_commands.append("PRAGMA foreign_keys = ON") if not has_journal_mode: init_commands.append("PRAGMA journal_mode = WAL") + if not has_synchronous: + init_commands.append("PRAGMA synchronous = NORMAL") if not has_busy_timeout: init_commands.append("PRAGMA busy_timeout = 5000") return init_commands From 4c638f0b21f12a0be357293fc25d8bbd1ecb5fb3 Mon Sep 17 00:00:00 2001 From: Nick Mills-Barrett Date: Fri, 1 Sep 2023 18:53:48 +0100 Subject: [PATCH 380/456] Add safety check before applying default synchronous Co-authored-by: Tulir Asokan --- mautrix/util/async_db/aiosqlite.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/util/async_db/aiosqlite.py b/mautrix/util/async_db/aiosqlite.py index 18594c9c..80fa3cf8 100644 --- a/mautrix/util/async_db/aiosqlite.py +++ b/mautrix/util/async_db/aiosqlite.py @@ -134,7 +134,7 @@ def _add_missing_pragmas(init_commands: list[str]) -> list[str]: init_commands.append("PRAGMA foreign_keys = ON") if not has_journal_mode: init_commands.append("PRAGMA journal_mode = WAL") - if not has_synchronous: + if not has_synchronous and "PRAGMA journal_mode = WAL" in init_commands: init_commands.append("PRAGMA synchronous = NORMAL") if not has_busy_timeout: init_commands.append("PRAGMA busy_timeout = 5000") From 1e135d1a874be325cf2a271ba2c94b0055db597e Mon Sep 17 00:00:00 2001 From: Ashish Kumar Date: Wed, 8 Nov 2023 19:23:43 +0400 Subject: [PATCH 381/456] Fix `guest_can_join` field name in room directory response (#163) --- mautrix/types/misc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/types/misc.py b/mautrix/types/misc.py index 699fc67e..cf576c2b 100644 --- a/mautrix/types/misc.py +++ b/mautrix/types/misc.py @@ -87,7 +87,7 @@ class PublicRoomInfo(SerializableAttrs): num_joined_members: int world_readable: bool - guests_can_join: bool + guest_can_join: bool name: str = None topic: str = None From c29c3712ed92e69ef7d468a2baed3183f4c44196 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 10 Nov 2023 22:04:21 +0200 Subject: [PATCH 382/456] Bump version to 0.20.3 --- CHANGELOG.md | 11 +++++++++++ mautrix/__init__.py | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9599f6cc..75b535e0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,14 @@ +## v0.20.3 (2023-11-10) + +* *(client)* Deprecated MSC2716 methods and added new Beeper-specific batch + send methods, as upstream MSC2716 support has been abandoned. +* *(util.async_db)* Added `PRAGMA synchronous = NORMAL;` to default pragmas. +* *(types)* Fixed `guest_can_join` field name in room directory response + (thanks to [@ashfame] in [#163]). + +[@ashfame]: https://github.com/ashfame +[#163]: https://github.com/mautrix/python/pull/163 + ## v0.20.2 (2023-09-09) * *(crypto)* Changed `OlmMachine.share_keys` to make the OTK count parameter diff --git a/mautrix/__init__.py b/mautrix/__init__.py index d52cd2af..c5e20954 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.20.2" +__version__ = "0.20.3" __author__ = "Tulir Asokan " __all__ = [ "api", From 8d39c733d7ad2d1a11b70c255babfad6941faf3a Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 26 Nov 2023 20:12:39 +0200 Subject: [PATCH 383/456] Mark Python 3.12 as supported --- .github/workflows/python-package.yml | 6 +++--- .pre-commit-config.yaml | 4 ++-- README.rst | 2 +- setup.py | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 8c6c7606..cde25b45 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -9,7 +9,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.9", "3.10", "3.11"] + python-version: ["3.9", "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v3 @@ -50,14 +50,14 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-python@v3 with: - python-version: "3.11" + python-version: "3.12" - uses: isort/isort-action@master with: sortPaths: "./mautrix" - uses: psf/black@stable with: src: "./mautrix" - version: "23.1.0" + version: "23.11.0" - name: pre-commit run: | pip install pre-commit diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 77beb7ac..7bed3e1e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v4.5.0 hooks: - id: trailing-whitespace exclude_types: [markdown] @@ -8,7 +8,7 @@ repos: - id: check-yaml - id: check-added-large-files - repo: https://github.com/psf/black - rev: 23.1.0 + rev: 23.11.0 hooks: - id: black language_version: python3 diff --git a/README.rst b/README.rst index 75ce8f6a..e03596d9 100644 --- a/README.rst +++ b/README.rst @@ -3,7 +3,7 @@ mautrix-python |PyPI| |Python versions| |License| |Docs| |Code style| |Imports| -A Python 3.8+ asyncio Matrix framework. +A Python 3.10+ asyncio Matrix framework. Matrix room: `#maunium:maunium.net`_ diff --git a/setup.py b/setup.py index 966e9f49..e81d6c73 100644 --- a/setup.py +++ b/setup.py @@ -42,9 +42,9 @@ "Framework :: AsyncIO", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", ], package_data={ From fc5604a8187db0221a893a3593109721371b1bb0 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 8 Jan 2024 14:08:32 +0200 Subject: [PATCH 384/456] Log media requests and raise on bad status --- CHANGELOG.md | 5 +++++ mautrix/api.py | 14 +++++++++++-- .../client/api/modules/media_repository.py | 20 +++++++++++++++++-- 3 files changed, 35 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 75b535e0..e4a28618 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +## v0.20.4 (unreleased) + +* *(client)* Changed media download methods to log requests and to raise + exceptions on non-successful status codes. + ## v0.20.3 (2023-11-10) * *(client)* Deprecated MSC2716 methods and added new Beeper-specific batch diff --git a/mautrix/api.py b/mautrix/api.py index d034149b..39871bb5 100644 --- a/mautrix/api.py +++ b/mautrix/api.py @@ -275,7 +275,7 @@ def _log_request( self, method: Method, url: URL, - content: str | bytes | bytearray | AsyncBody, + content: str | bytes | bytearray | AsyncBody | None, orig_content, query_params: dict[str, str], headers: dict[str, str], @@ -314,7 +314,7 @@ def _log_request( ) def _log_request_done( - self, path: PathBuilder, req_id: int, duration: float, status: int + self, path: PathBuilder | str, req_id: int, duration: float, status: int ) -> None: level = 5 if path == Path.v3.sync else 10 duration_str = f"{duration * 1000:.1f}ms" if duration < 1 else f"{duration:.3f}s" @@ -334,6 +334,16 @@ def _full_path(self, path: PathBuilder | str) -> str: base_path += "/" return urllib_join(base_path, path) + def log_download_request(self, url: URL, query_params: dict[str, str]) -> int: + req_id = _next_global_req_id() + self._log_request(Method.GET, url, None, None, query_params, {}, req_id, False) + return req_id + + def log_download_request_done( + self, url: URL, req_id: int, duration: float, status: int + ) -> None: + self._log_request_done(url.path.removeprefix("/_matrix/media/"), req_id, duration, status) + async def request( self, method: Method, diff --git a/mautrix/client/api/modules/media_repository.py b/mautrix/client/api/modules/media_repository.py index 0d51d9f6..fe52252a 100644 --- a/mautrix/client/api/modules/media_repository.py +++ b/mautrix/client/api/modules/media_repository.py @@ -182,8 +182,16 @@ async def download_media(self, url: ContentURI, timeout_ms: int | None = None) - query_params: dict[str, Any] = {"allow_redirect": "true"} if timeout_ms is not None: query_params["timeout_ms"] = timeout_ms + req_id = self.api.log_download_request(url, query_params) + start = time.monotonic() async with self.api.session.get(url, params=query_params) as response: - return await response.read() + try: + response.raise_for_status() + return await response.read() + finally: + self.api.log_download_request_done( + url, req_id, time.monotonic() - start, response.status + ) async def download_thumbnail( self, @@ -227,8 +235,16 @@ async def download_thumbnail( query_params["allow_remote"] = allow_remote if timeout_ms is not None: query_params["timeout_ms"] = timeout_ms + req_id = self.api.log_download_request(url, query_params) + start = time.monotonic() async with self.api.session.get(url, params=query_params) as response: - return await response.read() + try: + response.raise_for_status() + return await response.read() + finally: + self.api.log_download_request_done( + url, req_id, time.monotonic() - start, response.status + ) async def get_url_preview(self, url: str, timestamp: int | None = None) -> MXOpenGraph: """ From 7a43c4203de10c1cfeb7206be69dd62108b3955d Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 8 Jan 2024 14:41:46 +0200 Subject: [PATCH 385/456] Drop Python 3.9 support --- .github/workflows/python-package.yml | 11 +++++------ CHANGELOG.md | 1 + setup.py | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index cde25b45..41ebfd8f 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -4,17 +4,16 @@ on: [push, pull_request] jobs: build: - runs-on: ubuntu-latest strategy: fail-fast: false matrix: - python-version: ["3.9", "3.10", "3.11", "3.12"] + python-version: ["3.10", "3.11", "3.12"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install libolm @@ -47,8 +46,8 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v3 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: python-version: "3.12" - uses: isort/isort-action@master diff --git a/CHANGELOG.md b/CHANGELOG.md index e4a28618..295e06aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ ## v0.20.4 (unreleased) +* Dropped Python 3.9 support. * *(client)* Changed media download methods to log requests and to raise exceptions on non-successful status codes. diff --git a/setup.py b/setup.py index e81d6c73..51e99270 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,7 @@ "encryption": encryption_dependencies, }, tests_require=test_dependencies, - python_requires="~=3.9", + python_requires="~=3.10", classifiers=[ "Development Status :: 4 - Beta", From ad6c04e48da3a1b2e065d5a9989c0fa15a7234db Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 8 Jan 2024 14:46:24 +0200 Subject: [PATCH 386/456] Install ruamel.yaml for tests --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 51e99270..b56b987f 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ from mautrix import __version__ encryption_dependencies = ["python-olm", "unpaddedbase64", "pycryptodome"] -test_dependencies = ["aiosqlite", "asyncpg", *encryption_dependencies] +test_dependencies = ["ruamel.yaml", "aiosqlite", "asyncpg", *encryption_dependencies] setuptools.setup( name="mautrix", From 450c31564ec73a27b80ae2dc29beb0d0d87578da Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 8 Jan 2024 14:48:24 +0200 Subject: [PATCH 387/456] Also install commonmark --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b56b987f..dcef751b 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,8 @@ from mautrix import __version__ encryption_dependencies = ["python-olm", "unpaddedbase64", "pycryptodome"] -test_dependencies = ["ruamel.yaml", "aiosqlite", "asyncpg", *encryption_dependencies] +bridge_dependencies = ["ruamel.yaml", "commonmark"] +test_dependencies = ["aiosqlite", "asyncpg", *encryption_dependencies, *bridge_dependencies] setuptools.setup( name="mautrix", From 68b0d71b19cb8989ef11ff482610d72ea2ae5a7d Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 8 Jan 2024 14:51:57 +0200 Subject: [PATCH 388/456] Add ignore paths for pytest --- pyproject.toml | 1 + setup.py | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c0e41313..95426e1e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,3 +11,4 @@ target-version = ["py38"] [tool.pytest.ini_options] asyncio_mode = "auto" +addopts = "--ignore mautrix/util/db/ --ignore mautrix/bridge/" diff --git a/setup.py b/setup.py index dcef751b..51e99270 100644 --- a/setup.py +++ b/setup.py @@ -3,8 +3,7 @@ from mautrix import __version__ encryption_dependencies = ["python-olm", "unpaddedbase64", "pycryptodome"] -bridge_dependencies = ["ruamel.yaml", "commonmark"] -test_dependencies = ["aiosqlite", "asyncpg", *encryption_dependencies, *bridge_dependencies] +test_dependencies = ["aiosqlite", "asyncpg", *encryption_dependencies] setuptools.setup( name="mautrix", From f7b018c501e132d69ec2b3bbe3192efc0f56688f Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 8 Jan 2024 14:54:03 +0200 Subject: [PATCH 389/456] Add ruamel.yaml for tests again --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 51e99270..3f52625b 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ from mautrix import __version__ encryption_dependencies = ["python-olm", "unpaddedbase64", "pycryptodome"] -test_dependencies = ["aiosqlite", "asyncpg", *encryption_dependencies] +test_dependencies = ["aiosqlite", "asyncpg", "ruamel.yaml", *encryption_dependencies] setuptools.setup( name="mautrix", From 61bb33ffb17cf13ae134637a700befe2416a2d73 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 9 Jan 2024 14:14:58 +0200 Subject: [PATCH 390/456] Bump version to 0.20.4 --- CHANGELOG.md | 2 +- mautrix/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 295e06aa..aad156fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -## v0.20.4 (unreleased) +## v0.20.4 (2024-01-09) * Dropped Python 3.9 support. * *(client)* Changed media download methods to log requests and to raise diff --git a/mautrix/__init__.py b/mautrix/__init__.py index c5e20954..5d4ce561 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.20.3" +__version__ = "0.20.4" __author__ = "Tulir Asokan " __all__ = [ "api", From 889996a4e543272ec7426e666db58ff5ca0931de Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 29 Jan 2024 18:48:39 +0200 Subject: [PATCH 391/456] Update Black to 2024 style and Python 3.10 target --- .github/workflows/python-package.yml | 2 +- .pre-commit-config.yaml | 4 ++-- dev-requirements.txt | 2 +- mautrix/bridge/config.py | 20 ++++++++++++-------- mautrix/client/api/filtering.py | 8 +++++--- mautrix/client/api/rooms.py | 9 ++++++--- mautrix/crypto/encrypt_megolm.py | 6 +++--- mautrix/types/event/type.pyi | 1 + mautrix/util/bridge_state.py | 7 ++++--- mautrix/util/message_send_checkpoint.py | 15 +++++++++------ mautrix/util/utf16_surrogate.py | 8 +++++--- pyproject.toml | 2 +- 12 files changed, 50 insertions(+), 34 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 41ebfd8f..5a71c970 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -56,7 +56,7 @@ jobs: - uses: psf/black@stable with: src: "./mautrix" - version: "23.11.0" + version: "24.1.1" - name: pre-commit run: | pip install pre-commit diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7bed3e1e..a056ffdb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,13 +8,13 @@ repos: - id: check-yaml - id: check-added-large-files - repo: https://github.com/psf/black - rev: 23.11.0 + rev: 24.1.1 hooks: - id: black language_version: python3 files: ^mautrix/.*\.pyi?$ - repo: https://github.com/PyCQA/isort - rev: 5.12.0 + rev: 5.13.2 hooks: - id: isort files: ^mautrix/.*\.pyi?$ diff --git a/dev-requirements.txt b/dev-requirements.txt index 5cd14c23..bb8c2a0a 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,3 +1,3 @@ pre-commit>=2.10.1,<3 isort>=5.10.1,<6 -black>=23,<24 +black>=24,<25 diff --git a/mautrix/bridge/config.py b/mautrix/bridge/config.py index 1705e914..b98ebc52 100644 --- a/mautrix/bridge/config.py +++ b/mautrix/bridge/config.py @@ -200,14 +200,18 @@ def namespaces(self) -> dict[str, list[dict[str, Any]]]: "regex": re.escape(f"@{username_format}:{homeserver}").replace(regex_ph, ".*"), } ], - "aliases": [ - { - "exclusive": True, - "regex": re.escape(f"#{alias_format}:{homeserver}").replace(regex_ph, ".*"), - } - ] - if alias_format - else [], + "aliases": ( + [ + { + "exclusive": True, + "regex": re.escape(f"#{alias_format}:{homeserver}").replace( + regex_ph, ".*" + ), + } + ] + if alias_format + else [] + ), } def generate_registration(self) -> None: diff --git a/mautrix/client/api/filtering.py b/mautrix/client/api/filtering.py index 09889ec5..b8df1f36 100644 --- a/mautrix/client/api/filtering.py +++ b/mautrix/client/api/filtering.py @@ -50,9 +50,11 @@ async def create_filter(self, filter_params: Filter) -> FilterID: resp = await self.api.request( Method.POST, Path.v3.user[self.mxid].filter, - filter_params.serialize() - if isinstance(filter_params, Serializable) - else filter_params, + ( + filter_params.serialize() + if isinstance(filter_params, Serializable) + else filter_params + ), ) try: return resp["filter_id"] diff --git a/mautrix/client/api/rooms.py b/mautrix/client/api/rooms.py index 9f935430..a488fba6 100644 --- a/mautrix/client/api/rooms.py +++ b/mautrix/client/api/rooms.py @@ -354,9 +354,12 @@ async def join_room( except KeyError: raise MatrixResponseError("`room_id` not in response.") - fill_member_event_callback: Callable[ - [RoomID, UserID, MemberStateEventContent], Awaitable[MemberStateEventContent | None] - ] | None + fill_member_event_callback: ( + Callable[ + [RoomID, UserID, MemberStateEventContent], Awaitable[MemberStateEventContent | None] + ] + | None + ) async def fill_member_event( self, room_id: RoomID, user_id: UserID, content: MemberStateEventContent diff --git a/mautrix/crypto/encrypt_megolm.py b/mautrix/crypto/encrypt_megolm.py index 227fd3fe..459bbf1f 100644 --- a/mautrix/crypto/encrypt_megolm.py +++ b/mautrix/crypto/encrypt_megolm.py @@ -95,9 +95,9 @@ async def _encrypt_megolm_event( { "room_id": room_id, "type": event_type.serialize(), - "content": content.serialize() - if isinstance(content, Serializable) - else content, + "content": ( + content.serialize() if isinstance(content, Serializable) else content + ), } ) ) diff --git a/mautrix/types/event/type.pyi b/mautrix/types/event/type.pyi index bee0fde0..22922288 100644 --- a/mautrix/types/event/type.pyi +++ b/mautrix/types/event/type.pyi @@ -18,6 +18,7 @@ class EventType(Serializable): ACCOUNT_DATA = "account_data" EPHEMERAL = "ephemeral" TO_DEVICE = "to_device" + _by_event_type: ClassVar[dict[str, EventType]] ROOM_CANONICAL_ALIAS: "EventType" diff --git a/mautrix/util/bridge_state.py b/mautrix/util/bridge_state.py index dcb42977..d28448bf 100644 --- a/mautrix/util/bridge_state.py +++ b/mautrix/util/bridge_state.py @@ -115,9 +115,10 @@ async def send(self, url: str, token: str, log: logging.Logger, log_sent: bool = self.send_attempts_ += 1 headers = {"Authorization": f"Bearer {token}", "User-Agent": HTTPAPI.default_ua} try: - async with aiohttp.ClientSession() as sess, sess.post( - url, json=self.serialize(), headers=headers - ) as resp: + async with ( + aiohttp.ClientSession() as sess, + sess.post(url, json=self.serialize(), headers=headers) as resp, + ): if not 200 <= resp.status < 300: text = await resp.text() text = text.replace("\n", "\\n") diff --git a/mautrix/util/message_send_checkpoint.py b/mautrix/util/message_send_checkpoint.py index 5dd023b5..ee0c17f3 100644 --- a/mautrix/util/message_send_checkpoint.py +++ b/mautrix/util/message_send_checkpoint.py @@ -57,12 +57,15 @@ async def send(self, endpoint: str, as_token: str, log: logging.Logger) -> None: return try: headers = {"Authorization": f"Bearer {as_token}", "User-Agent": HTTPAPI.default_ua} - async with aiohttp.ClientSession() as sess, sess.post( - endpoint, - json={"checkpoints": [self.serialize()]}, - headers=headers, - timeout=ClientTimeout(30), - ) as resp: + async with ( + aiohttp.ClientSession() as sess, + sess.post( + endpoint, + json={"checkpoints": [self.serialize()]}, + headers=headers, + timeout=ClientTimeout(30), + ) as resp, + ): if not 200 <= resp.status < 300: text = await resp.text() text = text.replace("\n", "\\n") diff --git a/mautrix/util/utf16_surrogate.py b/mautrix/util/utf16_surrogate.py index d202e15e..92dc49c2 100644 --- a/mautrix/util/utf16_surrogate.py +++ b/mautrix/util/utf16_surrogate.py @@ -16,9 +16,11 @@ def add(text: str) -> str: The text with surrogate pairs. """ return "".join( - "".join(chr(y) for y in struct.unpack(" Date: Mon, 29 Jan 2024 18:49:28 +0200 Subject: [PATCH 392/456] Update missed version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 3f52625b..b01c50a1 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ ], extras_require={ "detect_mimetype": ["python-magic>=0.4.15,<0.5"], - "lint": ["black~=23.1", "isort"], + "lint": ["black~=24.1", "isort"], "test": ["pytest", "pytest-asyncio", *test_dependencies], "encryption": encryption_dependencies, }, From 9b4198a4c667e41ca25b657c8dc3305a978bd446 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 24 Feb 2024 22:10:37 +0200 Subject: [PATCH 393/456] Fix type error in download_thumbnail --- mautrix/client/api/modules/media_repository.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mautrix/client/api/modules/media_repository.py b/mautrix/client/api/modules/media_repository.py index fe52252a..779ec814 100644 --- a/mautrix/client/api/modules/media_repository.py +++ b/mautrix/client/api/modules/media_repository.py @@ -199,7 +199,7 @@ async def download_thumbnail( width: int | None = None, height: int | None = None, resize_method: Literal["crop", "scale"] = None, - allow_remote: bool = True, + allow_remote: bool | None = None, timeout_ms: int | None = None, ): """ @@ -232,7 +232,7 @@ async def download_thumbnail( if resize_method is not None: query_params["method"] = resize_method if allow_remote is not None: - query_params["allow_remote"] = allow_remote + query_params["allow_remote"] = str(allow_remote).lower() if timeout_ms is not None: query_params["timeout_ms"] = timeout_ms req_id = self.api.log_download_request(url, query_params) From 47e2157dd43e4854ef23a087e200a557868cd794 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 24 Feb 2024 22:11:18 +0200 Subject: [PATCH 394/456] Fix checking OTK count on server when no keys have been uploaded --- mautrix/crypto/machine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/crypto/machine.py b/mautrix/crypto/machine.py index 0cfe6ea3..3dc84856 100644 --- a/mautrix/crypto/machine.py +++ b/mautrix/crypto/machine.py @@ -292,7 +292,7 @@ async def _share_keys(self, current_otk_count: int | None) -> None: ): self.log.debug("Checking OTK count on server") current_otk_count = (await self.client.upload_keys()).get( - EncryptionKeyAlgorithm.SIGNED_CURVE25519 + EncryptionKeyAlgorithm.SIGNED_CURVE25519, 0 ) device_keys = ( self.account.get_device_keys(self.client.mxid, self.client.device_id) From 8eca64ed1dfd84acb40b36f9db950eb5b3e0ebd7 Mon Sep 17 00:00:00 2001 From: Joe Groocock Date: Thu, 28 Mar 2024 08:34:17 +0000 Subject: [PATCH 395/456] Support MSC2530 fields in media event types (#170) Add `filename`, `format` and `formatted_body` to media event content types. Signed-off-by: Joe Groocock --- mautrix/types/event/message.py | 55 +++++++++++++++++----------------- 1 file changed, 28 insertions(+), 27 deletions(-) diff --git a/mautrix/types/event/message.py b/mautrix/types/event/message.py index 8d5f1bba..30b1a153 100644 --- a/mautrix/types/event/message.py +++ b/mautrix/types/event/message.py @@ -271,33 +271,6 @@ class LocationInfo(SerializableAttrs): # region Event content -@dataclass -class MediaMessageEventContent(BaseMessageEventContent, SerializableAttrs): - """The content of a media message event (m.image, m.audio, m.video, m.file)""" - - url: Optional[ContentURI] = None - info: Optional[MediaInfo] = None - file: Optional[EncryptedFile] = None - - @staticmethod - @deserializer(MediaInfo) - @deserializer(Optional[MediaInfo]) - def deserialize_info(data: JSON) -> MediaInfo: - if not isinstance(data, dict): - return Obj() - msgtype = data.pop("__mautrix_msgtype", None) - if msgtype == "m.image" or msgtype == "m.sticker": - return ImageInfo.deserialize(data) - elif msgtype == "m.video": - return VideoInfo.deserialize(data) - elif msgtype == "m.audio": - return AudioInfo.deserialize(data) - elif msgtype == "m.file": - return FileInfo.deserialize(data) - else: - return Obj(**data) - - @dataclass class LocationMessageEventContent(BaseMessageEventContent, SerializableAttrs): geo_uri: str = None @@ -364,6 +337,34 @@ def _trim_reply_fallback_html(self) -> None: self.formatted_body = html_reply_fallback_regex.sub("", self.formatted_body) +@dataclass +class MediaMessageEventContent(TextMessageEventContent, SerializableAttrs): + """The content of a media message event (m.image, m.audio, m.video, m.file)""" + + url: Optional[ContentURI] = None + info: Optional[MediaInfo] = None + file: Optional[EncryptedFile] = None + filename: Optional[str] = None + + @staticmethod + @deserializer(MediaInfo) + @deserializer(Optional[MediaInfo]) + def deserialize_info(data: JSON) -> MediaInfo: + if not isinstance(data, dict): + return Obj() + msgtype = data.pop("__mautrix_msgtype", None) + if msgtype == "m.image" or msgtype == "m.sticker": + return ImageInfo.deserialize(data) + elif msgtype == "m.video": + return VideoInfo.deserialize(data) + elif msgtype == "m.audio": + return AudioInfo.deserialize(data) + elif msgtype == "m.file": + return FileInfo.deserialize(data) + else: + return Obj(**data) + + MessageEventContent = Union[ TextMessageEventContent, MediaMessageEventContent, LocationMessageEventContent, Obj ] From 55c53e06a204c7cf2a9277456d7615b7744e1f56 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 9 Jul 2024 10:47:08 +0300 Subject: [PATCH 396/456] Don't use cached base URL for double puppeting if one is configured --- mautrix/bridge/custom_puppet.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/mautrix/bridge/custom_puppet.py b/mautrix/bridge/custom_puppet.py index 9057877c..03034d55 100644 --- a/mautrix/bridge/custom_puppet.py +++ b/mautrix/bridge/custom_puppet.py @@ -132,8 +132,13 @@ def is_real_user(self) -> bool: return bool(self.custom_mxid and self.access_token) def _fresh_intent(self) -> IntentAPI: + _, server = self.az.intent.parse_user_id(self.custom_mxid) + try: + self.base_url = self.homeserver_url_map[server] + except KeyError: + if server == self.az.domain: + self.base_url = self.az.intent.api.base_url if self.access_token == "appservice-config" and self.custom_mxid: - _, server = self.az.intent.parse_user_id(self.custom_mxid) try: secret = self.login_shared_secret_map[server] except KeyError: From 1dbdc3fe3c91479a1b6fd0b30e9ed1bb848d4184 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 9 Jul 2024 11:10:32 +0300 Subject: [PATCH 397/456] Add support for authenticated media downloads --- mautrix/api.py | 8 ++++++- mautrix/appservice/api/intent.py | 2 ++ mautrix/appservice/appservice.py | 2 +- .../client/api/modules/media_repository.py | 21 +++++++++++++++---- mautrix/types/versions.py | 4 ++++ 5 files changed, 31 insertions(+), 6 deletions(-) diff --git a/mautrix/api.py b/mautrix/api.py index 39871bb5..1adde9ec 100644 --- a/mautrix/api.py +++ b/mautrix/api.py @@ -462,6 +462,7 @@ def get_download_url( mxc_uri: str, download_type: Literal["download", "thumbnail"] = "download", file_name: str | None = None, + authenticated: bool = False, ) -> URL: """ Get the full HTTP URL to download a ``mxc://`` URI. @@ -470,6 +471,7 @@ def get_download_url( mxc_uri: The MXC URI whose full URL to get. download_type: The type of download ("download" or "thumbnail"). file_name: Optionally, a file name to include in the download URL. + authenticated: Whether to use the new authenticated download endpoint in Matrix v1.11. Returns: The full HTTP URL. @@ -485,7 +487,11 @@ def get_download_url( "https://matrix-client.matrix.org/_matrix/media/v3/download/matrix.org/pqjkOuKZ1ZKRULWXgz2IVZV6/hello.png" """ server_name, media_id = self.parse_mxc_uri(mxc_uri) - url = self.base_url / str(APIPath.MEDIA) / "v3" / download_type / server_name / media_id + if authenticated: + url = self.base_url / str(APIPath.CLIENT) / "v1" / "media" + else: + url = self.base_url / str(APIPath.MEDIA) / "v3" + url = url / download_type / server_name / media_id if file_name: url /= file_name return url diff --git a/mautrix/appservice/api/intent.py b/mautrix/appservice/api/intent.py index 00990671..6e8e9fea 100644 --- a/mautrix/appservice/api/intent.py +++ b/mautrix/appservice/api/intent.py @@ -118,6 +118,8 @@ def __init__( ) -> None: super().__init__(mxid=mxid, api=api, state_store=state_store) self.bot = bot + if bot is not None: + self.versions_cache = bot.versions_cache self.log = api.base_log.getChild("intent") for method in ENSURE_REGISTERED_METHODS: diff --git a/mautrix/appservice/appservice.py b/mautrix/appservice/appservice.py index 892db195..65e202ae 100644 --- a/mautrix/appservice/appservice.py +++ b/mautrix/appservice/appservice.py @@ -13,7 +13,7 @@ from aiohttp import web import aiohttp -from mautrix.types import JSON, RoomAlias, UserID +from mautrix.types import JSON, RoomAlias, UserID, VersionsResponse from mautrix.util.logging import TraceLogger from ..api import HTTPAPI diff --git a/mautrix/client/api/modules/media_repository.py b/mautrix/client/api/modules/media_repository.py index 779ec814..80f7eb7d 100644 --- a/mautrix/client/api/modules/media_repository.py +++ b/mautrix/client/api/modules/media_repository.py @@ -10,6 +10,8 @@ import asyncio import time +from yarl import URL + from mautrix import __optional_imports__ from mautrix.api import MediaPath, Method from mautrix.errors import MatrixResponseError, make_request_error @@ -19,6 +21,7 @@ MediaRepoConfig, MXOpenGraph, SerializerError, + SpecVersions, ) from mautrix.util import background_task from mautrix.util.async_body import async_iter_bytes @@ -178,13 +181,17 @@ async def download_media(self, url: ContentURI, timeout_ms: int | None = None) - Returns: The raw downloaded data. """ - url = self.api.get_download_url(url) + authenticated = (await self.versions()).supports(SpecVersions.V111) + url = self.api.get_download_url(url, authenticated=authenticated) query_params: dict[str, Any] = {"allow_redirect": "true"} if timeout_ms is not None: query_params["timeout_ms"] = timeout_ms + headers: dict[str, str] = {} + if authenticated: + headers["Authorization"] = f"Bearer {self.api.token}" req_id = self.api.log_download_request(url, query_params) start = time.monotonic() - async with self.api.session.get(url, params=query_params) as response: + async with self.api.session.get(url, params=query_params, headers=headers) as response: try: response.raise_for_status() return await response.read() @@ -223,7 +230,10 @@ async def download_thumbnail( Returns: The raw downloaded data. """ - url = self.api.get_download_url(url, download_type="thumbnail") + authenticated = (await self.versions()).supports(SpecVersions.V111) + url = self.api.get_download_url( + url, download_type="thumbnail", authenticated=authenticated + ) query_params: dict[str, Any] = {"allow_redirect": "true"} if width is not None: query_params["width"] = width @@ -235,9 +245,12 @@ async def download_thumbnail( query_params["allow_remote"] = str(allow_remote).lower() if timeout_ms is not None: query_params["timeout_ms"] = timeout_ms + headers: dict[str, str] = {} + if authenticated: + headers["Authorization"] = f"Bearer {self.api.token}" req_id = self.api.log_download_request(url, query_params) start = time.monotonic() - async with self.api.session.get(url, params=query_params) as response: + async with self.api.session.get(url, params=query_params, headers=headers) as response: try: response.raise_for_status() return await response.read() diff --git a/mautrix/types/versions.py b/mautrix/types/versions.py index 9ad81710..52a62f59 100644 --- a/mautrix/types/versions.py +++ b/mautrix/types/versions.py @@ -74,6 +74,10 @@ class SpecVersions: V15 = Version.deserialize("v1.5") V16 = Version.deserialize("v1.6") V17 = Version.deserialize("v1.7") + V18 = Version.deserialize("v1.8") + V19 = Version.deserialize("v1.9") + V110 = Version.deserialize("v1.10") + V111 = Version.deserialize("v1.11") @dataclass From 3e89f1f55843634a5b9cad6d12620fa37d14588f Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 9 Jul 2024 11:44:02 +0300 Subject: [PATCH 398/456] Fix server URL overriding in _fresh_intent --- mautrix/bridge/custom_puppet.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/mautrix/bridge/custom_puppet.py b/mautrix/bridge/custom_puppet.py index 03034d55..f5befd5f 100644 --- a/mautrix/bridge/custom_puppet.py +++ b/mautrix/bridge/custom_puppet.py @@ -132,12 +132,13 @@ def is_real_user(self) -> bool: return bool(self.custom_mxid and self.access_token) def _fresh_intent(self) -> IntentAPI: - _, server = self.az.intent.parse_user_id(self.custom_mxid) - try: - self.base_url = self.homeserver_url_map[server] - except KeyError: - if server == self.az.domain: - self.base_url = self.az.intent.api.base_url + if self.custom_mxid: + _, server = self.az.intent.parse_user_id(self.custom_mxid) + try: + self.base_url = self.homeserver_url_map[server] + except KeyError: + if server == self.az.domain: + self.base_url = self.az.intent.api.base_url if self.access_token == "appservice-config" and self.custom_mxid: try: secret = self.login_shared_secret_map[server] From d645e208f70fef235420643c7110ca6d8109b51a Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 9 Jul 2024 11:46:42 +0300 Subject: [PATCH 399/456] Also set user_id for authenticated downloads --- mautrix/client/api/modules/media_repository.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mautrix/client/api/modules/media_repository.py b/mautrix/client/api/modules/media_repository.py index 80f7eb7d..b1d90cc1 100644 --- a/mautrix/client/api/modules/media_repository.py +++ b/mautrix/client/api/modules/media_repository.py @@ -189,6 +189,8 @@ async def download_media(self, url: ContentURI, timeout_ms: int | None = None) - headers: dict[str, str] = {} if authenticated: headers["Authorization"] = f"Bearer {self.api.token}" + if self.api.as_user_id: + query_params["user_id"] = self.api.as_user_id req_id = self.api.log_download_request(url, query_params) start = time.monotonic() async with self.api.session.get(url, params=query_params, headers=headers) as response: @@ -248,6 +250,8 @@ async def download_thumbnail( headers: dict[str, str] = {} if authenticated: headers["Authorization"] = f"Bearer {self.api.token}" + if self.api.as_user_id: + query_params["user_id"] = self.api.as_user_id req_id = self.api.log_download_request(url, query_params) start = time.monotonic() async with self.api.session.get(url, params=query_params, headers=headers) as response: From 6d1d944e32e2cf36dd30f7402dcda39769b649c2 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 9 Jul 2024 11:46:50 +0300 Subject: [PATCH 400/456] Bump version to 0.20.5 --- CHANGELOG.md | 11 +++++++++++ mautrix/__init__.py | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aad156fd..68e077bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,14 @@ +## v0.20.5 (2024-07-09) + +**Note:** The `bridge` module is deprecated as all bridges are being rewritten +in Go. See for more info. + +* *(client)* Added support for authenticated media downloads. +* *(bridge)* Stopped using cached homeserver URLs for double puppeting if one + is set in the config file. +* *(crypto)* Fixed error when checking OTK counts before uploading new keys. +* *(types)* Added MSC2530 (captions) fields to `MediaMessageEventContent`. + ## v0.20.4 (2024-01-09) * Dropped Python 3.9 support. diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 5d4ce561..8a0aa1ca 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.20.4" +__version__ = "0.20.5" __author__ = "Tulir Asokan " __all__ = [ "api", From 6fc1c6a1ca939652d3566daf87cc110276235ff0 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 12 Jul 2024 19:48:58 +0300 Subject: [PATCH 401/456] Register if /versions fails with M_FORBIDDEN --- mautrix/bridge/matrix.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/mautrix/bridge/matrix.py b/mautrix/bridge/matrix.py index da42a95e..e5399094 100644 --- a/mautrix/bridge/matrix.py +++ b/mautrix/bridge/matrix.py @@ -225,6 +225,12 @@ async def wait_for_connection(self) -> None: try: self.versions = await self.az.intent.versions() break + except MForbidden: + self.log.debug( + "/versions endpoint returned M_FORBIDDEN, " + "trying to register bridge bot before retrying..." + ) + await self.az.intent.ensure_registered() except Exception: self.log.exception("Connection to homeserver failed, retrying in 10 seconds") await asyncio.sleep(10) From c42da3deff9be83e740c4731f4e9381870899333 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 12 Jul 2024 19:49:05 +0300 Subject: [PATCH 402/456] Bump version to 0.20.6 --- CHANGELOG.md | 4 ++++ mautrix/__init__.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 68e077bb..b02d66ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## v0.20.6 (2024-07-12) + +* *(bridge)* Added `/register` call if `/versions` fails with `M_FORBIDDEN`. + ## v0.20.5 (2024-07-09) **Note:** The `bridge` module is deprecated as all bridges are being rewritten diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 8a0aa1ca..22035a89 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.20.5" +__version__ = "0.20.6" __author__ = "Tulir Asokan " __all__ = [ "api", From 6f25b62e80616fa8a8a57d12a03caca51b3c89b4 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 17 Nov 2024 21:25:44 +0200 Subject: [PATCH 403/456] Implement MSC2781 --- mautrix/types/event/message.py | 54 +--------------------------------- 1 file changed, 1 insertion(+), 53 deletions(-) diff --git a/mautrix/types/event/message.py b/mautrix/types/event/message.py index 30b1a153..32033581 100644 --- a/mautrix/types/event/message.py +++ b/mautrix/types/event/message.py @@ -119,7 +119,7 @@ def set_thread_parent( thread_parent.content.get_thread_parent() or self.relates_to.event_id ) if not disable_reply_fallback: - self.set_reply(last_event_in_thread or thread_parent, disable_fallback=True, **kwargs) + self.set_reply(last_event_in_thread or thread_parent, **kwargs) self.relates_to.is_falling_back = True def set_edit(self, edits: Union[EventID, "MessageEvent"]) -> None: @@ -287,25 +287,6 @@ class TextMessageEventContent(BaseMessageEventContent, SerializableAttrs): format: Format = None formatted_body: str = None - def set_reply( - self, - reply_to: Union["MessageEvent", EventID], - *, - displayname: Optional[str] = None, - disable_fallback: bool = False, - ) -> None: - super().set_reply(reply_to) - if isinstance(reply_to, str): - return - if isinstance(reply_to, MessageEvent) and not disable_fallback: - self.ensure_has_html() - if isinstance(reply_to.content, TextMessageEventContent): - reply_to.content.trim_reply_fallback() - self.formatted_body = ( - reply_to.make_reply_fallback_html(displayname) + self.formatted_body - ) - self.body = reply_to.make_reply_fallback_text(displayname) + self.body - def ensure_has_html(self) -> None: if not self.formatted_body or self.format != Format.HTML: self.format = Format.HTML @@ -424,36 +405,3 @@ def deserialize_content(data: JSON) -> MessageEventContent: return LocationMessageEventContent.deserialize(data) else: return Obj(**data) - - def make_reply_fallback_html(self, displayname: Optional[str] = None) -> str: - """Generate the HTML fallback for messages replying to this event.""" - if self.content.msgtype.is_text: - body = self.content.formatted_body or escape(self.content.body).replace("\n", "
") - else: - sent_type = media_reply_fallback_body_map[self.content.msgtype] or "a message" - body = f"sent {sent_type}" - displayname = escape(displayname) if displayname else self.sender - return html_reply_fallback_format.format( - room_id=self.room_id, - event_id=self.event_id, - sender=self.sender, - displayname=displayname, - content=body, - ) - - def make_reply_fallback_text(self, displayname: Optional[str] = None) -> str: - """Generate the plaintext fallback for messages replying to this event.""" - if self.content.msgtype.is_text: - body = self.content.body - else: - try: - body = media_reply_fallback_body_map[self.content.msgtype] - except KeyError: - body = "an unknown message type" - lines = body.strip().split("\n") - first_line, lines = lines[0], lines[1:] - fallback_text = f"> <{displayname or self.sender}> {first_line}" - for line in lines: - fallback_text += f"\n> {line}" - fallback_text += "\n\n" - return fallback_text From 7f2c0335cbc4a6ac9de6219b5ae552378db77660 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 3 Jan 2025 12:23:40 +0200 Subject: [PATCH 404/456] Update linters --- .github/workflows/python-package.yml | 6 +++--- .pre-commit-config.yaml | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 5a71c970..aef7eb74 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -8,7 +8,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.10", "3.11", "3.12"] + python-version: ["3.10", "3.11", "3.12", "3.13"] steps: - uses: actions/checkout@v4 @@ -49,14 +49,14 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: - python-version: "3.12" + python-version: "3.13" - uses: isort/isort-action@master with: sortPaths: "./mautrix" - uses: psf/black@stable with: src: "./mautrix" - version: "24.1.1" + version: "24.10.0" - name: pre-commit run: | pip install pre-commit diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a056ffdb..72cce792 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.5.0 + rev: v5.0.0 hooks: - id: trailing-whitespace exclude_types: [markdown] @@ -8,7 +8,7 @@ repos: - id: check-yaml - id: check-added-large-files - repo: https://github.com/psf/black - rev: 24.1.1 + rev: 24.10.0 hooks: - id: black language_version: python3 From c84730e2a4ea7f8a085924ed42d6e64de7d88246 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 3 Jan 2025 12:22:10 +0200 Subject: [PATCH 405/456] Bump version to 0.20.7 --- CHANGELOG.md | 7 +++++++ mautrix/__init__.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b02d66ad..08a4577d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## v0.20.7 (2025-01-03) + +* *(types)* Removed support for generating reply fallbacks to implement + [MSC2781]. Stripping fallbacks is still supported. + +[MSC2781]: https://github.com/matrix-org/matrix-spec-proposals/pull/2781 + ## v0.20.6 (2024-07-12) * *(bridge)* Added `/register` call if `/versions` fails with `M_FORBIDDEN`. diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 22035a89..d50477f4 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.20.6" +__version__ = "0.20.7" __author__ = "Tulir Asokan " __all__ = [ "api", From bba5542d8f11d1b9bbfc4e66a1ae3f37a1edbfb3 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 3 Jan 2025 12:31:48 +0200 Subject: [PATCH 406/456] Add Python 3.13 to classifiers --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index b01c50a1..d5c58eb0 100644 --- a/setup.py +++ b/setup.py @@ -45,6 +45,7 @@ "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", ], package_data={ From 33a034a95260286c1ce75cc7c381d997d1a2bb1f Mon Sep 17 00:00:00 2001 From: Marco Antonio Alvarez Date: Wed, 15 Jan 2025 17:19:02 +0100 Subject: [PATCH 407/456] Add support for MSC4190 (#175) --- mautrix/bridge/config.py | 4 ++++ mautrix/bridge/e2ee.py | 34 +++++++++++++++++++--------- mautrix/client/api/authentication.py | 15 ++++++++++++ 3 files changed, 42 insertions(+), 11 deletions(-) diff --git a/mautrix/bridge/config.py b/mautrix/bridge/config.py index b98ebc52..6127d5db 100644 --- a/mautrix/bridge/config.py +++ b/mautrix/bridge/config.py @@ -143,6 +143,7 @@ def do_update(self, helper: ConfigUpdateHelper) -> None: copy("bridge.encryption.default") copy("bridge.encryption.require") copy("bridge.encryption.appservice") + copy("bridge.encryption.msc4190") copy("bridge.encryption.delete_keys.delete_outbound_on_ack") copy("bridge.encryption.delete_keys.dont_store_outbound") copy("bridge.encryption.delete_keys.ratchet_on_decrypt") @@ -241,3 +242,6 @@ def generate_registration(self) -> None: if self["appservice.ephemeral_events"]: self._registration["de.sorunome.msc2409.push_ephemeral"] = True self._registration["push_ephemeral"] = True + + if self["bridge.encryption.msc4190"]: + self._registration["io.element.msc4190"] = True diff --git a/mautrix/bridge/e2ee.py b/mautrix/bridge/e2ee.py index 7ae66abf..266c8db9 100644 --- a/mautrix/bridge/e2ee.py +++ b/mautrix/bridge/e2ee.py @@ -57,6 +57,7 @@ class EncryptionManager: appservice_mode: bool periodically_delete_expired_keys: bool delete_outdated_inbound: bool + msc4190: bool bridge: br.Bridge az: AppService @@ -108,6 +109,7 @@ def __init__( self.crypto.send_keys_min_trust = TrustState.parse(verification_levels["receive"]) self.key_sharing_enabled = bridge.config["bridge.encryption.allow_key_sharing"] self.appservice_mode = bridge.config["bridge.encryption.appservice"] + self.msc4190 = bridge.config["bridge.encryption.msc4190"] if self.appservice_mode: self.az.otk_handler = self.crypto.handle_as_otk_counts self.az.device_list_handler = self.crypto.handle_as_device_lists @@ -246,7 +248,7 @@ async def decrypt(self, evt: EncryptedEvent, wait_session_timeout: int = 5) -> M async def start(self) -> None: flows = await self.client.get_login_flows() - if not flows.supports_type(LoginType.APPSERVICE): + if not self.msc4190 and not flows.supports_type(LoginType.APPSERVICE): self.log.critical( "Encryption enabled in config, but homeserver does not support appservice login" ) @@ -261,16 +263,26 @@ async def start(self) -> None: device_id = await self.crypto_store.get_device_id() if device_id: self.log.debug(f"Found device ID in database: {device_id}") - # We set the API token to the AS token here to authenticate the appservice login - # It'll get overridden after the login - self.client.api.token = self.az.as_token - await self.client.login( - login_type=LoginType.APPSERVICE, - device_name=self.device_name, - device_id=device_id, - store_access_token=True, - update_hs_url=False, - ) + + if self.msc4190: + if not device_id: + self.log.debug("Creating bot device with MSC4190") + self.client.api.token = self.az.as_token + await self.client.create_device_msc4190( + device_id=device_id, initial_display_name=self.device_name + ) + else: + # We set the API token to the AS token here to authenticate the appservice login + # It'll get overridden after the login + self.client.api.token = self.az.as_token + await self.client.login( + login_type=LoginType.APPSERVICE, + device_name=self.device_name, + device_id=device_id, + store_access_token=True, + update_hs_url=False, + ) + await self.crypto.load() if not device_id: await self.crypto_store.put_device_id(self.client.device_id) diff --git a/mautrix/client/api/authentication.py b/mautrix/client/api/authentication.py index cd77272d..0f6249ea 100644 --- a/mautrix/client/api/authentication.py +++ b/mautrix/client/api/authentication.py @@ -5,6 +5,8 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import annotations +import secrets + from mautrix.api import Method, Path from mautrix.errors import MatrixResponseError from mautrix.types import ( @@ -117,6 +119,19 @@ async def login( self.api.base_url = base_url.rstrip("/") return resp_data + async def create_device_msc4190(self, device_id: str, initial_display_name: str) -> None: + """ + Create a Device for a user of the homeserver using appservice interface defined in MSC4190 + """ + if len(device_id) == 0: + device_id = DeviceID(secrets.token_urlsafe(10)) + self.api.as_user_id = self.mxid + await self.api.request( + Method.PUT, Path.v3.devices[device_id], {"display_name": initial_display_name} + ) + self.api.as_device_id = device_id + self.device_id = device_id + async def logout(self, clear_access_token: bool = True) -> None: """ Invalidates an existing access token, so that it can no longer be used for authorization. From 626fcada6d82ee163afd888808b634df1b81eb2e Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 15 Jan 2025 18:02:12 +0200 Subject: [PATCH 408/456] Update push_ephemeral -> receive_ephemeral --- mautrix/bridge/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/bridge/config.py b/mautrix/bridge/config.py index 6127d5db..46fb88d2 100644 --- a/mautrix/bridge/config.py +++ b/mautrix/bridge/config.py @@ -241,7 +241,7 @@ def generate_registration(self) -> None: if self["appservice.ephemeral_events"]: self._registration["de.sorunome.msc2409.push_ephemeral"] = True - self._registration["push_ephemeral"] = True + self._registration["receive_ephemeral"] = True if self["bridge.encryption.msc4190"]: self._registration["io.element.msc4190"] = True From ca41430f91c545629a540cf05f57d82d987c066f Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 15 Jan 2025 18:52:44 +0200 Subject: [PATCH 409/456] Update changelog --- CHANGELOG.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 08a4577d..56f25916 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,13 @@ +## v0.20.8 (unreleased) + +* *(bridge)* Added support for [MSC4190] (thanks to [@surakin] in [#175]). +* *(appservice)* Renamed `push_ephemeral` in generated registrations to + `receive_ephemeral` to match the accepted version of [MSC2409]. + +[MSC4190]: https://github.com/matrix-org/matrix-spec-proposals/pull/2781 +[@surakin]: https://github.com/surakin +[#175]: https://github.com/mautrix/python/pull/175 + ## v0.20.7 (2025-01-03) * *(types)* Removed support for generating reply fallbacks to implement From 8eac9db01e2b5fd9a30620bcbc8ebbaa36c71ecb Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 15 Jan 2025 18:54:14 +0200 Subject: [PATCH 410/456] Bump version to 0.20.8b1 --- mautrix/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/__init__.py b/mautrix/__init__.py index d50477f4..91d7ec3c 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.20.7" +__version__ = "0.20.8b1" __author__ = "Tulir Asokan " __all__ = [ "api", From 4e3220c891c257caa4366283f1b48a47496a221e Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 1 Jun 2025 19:15:51 +0300 Subject: [PATCH 411/456] Start event loop in prepare step --- mautrix/util/program.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/mautrix/util/program.py b/mautrix/util/program.py index 2fb6b141..aa37b374 100644 --- a/mautrix/util/program.py +++ b/mautrix/util/program.py @@ -98,7 +98,7 @@ def _prepare(self) -> None: self.log.info(f"Initializing {self.name} {self.version}") try: - self.prepare() + self.loop.run_until_complete(self._async_prepare()) except Exception: self.log.critical("Unexpected error in initialization", exc_info=True) sys.exit(1) @@ -117,6 +117,7 @@ def preinit(self) -> None: self.prepare_config() self.prepare_log() self.check_config() + self.init_loop() @property def base_config_path(self) -> str: @@ -168,14 +169,16 @@ def prepare_log(self) -> None: logging.config.dictConfig(copy.deepcopy(self.config["logging"])) self.log = cast(TraceLogger, logging.getLogger("mau.init")) + async def _async_prepare(self) -> None: + self.prepare() + def prepare(self) -> None: """ Lifecycle method where the primary program initialization happens. Use this to fill startup_actions with async startup tasks. """ - self.prepare_loop() - def prepare_loop(self) -> None: + def init_loop(self) -> None: """Init lifecycle method where the asyncio event loop is created.""" if uvloop is not None: uvloop.install() From dbf642555de16374108ba3745fdbe3bbabfa0a3f Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 1 Jun 2025 19:21:22 +0300 Subject: [PATCH 412/456] Bump version to 0.20.8 --- CHANGELOG.md | 3 ++- mautrix/__init__.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 56f25916..8769162e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,9 @@ -## v0.20.8 (unreleased) +## v0.20.8 (2025-06-01) * *(bridge)* Added support for [MSC4190] (thanks to [@surakin] in [#175]). * *(appservice)* Renamed `push_ephemeral` in generated registrations to `receive_ephemeral` to match the accepted version of [MSC2409]. +* *(bridge)* Fixed compatibility with breaking change in aiohttp 3.12.6. [MSC4190]: https://github.com/matrix-org/matrix-spec-proposals/pull/2781 [@surakin]: https://github.com/surakin diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 91d7ec3c..69c8862a 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.20.8b1" +__version__ = "0.20.8" __author__ = "Tulir Asokan " __all__ = [ "api", From 0349445bd6992ac8f294582e85c3f61ce5c863b3 Mon Sep 17 00:00:00 2001 From: Christopher Johnstone Date: Mon, 11 Aug 2025 13:07:44 -0400 Subject: [PATCH 413/456] Check login flows only if MSC4190 is not enabled (#178) Currently, the homeserver login flows are checked even if MSC4190 is enabled. However, the `flows` variable is unused when MSC4190 is enabled. This is an unnecessary network call, and also e.g. requires a reverse proxy soley for this purpose if bridges would otherwise directly connect to a homeserver that is delegating OIDC authentication. Closes #177. --- mautrix/bridge/e2ee.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/mautrix/bridge/e2ee.py b/mautrix/bridge/e2ee.py index 266c8db9..56270942 100644 --- a/mautrix/bridge/e2ee.py +++ b/mautrix/bridge/e2ee.py @@ -247,12 +247,13 @@ async def decrypt(self, evt: EncryptedEvent, wait_session_timeout: int = 5) -> M return decrypted async def start(self) -> None: - flows = await self.client.get_login_flows() - if not self.msc4190 and not flows.supports_type(LoginType.APPSERVICE): - self.log.critical( - "Encryption enabled in config, but homeserver does not support appservice login" - ) - sys.exit(30) + if not self.msc4190: + flows = await self.client.get_login_flows() + if not flows.supports_type(LoginType.APPSERVICE): + self.log.critical( + "Encryption enabled in config, but homeserver does not support appservice login" + ) + sys.exit(30) self.log.debug("Logging in with bridge bot user") if self.crypto_db: try: From c95603784baa0f1c3036d447ed1fcda6fe5ac64f Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 11 Jul 2025 14:18:07 +0300 Subject: [PATCH 414/456] Add support for creator power --- mautrix/appservice/api/intent.py | 2 + mautrix/client/api/events.py | 46 +++++++++++++++++-- mautrix/client/state_store/abstract.py | 21 ++++++++- mautrix/client/state_store/asyncpg/store.py | 24 ++++++++++ mautrix/client/state_store/asyncpg/upgrade.py | 12 +++-- mautrix/client/state_store/file.py | 5 ++ mautrix/client/state_store/memory.py | 19 ++++++++ mautrix/client/store_updater.py | 35 ++++++++------ mautrix/types/event/state.py | 43 ++++++++++++++++- 9 files changed, 183 insertions(+), 24 deletions(-) diff --git a/mautrix/appservice/api/intent.py b/mautrix/appservice/api/intent.py index 6e8e9fea..626b34f1 100644 --- a/mautrix/appservice/api/intent.py +++ b/mautrix/appservice/api/intent.py @@ -710,6 +710,8 @@ async def _ensure_has_power_level_for( if not await self.state_store.has_power_levels_cached(room_id): # TODO add option to not try to fetch power levels from server await self.get_power_levels(room_id, ignore_cache=True, ensure_joined=False) + if not await self.state_store.has_create_cached(room_id): + await self.get_state_event(room_id, EventType.ROOM_CREATE, format="event") if not await self.state_store.has_power_level(room_id, self.mxid, event_type): # TODO implement something better raise IntentError( diff --git a/mautrix/client/api/events.py b/mautrix/client/api/events.py index 99e1c178..fd6b2c9d 100644 --- a/mautrix/client/api/events.py +++ b/mautrix/client/api/events.py @@ -5,7 +5,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import annotations -from typing import Awaitable +from typing import Awaitable, Literal, overload import json from mautrix.api import Method, Path @@ -168,12 +168,41 @@ async def get_event_context( ) return EventContext.deserialize(resp) + @overload async def get_state_event( self, room_id: RoomID, event_type: EventType, state_key: str = "", - ) -> StateEventContent: + *, + format: Literal["content"] = "content", + ) -> StateEventContent: ... + @overload + async def get_state_event( + self, + room_id: RoomID, + event_type: EventType, + state_key: str = "", + *, + format: Literal["event"], + ) -> StateEvent: ... + @overload + async def get_state_event( + self, + room_id: RoomID, + event_type: EventType, + state_key: str = "", + *, + format: str = "content", + ) -> StateEventContent | StateEvent: ... + async def get_state_event( + self, + room_id: RoomID, + event_type: EventType, + state_key: str = "", + *, + format: str = "content", + ) -> StateEventContent | StateEvent: """ Looks up the contents of a state event in a room. If the user is joined to the room then the state is taken from the current state of the room. If the user has left the room then the @@ -185,6 +214,9 @@ async def get_state_event( room_id: The ID of the room to look up the state in. event_type: The type of state to look up. state_key: The key of the state to look up. Defaults to empty string. + format: The format of the state event to return. Defaults to "content", which only returns + the content of the state event. If set to "event", the full event is returned. + See https://github.com/matrix-org/matrix-spec/issues/1047 for more info. Returns: The state event. @@ -192,11 +224,17 @@ async def get_state_event( content = await self.api.request( Method.GET, Path.v3.rooms[room_id].state[event_type][state_key], + query_params={"format": format} if format != "content" else None, metrics_method="getStateEvent", ) - content["__mautrix_event_type"] = event_type try: - return StateEvent.deserialize_content(content) + if format == "content": + content["__mautrix_event_type"] = event_type + return StateEvent.deserialize_content(content) + elif format == "event": + return StateEvent.deserialize(content) + else: + return content except SerializerError as e: raise MatrixResponseError("Invalid state event in response") from e diff --git a/mautrix/client/state_store/abstract.py b/mautrix/client/state_store/abstract.py index 3d37087d..d5b1f5f2 100644 --- a/mautrix/client/state_store/abstract.py +++ b/mautrix/client/state_store/abstract.py @@ -121,6 +121,18 @@ async def set_power_levels( ) -> None: pass + @abstractmethod + async def has_create_cached(self, room_id: RoomID) -> bool: + pass + + @abstractmethod + async def get_create(self, room_id: RoomID) -> StateEvent | None: + pass + + @abstractmethod + async def set_create(self, event: StateEvent) -> None: + pass + @abstractmethod async def has_encryption_info_cached(self, room_id: RoomID) -> bool: pass @@ -135,7 +147,7 @@ async def get_encryption_info(self, room_id: RoomID) -> RoomEncryptionStateEvent @abstractmethod async def set_encryption_info( - self, room_id: RoomID, content: RoomEncryptionStateEventContent | dict[str, any] + self, room_id: RoomID, content: RoomEncryptionStateEventContent | dict[str, Any] ) -> None: pass @@ -149,6 +161,8 @@ async def update_state(self, evt: StateEvent) -> None: await self.set_member(evt.room_id, UserID(evt.state_key), evt.content) elif evt.type == EventType.ROOM_ENCRYPTION: await self.set_encryption_info(evt.room_id, evt.content) + elif evt.type == EventType.ROOM_CREATE and evt.sender: + await self.set_create(evt) async def get_membership(self, room_id: RoomID, user_id: UserID) -> Membership: member = await self.get_member(room_id, user_id) @@ -172,4 +186,7 @@ async def has_power_level( room_levels = await self.get_power_levels(room_id) if not room_levels: return None - return room_levels.get_user_level(user_id) >= room_levels.get_event_level(event_type) + create_event = await self.get_create(room_id) + return room_levels.get_user_level(user_id, create_event) >= room_levels.get_event_level( + event_type + ) diff --git a/mautrix/client/state_store/asyncpg/store.py b/mautrix/client/state_store/asyncpg/store.py index d78c04d6..f4f8436f 100644 --- a/mautrix/client/state_store/asyncpg/store.py +++ b/mautrix/client/state_store/asyncpg/store.py @@ -16,6 +16,7 @@ RoomEncryptionStateEventContent, RoomID, Serializable, + StateEvent, UserID, ) from mautrix.util.async_db import Database, Scheme @@ -223,6 +224,29 @@ async def set_power_levels( json.dumps(content.serialize() if isinstance(content, Serializable) else content), ) + async def has_create_cached(self, room_id: RoomID) -> bool: + return bool( + await self.db.fetchval( + "SELECT create_event IS NOT NULL FROM mx_room_state WHERE room_id=$1", room_id + ) + ) + + async def get_create(self, room_id: RoomID) -> StateEvent | None: + create_event_json = await self.db.fetchval( + "SELECT create_event FROM mx_room_state WHERE room_id=$1", room_id + ) + if create_event_json is None: + return None + return StateEvent.parse_json(create_event_json) + + async def set_create(self, event: StateEvent) -> None: + await self.db.execute( + "INSERT INTO mx_room_state (room_id, create_event) VALUES ($1, $2) " + "ON CONFLICT (room_id) DO UPDATE SET create_event=$2", + event.room_id, + json.dumps(event.serialize() if isinstance(event, Serializable) else event), + ) + async def has_encryption_info_cached(self, room_id: RoomID) -> bool: return bool( await self.db.fetchval( diff --git a/mautrix/client/state_store/asyncpg/upgrade.py b/mautrix/client/state_store/asyncpg/upgrade.py index 88f115f2..0d722709 100644 --- a/mautrix/client/state_store/asyncpg/upgrade.py +++ b/mautrix/client/state_store/asyncpg/upgrade.py @@ -14,15 +14,16 @@ ) -@upgrade_table.register(description="Latest revision", upgrades_to=3) -async def upgrade_blank_to_v3(conn: Connection, scheme: Scheme) -> None: +@upgrade_table.register(description="Latest revision", upgrades_to=4) +async def upgrade_blank_to_v4(conn: Connection, scheme: Scheme) -> None: await conn.execute( """CREATE TABLE mx_room_state ( room_id TEXT PRIMARY KEY, is_encrypted BOOLEAN, has_full_member_list BOOLEAN, encryption TEXT, - power_levels TEXT + power_levels TEXT, + create_event TEXT, )""" ) membership_check = "" @@ -69,3 +70,8 @@ async def upgrade_v3(conn: Connection) -> None: WHERE mx_room_state.encryption IS NULL """ ) + + +@upgrade_table.register(description="Add create event to room state cache") +async def upgrade_v4(conn: Connection) -> None: + await conn.execute("ALTER TABLE mx_room_state ADD COLUMN create_event TYPE TEXT") diff --git a/mautrix/client/state_store/file.py b/mautrix/client/state_store/file.py index d567c853..a5d53663 100644 --- a/mautrix/client/state_store/file.py +++ b/mautrix/client/state_store/file.py @@ -15,6 +15,7 @@ PowerLevelStateEventContent, RoomEncryptionStateEventContent, RoomID, + StateEvent, UserID, ) from mautrix.util.file_store import Filer, FileStore @@ -65,3 +66,7 @@ async def set_power_levels( ) -> None: await super().set_power_levels(room_id, content) self._time_limited_flush() + + async def set_create(self, event: StateEvent) -> None: + await super().set_create(event) + self._time_limited_flush() diff --git a/mautrix/client/state_store/memory.py b/mautrix/client/state_store/memory.py index 8f2edac5..2b010a75 100644 --- a/mautrix/client/state_store/memory.py +++ b/mautrix/client/state_store/memory.py @@ -14,6 +14,7 @@ PowerLevelStateEventContent, RoomEncryptionStateEventContent, RoomID, + StateEvent, UserID, ) @@ -25,6 +26,7 @@ class SerializedStateStore(TypedDict): full_member_list: dict[RoomID, bool] power_levels: dict[RoomID, Any] encryption: dict[RoomID, Any] + create: dict[RoomID, Any] class MemoryStateStore(StateStore): @@ -32,12 +34,14 @@ class MemoryStateStore(StateStore): full_member_list: dict[RoomID, bool] power_levels: dict[RoomID, PowerLevelStateEventContent] encryption: dict[RoomID, RoomEncryptionStateEventContent | None] + create: dict[RoomID, StateEvent] def __init__(self) -> None: self.members = {} self.full_member_list = {} self.power_levels = {} self.encryption = {} + self.create = {} def serialize(self) -> SerializedStateStore: """ @@ -58,6 +62,7 @@ def serialize(self) -> SerializedStateStore: room_id: (content.serialize() if content is not None else None) for room_id, content in self.encryption.items() }, + "create": {room_id: evt.serialize() for room_id, evt in self.create.items()}, } def deserialize(self, data: SerializedStateStore) -> None: @@ -84,6 +89,9 @@ def deserialize(self, data: SerializedStateStore) -> None: ) for room_id, content in data["encryption"].items() } + self.create = { + room_id: StateEvent.deserialize(evt) for room_id, evt in data["create"].items() + } async def get_member(self, room_id: RoomID, user_id: UserID) -> Member | None: try: @@ -176,6 +184,17 @@ async def set_power_levels( content = PowerLevelStateEventContent.deserialize(content) self.power_levels[room_id] = content + async def has_create_cached(self, room_id: RoomID) -> bool: + return room_id in self.create + + async def get_create(self, room_id: RoomID) -> StateEvent | None: + return self.create.get(room_id) + + async def set_create(self, event: StateEvent | dict[str, Any]) -> None: + if not isinstance(event, StateEvent): + event = StateEvent.deserialize(event) + self.create[event.room_id] = event + async def has_encryption_info_cached(self, room_id: RoomID) -> bool: return room_id in self.encryption diff --git a/mautrix/client/store_updater.py b/mautrix/client/store_updater.py index a5530bde..35280324 100644 --- a/mautrix/client/store_updater.py +++ b/mautrix/client/store_updater.py @@ -5,6 +5,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import annotations +from typing import Literal import asyncio from mautrix.errors import MForbidden, MNotFound @@ -196,20 +197,28 @@ async def send_state_event( return event_id async def get_state_event( - self, room_id: RoomID, event_type: EventType, state_key: str = "" - ) -> StateEventContent: - event = await super().get_state_event(room_id, event_type, state_key) + self, + room_id: RoomID, + event_type: EventType, + state_key: str = "", + *, + format: str = "content", + ) -> StateEventContent | StateEvent: + event = await super().get_state_event(room_id, event_type, state_key, format=format) if self.state_store: - fake_event = StateEvent( - type=event_type, - room_id=room_id, - event_id=EventID(""), - sender=UserID(""), - state_key=state_key, - timestamp=0, - content=event, - ) - await self.state_store.update_state(fake_event) + if isinstance(event, StateEvent): + await self.state_store.update_state(event) + else: + fake_event = StateEvent( + type=event_type, + room_id=room_id, + event_id=EventID(""), + sender=UserID(""), + state_key=state_key, + timestamp=0, + content=event, + ) + await self.state_store.update_state(fake_event) return event async def get_joined_members(self, room_id: RoomID) -> dict[UserID, Member]: diff --git a/mautrix/types/event/state.py b/mautrix/types/event/state.py index c6b351c6..ef51cff4 100644 --- a/mautrix/types/event/state.py +++ b/mautrix/types/event/state.py @@ -41,7 +41,19 @@ class PowerLevelStateEventContent(SerializableAttrs): ban: int = 50 redact: int = 50 - def get_user_level(self, user_id: UserID) -> int: + def get_user_level( + self, + user_id: UserID, + create: Optional["StateEvent"] = None, + ) -> int: + if ( + create + and create.content.supports_creator_power + and (user_id == create.sender or user_id in (create.content.additional_creators or [])) + ): + # This is really meant to be infinity, but involving floats would be annoying, + # so we use an integer larger than the maximum power level (2^53-1) instead. + return 2**60 - 1 return int(self.users.get(user_id, self.users_default)) def set_user_level(self, user_id: UserID, level: int) -> None: @@ -50,7 +62,16 @@ def set_user_level(self, user_id: UserID, level: int) -> None: else: self.users[user_id] = level - def ensure_user_level(self, user_id: UserID, level: int) -> bool: + def ensure_user_level( + self, user_id: UserID, level: int, create: Optional["StateEvent"] = None + ) -> bool: + if ( + create + and create.content.supports_creator_power + and (user_id == create.sender or user_id in (create.content.additional_creators or [])) + ): + # Don't try to set creator power levels + return False if self.get_user_level(user_id) != level: self.set_user_level(user_id, level) return True @@ -193,6 +214,24 @@ class RoomCreateStateEventContent(SerializableAttrs): federate: bool = field(json="m.federate", omit_default=True, default=True) predecessor: Optional[RoomPredecessor] = None type: Optional[RoomType] = None + additional_creators: Optional[List[UserID]] = None + + @property + def supports_creator_power(self) -> bool: + return self.room_version not in ( + "", + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10", + "11", + ) @dataclass From 3b1690bd626b40219bfbdfdc63602a7b08e6a481 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 11 Aug 2025 22:27:42 +0300 Subject: [PATCH 415/456] Bump version to 0.20.9rc1 --- CHANGELOG.md | 9 +++++++++ mautrix/__init__.py | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8769162e..6bbb268b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,12 @@ +## v0.20.9 (unreleased) + +* *(event)* Added support for creator power in room v12+. +* *(bridge)* Removed check for login flows when using MSC4190 + (thanks to [@meson800] in [#178]). + +[@meson800]: https://github.com/meson800 +[#178]: https://github.com/mautrix/python/pull/178 + ## v0.20.8 (2025-06-01) * *(bridge)* Added support for [MSC4190] (thanks to [@surakin] in [#175]). diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 69c8862a..44635a05 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.20.8" +__version__ = "0.20.9rc1" __author__ = "Tulir Asokan " __all__ = [ "api", From 6acd0bebc8dd17a565bc7f1e95f0091e4eeb6cc0 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 11 Aug 2025 22:49:22 +0300 Subject: [PATCH 416/456] Fix typo in sql store upgrade --- mautrix/__init__.py | 2 +- mautrix/client/state_store/asyncpg/upgrade.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 44635a05..deb5af97 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.20.9rc1" +__version__ = "0.20.9rc2" __author__ = "Tulir Asokan " __all__ = [ "api", diff --git a/mautrix/client/state_store/asyncpg/upgrade.py b/mautrix/client/state_store/asyncpg/upgrade.py index 0d722709..72bc6d68 100644 --- a/mautrix/client/state_store/asyncpg/upgrade.py +++ b/mautrix/client/state_store/asyncpg/upgrade.py @@ -23,7 +23,7 @@ async def upgrade_blank_to_v4(conn: Connection, scheme: Scheme) -> None: has_full_member_list BOOLEAN, encryption TEXT, power_levels TEXT, - create_event TEXT, + create_event TEXT )""" ) membership_check = "" From a8c0ad51339e0d68d70565cba63150c26e18cc74 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 12 Aug 2025 10:09:30 +0300 Subject: [PATCH 417/456] Fix another typo in sql store upgrade --- mautrix/__init__.py | 2 +- mautrix/client/state_store/asyncpg/upgrade.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mautrix/__init__.py b/mautrix/__init__.py index deb5af97..beb96400 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.20.9rc2" +__version__ = "0.20.9rc3" __author__ = "Tulir Asokan " __all__ = [ "api", diff --git a/mautrix/client/state_store/asyncpg/upgrade.py b/mautrix/client/state_store/asyncpg/upgrade.py index 72bc6d68..c1f32664 100644 --- a/mautrix/client/state_store/asyncpg/upgrade.py +++ b/mautrix/client/state_store/asyncpg/upgrade.py @@ -74,4 +74,4 @@ async def upgrade_v3(conn: Connection) -> None: @upgrade_table.register(description="Add create event to room state cache") async def upgrade_v4(conn: Connection) -> None: - await conn.execute("ALTER TABLE mx_room_state ADD COLUMN create_event TYPE TEXT") + await conn.execute("ALTER TABLE mx_room_state ADD COLUMN create_event TEXT") From 970f371b033b39fd00135792fa3b190fe43d4e27 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 13 Aug 2025 22:14:11 +0300 Subject: [PATCH 418/456] Update link to C-S spec --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index e03596d9..f1342c19 100644 --- a/README.rst +++ b/README.rst @@ -49,7 +49,7 @@ Components .. _#maunium:maunium.net: https://matrix.to/#/#maunium:maunium.net .. _python-appservice-framework: https://github.com/Cadair/python-appservice-framework/ -.. _Client API: https://matrix.org/docs/spec/client_server/r0.6.1.html +.. _Client API: https://spec.matrix.org/latest/client-server-api/ .. _mautrix.api: https://docs.mau.fi/python/latest/api/mautrix.api.html .. _mautrix.client.api: https://docs.mau.fi/python/latest/api/mautrix.client.api.html From ba883ea18ae7af80a83ee5203b21dde17a993aa0 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 20 Aug 2025 10:07:15 +0300 Subject: [PATCH 419/456] Allow filtering event handlers by sync source --- mautrix/client/syncer.py | 52 ++++++++++++++++++++++++++-------------- 1 file changed, 34 insertions(+), 18 deletions(-) diff --git a/mautrix/client/syncer.py b/mautrix/client/syncer.py index 8e115bbb..8183e142 100644 --- a/mautrix/client/syncer.py +++ b/mautrix/client/syncer.py @@ -5,11 +5,11 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import annotations -from typing import Any, Awaitable, Callable, Type, TypeVar +from typing import Any, Awaitable, Callable, NamedTuple, Optional, Type, TypeVar from abc import ABC, abstractmethod -from contextlib import suppress from enum import Enum, Flag, auto import asyncio +import itertools import time from mautrix.errors import MUnknownToken @@ -25,7 +25,6 @@ Filter, FilterID, GenericEvent, - MessageEvent, PresenceState, SerializerError, StateEvent, @@ -79,13 +78,18 @@ class InternalEventType(Enum): DEVICE_OTK_COUNT = auto() +class EventHandlerProps(NamedTuple): + wait_sync: bool + sync_stream: Optional[SyncStream] + + class Syncer(ABC): loop: asyncio.AbstractEventLoop log: TraceLogger mxid: UserID - global_event_handlers: list[tuple[EventHandler, bool]] - event_handlers: dict[EventType | InternalEventType, list[tuple[EventHandler, bool]]] + global_event_handlers: dict[EventHandler, EventHandlerProps] + event_handlers: dict[EventType | InternalEventType, dict[EventHandler, EventHandlerProps]] dispatchers: dict[Type[dispatcher.Dispatcher], dispatcher.Dispatcher] syncing_task: asyncio.Task | None ignore_initial_sync: bool @@ -95,7 +99,7 @@ class Syncer(ABC): sync_store: SyncStore def __init__(self, sync_store: SyncStore) -> None: - self.global_event_handlers = [] + self.global_event_handlers = {} self.event_handlers = {} self.dispatchers = {} self.syncing_task = None @@ -158,6 +162,7 @@ def add_event_handler( event_type: InternalEventType | EventType, handler: EventHandler, wait_sync: bool = False, + sync_stream: Optional[SyncStream] = None, ) -> None: """ Add a new event handler. @@ -167,13 +172,15 @@ def add_event_handler( event types. handler: The handler function to add. wait_sync: Whether or not the handler should be awaited before the next sync request. + sync_stream: The sync streams to listen to. Defaults to all. """ if not isinstance(event_type, (EventType, InternalEventType)): raise ValueError("Invalid event type") + props = EventHandlerProps(wait_sync=wait_sync, sync_stream=sync_stream) if event_type == EventType.ALL: - self.global_event_handlers.append((handler, wait_sync)) + self.global_event_handlers[handler] = props else: - self.event_handlers.setdefault(event_type, []).append((handler, wait_sync)) + self.event_handlers.setdefault(event_type, {})[handler] = props def remove_event_handler( self, event_type: EventType | InternalEventType, handler: EventHandler @@ -197,11 +204,7 @@ def remove_event_handler( # No handlers for this event type registered return - # FIXME this is a bit hacky - with suppress(ValueError): - handler_list.remove((handler, True)) - with suppress(ValueError): - handler_list.remove((handler, False)) + handler_list.pop(handler, None) if len(handler_list) == 0 and event_type != EventType.ALL: del self.event_handlers[event_type] @@ -229,7 +232,9 @@ def dispatch_event(self, event: Event | None, source: SyncStream) -> list[asynci else: event.type = event.type.with_class(EventType.Class.MESSAGE) setattr(event, "source", source) - return self.dispatch_manual_event(event.type, event, include_global_handlers=True) + return self.dispatch_manual_event( + event.type, event, include_global_handlers=True, source=source + ) async def _catch_errors(self, handler: EventHandler, data: Any) -> None: try: @@ -243,13 +248,22 @@ def dispatch_manual_event( data: Any, include_global_handlers: bool = False, force_synchronous: bool = False, + source: Optional[SyncStream] = None, ) -> list[asyncio.Task]: - handlers = self.event_handlers.get(event_type, []) + handlers = self.event_handlers.get(event_type, {}).items() if include_global_handlers: - handlers = self.global_event_handlers + handlers + handlers = itertools.chain(self.global_event_handlers.items(), handlers) tasks = [] - for handler, wait_sync in handlers: - if force_synchronous or wait_sync: + if source is None: + source = getattr(data, "source", None) + for handler, props in handlers: + if ( + props.sync_stream is not None + and source is not None + and not props.sync_stream & source + ): + continue + if force_synchronous or props.wait_sync: tasks.append(asyncio.create_task(self._catch_errors(handler, data))) else: background_task.create(self._catch_errors(handler, data)) @@ -263,6 +277,7 @@ async def run_internal_event( event_type, custom_type if custom_type is not None else kwargs, include_global_handlers=False, + source=SyncStream.INTERNAL, ) await asyncio.gather(*tasks) @@ -274,6 +289,7 @@ def dispatch_internal_event( event_type, custom_type if custom_type is not None else kwargs, include_global_handlers=False, + source=SyncStream.INTERNAL, ) def _try_deserialize(self, type: Type[T], data: JSON) -> T | GenericEvent: From ecd10f57bf13da3508f5ca98294c1ca2167bbf39 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 13 Sep 2025 19:45:17 +0300 Subject: [PATCH 420/456] Fix style in variation selector code --- mautrix/util/variation_selector.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mautrix/util/variation_selector.py b/mautrix/util/variation_selector.py index 498f1abe..dec0554d 100644 --- a/mautrix/util/variation_selector.py +++ b/mautrix/util/variation_selector.py @@ -59,11 +59,11 @@ async def fetch_data() -> dict[str, str]: ADD_VARIATION_TRANSLATION = str.maketrans( {ord(emoji): f"{emoji}{VARIATION_SELECTOR_16}" for emoji in read_data().values()} ) -SKIN_TONE_MODIFIERS = ("\U0001F3FB", "\U0001F3FC", "\U0001F3FD", "\U0001F3FE", "\U0001F3FF") +SKIN_TONE_MODIFIERS = ("\U0001f3fb", "\U0001f3fc", "\U0001f3fd", "\U0001f3fe", "\U0001f3ff") SKIN_TONE_REPLACEMENTS = {f"{VARIATION_SELECTOR_16}{mod}": mod for mod in SKIN_TONE_MODIFIERS} VARIATION_SELECTOR_REPLACEMENTS = { **SKIN_TONE_REPLACEMENTS, - "\U0001F408\ufe0f\u200d\u2b1b\ufe0f": "\U0001F408\u200d\u2b1b", + "\U0001f408\ufe0f\u200d\u2b1b\ufe0f": "\U0001f408\u200d\u2b1b", } From 799cb27267d0addb02e28fe7eb33e38987dc20d7 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 13 Sep 2025 19:23:43 +0300 Subject: [PATCH 421/456] Add utilities for SSSS --- mautrix/crypto/ssss/__init__.py | 7 + mautrix/crypto/ssss/key.py | 136 +++++++++++++++++++ mautrix/crypto/ssss/key_test.py | 199 ++++++++++++++++++++++++++++ mautrix/crypto/ssss/types.py | 51 +++++++ mautrix/crypto/ssss/util.py | 79 +++++++++++ mautrix/types/__init__.py | 5 + mautrix/types/event/__init__.py | 2 + mautrix/types/event/account_data.py | 20 ++- mautrix/types/event/type.py | 5 + mautrix/types/event/type.pyi | 5 + optional-requirements.txt | 1 + 11 files changed, 508 insertions(+), 2 deletions(-) create mode 100644 mautrix/crypto/ssss/__init__.py create mode 100644 mautrix/crypto/ssss/key.py create mode 100644 mautrix/crypto/ssss/key_test.py create mode 100644 mautrix/crypto/ssss/types.py create mode 100644 mautrix/crypto/ssss/util.py diff --git a/mautrix/crypto/ssss/__init__.py b/mautrix/crypto/ssss/__init__.py new file mode 100644 index 00000000..77389424 --- /dev/null +++ b/mautrix/crypto/ssss/__init__.py @@ -0,0 +1,7 @@ +from .key import Key, KeyMetadata, PassphraseMetadata +from .types import ( + Algorithm, + EncryptedAccountDataEventContent, + EncryptedKeyData, + PassphraseAlgorithm, +) diff --git a/mautrix/crypto/ssss/key.py b/mautrix/crypto/ssss/key.py new file mode 100644 index 00000000..0a159b3e --- /dev/null +++ b/mautrix/crypto/ssss/key.py @@ -0,0 +1,136 @@ +# Copyright (c) 2025 Tulir Asokan +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +from typing import Optional +import base64 +import hashlib +import hmac + +from attr import dataclass +import unpaddedbase64 + +from mautrix.types import EventType, SerializableAttrs + +from .types import Algorithm, EncryptedKeyData, PassphraseAlgorithm +from .util import ( + calculate_hash, + cryptorand, + decode_base58_recovery_key, + derive_keys, + encode_base58_recovery_key, + prepare_aes, +) + +try: + from Crypto.Cipher import AES + from Crypto.Util import Counter +except ImportError: + from Cryptodome.Cipher import AES + from Cryptodome.Util import Counter + + +@dataclass +class PassphraseMetadata(SerializableAttrs): + algorithm: PassphraseAlgorithm + iterations: int + salt: str + bits: int = 256 + + def get_key(self, passphrase: str) -> bytes: + if self.algorithm != PassphraseAlgorithm.PBKDF2: + raise ValueError(f"Unsupported passphrase algorithm {self.algorithm}") + return hashlib.pbkdf2_hmac( + "sha512", + passphrase.encode("utf-8"), + self.salt.encode("utf-8"), + self.iterations, + self.bits // 8, + ) + + +@dataclass +class KeyMetadata(SerializableAttrs): + algorithm: Algorithm + + iv: str | None = None + mac: str | None = None + + name: str | None = None + passphrase: Optional[PassphraseMetadata] = None + + def verify_passphrase(self, key_id: str, phrase: str) -> "Key": + if not self.passphrase: + raise ValueError("Passphrase not set on this key") + return self.verify_raw_key(key_id, self.passphrase.get_key(phrase)) + + def verify_recovery_key(self, key_id: str, recovery_key: str) -> "Key": + return self.verify_raw_key(key_id, decode_base58_recovery_key(recovery_key)) + + def verify_raw_key(self, key_id: str, key: bytes) -> "Key": + if self.mac.rstrip("=") != calculate_hash(key, self.iv): + raise ValueError("Key MAC does not match") + return Key(id=key_id, key=key, metadata=self) + + +@dataclass +class Key: + id: str + key: bytes + metadata: KeyMetadata + + @classmethod + def generate(cls, passphrase: str | None = None) -> "Key": + passphrase_meta = ( + PassphraseMetadata( + algorithm=PassphraseAlgorithm.PBKDF2, + iterations=500_000, + salt=base64.b64encode(cryptorand.read(24)).decode("utf-8"), + bits=256, + ) + if passphrase + else None + ) + key = passphrase_meta.get_key(passphrase) if passphrase else cryptorand.read(32) + iv = unpaddedbase64.encode_base64(cryptorand.read(16)) + metadata = KeyMetadata( + algorithm=Algorithm.AES_HMAC_SHA2, + passphrase=passphrase_meta, + mac=calculate_hash(key, iv), + iv=iv, + ) + key_id = unpaddedbase64.encode_base64(cryptorand.read(24)) + return cls(key=key, id=key_id, metadata=metadata) + + @property + def recovery_key(self) -> str: + return encode_base58_recovery_key(self.key) + + def encrypt(self, event_type: str | EventType, data: str | bytes) -> EncryptedKeyData: + if isinstance(data, str): + data = data.encode("utf-8") + data = base64.b64encode(data).rstrip(b"=") + + aes_key, hmac_key = derive_keys(self.key, event_type) + iv = bytearray(cryptorand.read(16)) + iv[8] &= 0x7F + ciphertext = prepare_aes(aes_key, iv).encrypt(data) + digest = hmac.digest(hmac_key, ciphertext, hashlib.sha256) + return EncryptedKeyData( + ciphertext=unpaddedbase64.encode_base64(ciphertext), + iv=unpaddedbase64.encode_base64(iv), + mac=unpaddedbase64.encode_base64(digest), + ) + + def decrypt(self, event_type: str | EventType, data: EncryptedKeyData) -> bytes: + aes_key, hmac_key = derive_keys(self.key, event_type) + ciphertext = unpaddedbase64.decode_base64(data.ciphertext) + mac = unpaddedbase64.decode_base64(data.mac) + + expected_mac = hmac.digest(hmac_key, ciphertext, hashlib.sha256) + if not hmac.compare_digest(mac, expected_mac): + raise ValueError("Invalid MAC") + + plaintext = prepare_aes(aes_key, data.iv).decrypt(ciphertext) + return unpaddedbase64.decode_base64(plaintext.decode("utf-8")) diff --git a/mautrix/crypto/ssss/key_test.py b/mautrix/crypto/ssss/key_test.py new file mode 100644 index 00000000..e60f1e3c --- /dev/null +++ b/mautrix/crypto/ssss/key_test.py @@ -0,0 +1,199 @@ +# Copyright (c) 2025 Tulir Asokan +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +import pytest + +from ...types.event.type import EventType +from .key import Key, KeyMetadata +from .types import EncryptedAccountDataEventContent + +KEY1_CROSS_SIGNING_MASTER_KEY = """{ + "encrypted": { + "gEJqbfSEMnP5JXXcukpXEX1l0aI3MDs0": { + "iv": "BpKP9nQJTE9jrsAssoxPqQ==", + "ciphertext": "fNRiiiidezjerTgV+G6pUtmeF3izzj5re/mVvY0hO2kM6kYGrxLuIu2ej80=", + "mac": "/gWGDGMyOLmbJp+aoSLh5JxCs0AdS6nAhjzpe+9G2Q0=" + } + } +}""" + +KEY1_CROSS_SIGNING_MASTER_KEY_DECRYPTED = bytes( + [ + 0x68, + 0xF9, + 0x7F, + 0xD1, + 0x92, + 0x2E, + 0xEC, + 0xF6, + 0xB8, + 0x2B, + 0xB8, + 0x90, + 0xD2, + 0x4D, + 0x06, + 0x52, + 0x98, + 0x4E, + 0x7A, + 0x1D, + 0x70, + 0x3B, + 0x9E, + 0x86, + 0x7B, + 0x7E, + 0xBA, + 0xF7, + 0xFE, + 0xB9, + 0x5B, + 0x6F, + ] +) + +KEY1_META = """{ + "algorithm": "m.secret_storage.v1.aes-hmac-sha2", + "passphrase": { + "algorithm": "m.pbkdf2", + "iterations": 500000, + "salt": "y863BOoqOadgDp8S3FtHXikDJEalsQ7d" + }, + "iv": "xxkTK0L4UzxgAFkQ6XPwsw", + "mac": "MEhooO0ZhFJNxUhvRMSxBnJfL20wkLgle3ocY0ee/eA" +}""" +KEY1_ID = "gEJqbfSEMnP5JXXcukpXEX1l0aI3MDs0" +KEY1_RECOVERY_KEY = "EsTE s92N EtaX s2h6 VQYF 9Kao tHYL mkyL GKMh isZb KJ4E tvoC" +KEY1_PASSPHRASE = "correct horse battery staple" + +KEY2_META = """{ + "algorithm": "m.secret_storage.v1.aes-hmac-sha2", + "iv": "O0BOvTqiIAYjC+RMcyHfWw==", + "mac": "7k6OruQlWg0UmQjxGZ0ad4Q6DdwkgnoI7G6X3IjBYtI=" +}""" +KEY2_ID = "NVe5vK6lZS9gEMQLJw0yqkzmE5Mr7dLv" +KEY2_RECOVERY_KEY = "EsUC xSxt XJgQ dz19 8WBZ rHdE GZo7 ybsn EFmG Y5HY MDAG GNWe" + +KEY2_META_BROKEN_IV = """{ + "algorithm": "m.secret_storage.v1.aes-hmac-sha2", + "iv": "O0BOvTqiIAYjC+RMcyHfWwMeowMeowMeow", + "mac": "7k6OruQlWg0UmQjxGZ0ad4Q6DdwkgnoI7G6X3IjBYtI=" +}""" + +KEY2_META_BROKEN_MAC = """{ + "algorithm": "m.secret_storage.v1.aes-hmac-sha2", + "iv": "O0BOvTqiIAYjC+RMcyHfWw==", + "mac": "7k6OruQlWg0UmQjxGZ0ad4Q6DdwkgnoI7G6X3IjBYtIMeowMeowMeow" +}""" + + +def get_key_meta(meta: str) -> KeyMetadata: + return KeyMetadata.parse_json(meta) + + +def get_key1() -> Key: + return get_key_meta(KEY1_META).verify_recovery_key(KEY1_ID, KEY1_RECOVERY_KEY) + + +def get_key2() -> Key: + return get_key_meta(KEY2_META).verify_recovery_key(KEY2_ID, KEY2_RECOVERY_KEY) + + +def get_encrypted_master_key() -> EncryptedAccountDataEventContent: + return EncryptedAccountDataEventContent.parse_json(KEY1_CROSS_SIGNING_MASTER_KEY) + + +def test_decrypt_success() -> None: + key = get_key1() + emk = get_encrypted_master_key() + assert ( + emk.decrypt(EventType.CROSS_SIGNING_MASTER, key) == KEY1_CROSS_SIGNING_MASTER_KEY_DECRYPTED + ) + + +def test_decrypt_fail_wrong_key() -> None: + key = get_key2() + emk = get_encrypted_master_key() + with pytest.raises(ValueError): + emk.decrypt(EventType.CROSS_SIGNING_MASTER, key) + + +def test_decrypt_fail_fake_key() -> None: + key = get_key2() + key.id = KEY1_ID + emk = get_encrypted_master_key() + with pytest.raises(ValueError): + emk.decrypt(EventType.CROSS_SIGNING_MASTER, key) + + +def test_decrypt_fail_wrong_type() -> None: + key = get_key1() + emk = get_encrypted_master_key() + with pytest.raises(ValueError): + emk.decrypt(EventType.CROSS_SIGNING_SELF_SIGNING, key) + + +def test_encrypt_roundtrip() -> None: + key = get_key1() + data = bytes([0xDE, 0xAD, 0xBE, 0xEF]) + ciphertext = key.encrypt("net.maunium.data", data) + plaintext = key.decrypt("net.maunium.data", ciphertext) + assert plaintext == data + + +def test_verify_recovery_key_correct() -> None: + meta = get_key_meta(KEY1_META) + key = meta.verify_recovery_key(KEY1_ID, KEY1_RECOVERY_KEY) + assert key.recovery_key == KEY1_RECOVERY_KEY + + +def test_verify_recovery_key_correct2() -> None: + meta = get_key_meta(KEY2_META) + key = meta.verify_recovery_key(KEY2_ID, KEY2_RECOVERY_KEY) + assert key.recovery_key == KEY2_RECOVERY_KEY + + +def test_verify_recovery_key_invalid() -> None: + meta = get_key_meta(KEY1_META) + with pytest.raises(ValueError): + meta.verify_recovery_key(KEY1_ID, "foo") + + +def test_verify_recovery_key_incorrect() -> None: + meta = get_key_meta(KEY1_META) + with pytest.raises(ValueError): + meta.verify_recovery_key(KEY2_ID, KEY2_RECOVERY_KEY) + + +def test_verify_recovery_key_broken_iv() -> None: + meta = get_key_meta(KEY2_META_BROKEN_IV) + with pytest.raises(ValueError): + meta.verify_recovery_key(KEY2_ID, KEY2_RECOVERY_KEY) + + +def test_verify_recovery_key_broken_mac() -> None: + meta = get_key_meta(KEY2_META_BROKEN_MAC) + with pytest.raises(ValueError): + meta.verify_recovery_key(KEY2_ID, KEY2_RECOVERY_KEY) + + +def test_verify_passphrase_correct() -> None: + meta = get_key_meta(KEY1_META) + key = meta.verify_passphrase(KEY1_ID, KEY1_PASSPHRASE) + assert key.recovery_key == KEY1_RECOVERY_KEY + + +def test_verify_passphrase_incorrect() -> None: + meta = get_key_meta(KEY1_META) + with pytest.raises(ValueError): + meta.verify_passphrase(KEY1_ID, "incorrect horse battery staple") + + +def test_verify_passphrase_notset() -> None: + meta = get_key_meta(KEY2_META) + with pytest.raises(ValueError): + meta.verify_passphrase(KEY2_ID, "hmm") diff --git a/mautrix/crypto/ssss/types.py b/mautrix/crypto/ssss/types.py new file mode 100644 index 00000000..4a47f743 --- /dev/null +++ b/mautrix/crypto/ssss/types.py @@ -0,0 +1,51 @@ +# Copyright (c) 2025 Tulir Asokan +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +from typing import TYPE_CHECKING + +from attr import dataclass + +from mautrix.types import EventType, SerializableAttrs, SerializableEnum +from mautrix.types.event.account_data import account_data_event_content_map + +if TYPE_CHECKING: + from .key import Key + + +class Algorithm(SerializableEnum): + AES_HMAC_SHA2 = "m.secret_storage.v1.aes-hmac-sha2" + CURVE25519_AES_SHA2 = "m.secret_storage.v1.curve25519-aes-sha2" + + +class PassphraseAlgorithm(SerializableEnum): + PBKDF2 = "m.pbkdf2" + + +@dataclass +class EncryptedKeyData(SerializableAttrs): + ciphertext: str + iv: str + mac: str + + +@dataclass +class EncryptedAccountDataEventContent(SerializableAttrs): + encrypted: dict[str, EncryptedKeyData] + + def decrypt(self, event_type: str | EventType, key: "Key") -> bytes: + try: + encrypted_data = self.encrypted[key.id] + except KeyError as e: + raise ValueError(f"Event not encrypted for provided key") from e + return key.decrypt(event_type, encrypted_data) + + +for encrypted_account_data_type in ( + EventType.CROSS_SIGNING_MASTER, + EventType.CROSS_SIGNING_USER_SIGNING, + EventType.CROSS_SIGNING_SELF_SIGNING, + EventType.MEGOLM_BACKUP_V1, +): + account_data_event_content_map[encrypted_account_data_type] = EncryptedAccountDataEventContent diff --git a/mautrix/crypto/ssss/util.py b/mautrix/crypto/ssss/util.py new file mode 100644 index 00000000..9ebf6dc9 --- /dev/null +++ b/mautrix/crypto/ssss/util.py @@ -0,0 +1,79 @@ +# Copyright (c) 2025 Tulir Asokan +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +import hashlib +import hmac +import struct + +import base58 +import unpaddedbase64 + +from mautrix.types import EventType + +try: + from Crypto import Random + from Crypto.Cipher import AES + from Crypto.Hash import SHA256 + from Crypto.Protocol.KDF import HKDF + from Crypto.Util import Counter +except ImportError: + from Cryptodome import Random + from Cryptodome.Cipher import AES + from Cryptodome.Hash import SHA256 + from Cryptodome.Protocol.KDF import HKDF + from Cryptodome.Util import Counter + +cryptorand = Random.new() + + +def decode_base58_recovery_key(key: str) -> bytes | None: + key_bytes = base58.b58decode(key.replace(" ", "")) + if len(key_bytes) != 35 or key_bytes[0] != 0x8B or key_bytes[1] != 1: + return None + parity = 0 + for byte in key_bytes[:34]: + parity ^= byte + return key_bytes[2:34] if parity == key_bytes[34] else None + + +def encode_base58_recovery_key(key: bytes) -> str: + key_bytes = bytearray(35) + key_bytes[0] = 0x8B + key_bytes[1] = 1 + key_bytes[2:34] = key + parity = 0 + for byte in key_bytes: + parity ^= byte + key_bytes[34] = parity + encoded_key = base58.b58encode(key_bytes).decode("utf-8") + return " ".join(encoded_key[i : i + 4] for i in range(0, len(encoded_key), 4)) + + +def derive_keys(key: bytes, name: str | EventType = "") -> tuple[bytes, bytes]: + aes_key, hmac_key = HKDF( + master=key, + key_len=32, + salt=b"\x00" * 32, + hashmod=SHA256, + num_keys=2, + context=str(name).encode("utf-8"), + ) + return aes_key, hmac_key + + +def prepare_aes(key: bytes, iv: str | bytes) -> AES: + if isinstance(iv, str): + iv = unpaddedbase64.decode_base64(iv) + # initial_value = struct.unpack(">Q", iv[8:])[0] + # counter = Counter.new(64, prefix=iv[:8], initial_value=initial_value) + counter = Counter.new(128, initial_value=int.from_bytes(iv)) + return AES.new(key=key, mode=AES.MODE_CTR, counter=counter) + + +def calculate_hash(key: bytes, iv: str | bytes) -> str: + aes_key, hmac_key = derive_keys(key) + cipher = prepare_aes(aes_key, iv).decrypt(b"\x00" * 32) + digest = hmac.digest(hmac_key, cipher, hashlib.sha256) + return unpaddedbase64.encode_base64(digest) diff --git a/mautrix/types/__init__.py b/mautrix/types/__init__.py index 42b9068c..ceceeaca 100644 --- a/mautrix/types/__init__.py +++ b/mautrix/types/__init__.py @@ -57,6 +57,7 @@ CallRejectEventContent, CallSelectAnswerEventContent, CanonicalAliasStateEventContent, + DirectAccountDataEventContent, EncryptedEvent, EncryptedEventContent, EncryptedFile, @@ -122,6 +123,7 @@ RoomTombstoneStateEventContent, RoomTopicStateEventContent, RoomType, + SecretStorageDefaultKeyEventContent, SingleReceiptEventContent, SpaceChildStateEventContent, SpaceParentStateEventContent, @@ -259,6 +261,7 @@ "CallRejectEventContent", "CallSelectAnswerEventContent", "CanonicalAliasStateEventContent", + "DirectAccountDataEventContent", "EncryptedEvent", "EncryptedEventContent", "EncryptedFile", @@ -324,6 +327,7 @@ "RoomTombstoneStateEventContent", "RoomTopicStateEventContent", "RoomType", + "SecretStorageDefaultKeyEventContent", "SingleReceiptEventContent", "SpaceChildStateEventContent", "SpaceParentStateEventContent", @@ -354,6 +358,7 @@ "OpenGraphImage", "OpenGraphVideo", "BatchSendResponse", + "BeeperBatchSendResponse", "DeviceLists", "DeviceOTKCount", "DirectoryPaginationToken", diff --git a/mautrix/types/event/__init__.py b/mautrix/types/event/__init__.py index b391e912..db0658db 100644 --- a/mautrix/types/event/__init__.py +++ b/mautrix/types/event/__init__.py @@ -6,8 +6,10 @@ from .account_data import ( AccountDataEvent, AccountDataEventContent, + DirectAccountDataEventContent, RoomTagAccountDataEventContent, RoomTagInfo, + SecretStorageDefaultKeyEventContent, ) from .base import BaseEvent, BaseRoomEvent, BaseUnsigned, GenericEvent from .batch import BatchSendEvent, BatchSendStateEvent diff --git a/mautrix/types/event/account_data.py b/mautrix/types/event/account_data.py index 3c144a36..fdfa0a30 100644 --- a/mautrix/types/event/account_data.py +++ b/mautrix/types/event/account_data.py @@ -3,7 +3,7 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from typing import Dict, List, Union +from typing import TYPE_CHECKING, Dict, List, Union from attr import dataclass import attr @@ -12,6 +12,9 @@ from ..util import Obj, SerializableAttrs, deserializer from .base import BaseEvent, EventType +if TYPE_CHECKING: + from mautrix.crypto.ssss import EncryptedAccountDataEventContent, KeyMetadata + @dataclass class RoomTagInfo(SerializableAttrs): @@ -23,11 +26,24 @@ class RoomTagAccountDataEventContent(SerializableAttrs): tags: Dict[str, RoomTagInfo] = attr.ib(default=None, metadata={"json": "tags"}) +@dataclass +class SecretStorageDefaultKeyEventContent(SerializableAttrs): + key: str + + DirectAccountDataEventContent = Dict[UserID, List[RoomID]] -AccountDataEventContent = Union[RoomTagAccountDataEventContent, DirectAccountDataEventContent, Obj] +AccountDataEventContent = Union[ + RoomTagAccountDataEventContent, + DirectAccountDataEventContent, + SecretStorageDefaultKeyEventContent, + "EncryptedAccountDataEventContent", + "KeyMetadata", + Obj, +] account_data_event_content_map = { EventType.TAG: RoomTagAccountDataEventContent, + EventType.SECRET_STORAGE_DEFAULT_KEY: SecretStorageDefaultKeyEventContent, # m.direct doesn't really need deserializing # EventType.DIRECT: DirectAccountDataEventContent, } diff --git a/mautrix/types/event/type.py b/mautrix/types/event/type.py index 509d6857..5faf3785 100644 --- a/mautrix/types/event/type.py +++ b/mautrix/types/event/type.py @@ -207,6 +207,11 @@ def is_to_device(self) -> bool: "m.push_rules": "PUSH_RULES", "m.tag": "TAG", "m.ignored_user_list": "IGNORED_USER_LIST", + "m.secret_storage.default_key": "SECRET_STORAGE_DEFAULT_KEY", + "m.cross_signing.master": "CROSS_SIGNING_MASTER", + "m.cross_signing.self_signing": "CROSS_SIGNING_SELF_SIGNING", + "m.cross_signing.user_signing": "CROSS_SIGNING_USER_SIGNING", + "m.megolm_backup.v1": "MEGOLM_BACKUP_V1", }, EventType.Class.TO_DEVICE: { "m.room.encrypted": "TO_DEVICE_ENCRYPTED", diff --git a/mautrix/types/event/type.pyi b/mautrix/types/event/type.pyi index 22922288..a2788d6f 100644 --- a/mautrix/types/event/type.pyi +++ b/mautrix/types/event/type.pyi @@ -61,6 +61,11 @@ class EventType(Serializable): PUSH_RULES: "EventType" TAG: "EventType" IGNORED_USER_LIST: "EventType" + SECRET_STORAGE_DEFAULT_KEY: "EventType" + CROSS_SIGNING_MASTER: "EventType" + CROSS_SIGNING_SELF_SIGNING: "EventType" + CROSS_SIGNING_USER_SIGNING: "EventType" + MEGOLM_BACKUP_V1: "EventType" TO_DEVICE_ENCRYPTED: "EventType" TO_DEVICE_DUMMY: "EventType" diff --git a/optional-requirements.txt b/optional-requirements.txt index a5660f4a..a6e0227d 100644 --- a/optional-requirements.txt +++ b/optional-requirements.txt @@ -11,3 +11,4 @@ uvloop python-olm unpaddedbase64 pycryptodome +base58 From 5e3dbf5c6e82a74960f9a905bb7be2393675a18e Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 13 Sep 2025 19:45:02 +0300 Subject: [PATCH 422/456] Add API calls for SSSS --- mautrix/crypto/base.py | 2 + mautrix/crypto/machine.py | 3 ++ mautrix/crypto/ssss/__init__.py | 1 + mautrix/crypto/ssss/machine.py | 65 +++++++++++++++++++++++++++++++++ 4 files changed, 71 insertions(+) create mode 100644 mautrix/crypto/ssss/machine.py diff --git a/mautrix/crypto/base.py b/mautrix/crypto/base.py index 1f8e9a62..b7b2171f 100644 --- a/mautrix/crypto/base.py +++ b/mautrix/crypto/base.py @@ -31,6 +31,7 @@ from mautrix.util.logging import TraceLogger from .. import client as cli, crypto +from .ssss import Machine as SSSSMachine class SignedObject(TypedDict): @@ -40,6 +41,7 @@ class SignedObject(TypedDict): class BaseOlmMachine: client: cli.Client + ssss: SSSSMachine log: TraceLogger crypto_store: crypto.CryptoStore state_store: crypto.StateStore diff --git a/mautrix/crypto/machine.py b/mautrix/crypto/machine.py index 3dc84856..adca95b2 100644 --- a/mautrix/crypto/machine.py +++ b/mautrix/crypto/machine.py @@ -36,6 +36,7 @@ from .encrypt_megolm import MegolmEncryptionMachine from .key_request import KeyRequestingMachine from .key_share import KeySharingMachine +from .ssss import Machine as SSSSMachine from .store import CryptoStore, StateStore from .unwedge import OlmUnwedgingMachine @@ -58,6 +59,7 @@ class OlmMachine( log: TraceLogger crypto_store: CryptoStore state_store: StateStore + ssss: SSSSMachine account: Optional[OlmAccount] @@ -70,6 +72,7 @@ def __init__( ) -> None: super().__init__() self.client = client + self.ssss = SSSSMachine(client) self.log = log or logging.getLogger("mau.crypto") self.crypto_store = crypto_store self.state_store = state_store diff --git a/mautrix/crypto/ssss/__init__.py b/mautrix/crypto/ssss/__init__.py index 77389424..9224418d 100644 --- a/mautrix/crypto/ssss/__init__.py +++ b/mautrix/crypto/ssss/__init__.py @@ -1,4 +1,5 @@ from .key import Key, KeyMetadata, PassphraseMetadata +from .machine import Machine from .types import ( Algorithm, EncryptedAccountDataEventContent, diff --git a/mautrix/crypto/ssss/machine.py b/mautrix/crypto/ssss/machine.py new file mode 100644 index 00000000..c43e25e3 --- /dev/null +++ b/mautrix/crypto/ssss/machine.py @@ -0,0 +1,65 @@ +# Copyright (c) 2025 Tulir Asokan +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +from mautrix import client as cli +from mautrix.errors import MNotFound +from mautrix.types import EventType, SecretStorageDefaultKeyEventContent + +from .key import Key, KeyMetadata +from .types import EncryptedAccountDataEventContent + + +class Machine: + client: cli.Client + + def __init__(self, client: cli.Client) -> None: + self.client = client + + async def get_default_key_id(self) -> str | None: + try: + data = await self.client.get_account_data(EventType.SECRET_STORAGE_DEFAULT_KEY) + return SecretStorageDefaultKeyEventContent.deserialize(data).key + except (MNotFound, ValueError): + return None + + async def set_default_key_id(self, key_id: str) -> None: + await self.client.set_account_data( + EventType.SECRET_STORAGE_DEFAULT_KEY, + SecretStorageDefaultKeyEventContent(key=key_id), + ) + + async def get_key_data(self, key_id: str) -> KeyMetadata: + data = await self.client.get_account_data(f"m.secret_storage.key.{key_id}") + return KeyMetadata.deserialize(data) + + async def set_key_data(self, key_id: str, data: KeyMetadata) -> None: + await self.client.set_account_data(f"m.secret_storage.key.{key_id}", data) + + async def get_default_key_data(self) -> tuple[str, KeyMetadata]: + key_id = await self.get_default_key_id() + if not key_id: + raise ValueError("No default key ID set") + return key_id, await self.get_key_data(key_id) + + async def get_decrypted_account_data(self, event_type: EventType | str, key: Key) -> bytes: + data = await self.client.get_account_data(event_type) + parsed = EncryptedAccountDataEventContent.deserialize(data) + return parsed.decrypt(event_type, key) + + async def set_encrypted_account_data( + self, event_type: EventType | str, data: bytes, *keys: Key + ) -> None: + encrypted_data = {} + for key in keys: + encrypted_data[key.id] = key.encrypt(event_type, data) + await self.client.set_account_data( + event_type, + EncryptedAccountDataEventContent(encrypted=encrypted_data), + ) + + async def generate_and_upload_key(self, passphrase: str | None = None) -> Key: + key = Key.generate(passphrase) + await self.set_key_data(key.id, key.metadata) + return key From ec318c0406d7ab7368d70fd77669afe0ce348d1d Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 13 Sep 2025 20:32:53 +0300 Subject: [PATCH 423/456] Move verify_signature_json to new file and add test --- mautrix/crypto/base.py | 39 +------------------------- mautrix/crypto/device_lists.py | 3 +- mautrix/crypto/encrypt_olm.py | 3 +- mautrix/crypto/signature.py | 47 ++++++++++++++++++++++++++++++++ mautrix/crypto/signature_test.py | 39 ++++++++++++++++++++++++++ mautrix/crypto/ssss/util.py | 1 - 6 files changed, 91 insertions(+), 41 deletions(-) create mode 100644 mautrix/crypto/signature.py create mode 100644 mautrix/crypto/signature_test.py diff --git a/mautrix/crypto/base.py b/mautrix/crypto/base.py index b7b2171f..f7015ca1 100644 --- a/mautrix/crypto/base.py +++ b/mautrix/crypto/base.py @@ -5,26 +5,18 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import annotations -from typing import Any, Awaitable, Callable, TypedDict +from typing import Awaitable, Callable import asyncio -import functools -import json - -import olm from mautrix.errors import MForbidden, MNotFound from mautrix.types import ( - DeviceID, - EncryptionKeyAlgorithm, EventType, IdentityKey, - KeyID, RequestedKeyInfo, RoomEncryptionStateEventContent, RoomID, RoomKeyEventContent, SessionID, - SigningKey, TrustState, UserID, ) @@ -34,11 +26,6 @@ from .ssss import Machine as SSSSMachine -class SignedObject(TypedDict): - signatures: dict[UserID, dict[str, str]] - unsigned: Any - - class BaseOlmMachine: client: cli.Client ssss: SSSSMachine @@ -118,27 +105,3 @@ async def _fill_encryption_info(self, evt: RoomKeyEventContent) -> None: evt.beeper_max_age_ms = encryption_info.rotation_period_ms if not evt.beeper_max_messages: evt.beeper_max_messages = encryption_info.rotation_period_msgs - - -canonical_json = functools.partial( - json.dumps, ensure_ascii=False, separators=(",", ":"), sort_keys=True -) - - -def verify_signature_json( - data: "SignedObject", user_id: UserID, key_name: DeviceID | str, key: SigningKey -) -> bool: - data_copy = {**data} - data_copy.pop("unsigned", None) - signatures = data_copy.pop("signatures") - key_id = str(KeyID(EncryptionKeyAlgorithm.ED25519, key_name)) - try: - signature = signatures[user_id][key_id] - except KeyError: - return False - signed_data = canonical_json(data_copy) - try: - olm.ed25519_verify(key, signed_data, signature) - return True - except olm.OlmVerifyError: - return False diff --git a/mautrix/crypto/device_lists.py b/mautrix/crypto/device_lists.py index 1b5e0dbd..93648889 100644 --- a/mautrix/crypto/device_lists.py +++ b/mautrix/crypto/device_lists.py @@ -23,7 +23,8 @@ UserID, ) -from .base import BaseOlmMachine, verify_signature_json +from .base import BaseOlmMachine +from .signature import verify_signature_json class DeviceListMachine(BaseOlmMachine): diff --git a/mautrix/crypto/encrypt_olm.py b/mautrix/crypto/encrypt_olm.py index 620cbbc8..029ad1e5 100644 --- a/mautrix/crypto/encrypt_olm.py +++ b/mautrix/crypto/encrypt_olm.py @@ -18,8 +18,9 @@ UserID, ) -from .base import BaseOlmMachine, verify_signature_json +from .base import BaseOlmMachine from .sessions import Session +from .signature import verify_signature_json ClaimKeysList = Dict[UserID, Dict[DeviceID, DeviceIdentity]] diff --git a/mautrix/crypto/signature.py b/mautrix/crypto/signature.py new file mode 100644 index 00000000..32f4342a --- /dev/null +++ b/mautrix/crypto/signature.py @@ -0,0 +1,47 @@ +# Copyright (c) 2025 Tulir Asokan +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +from typing import Any, TypedDict +import functools +import json + +import olm + +from mautrix.types import DeviceID, EncryptionKeyAlgorithm, KeyID, SigningKey, UserID + +try: + from Crypto.PublicKey import ECC + from Crypto.Signature import eddsa +except ImportError: + from Cryptodome.PublicKey import ECC + from Cryptodome.Signature import eddsa + +canonical_json = functools.partial( + json.dumps, ensure_ascii=False, separators=(",", ":"), sort_keys=True +) + + +class SignedObject(TypedDict): + signatures: dict[UserID, dict[str, str]] + unsigned: Any + + +def verify_signature_json( + data: "SignedObject", user_id: UserID, key_name: DeviceID | str, key: SigningKey +) -> bool: + data_copy = {**data} + data_copy.pop("unsigned", None) + signatures = data_copy.pop("signatures") + key_id = str(KeyID(EncryptionKeyAlgorithm.ED25519, key_name)) + try: + signature = signatures[user_id][key_id] + except KeyError: + return False + signed_data = canonical_json(data_copy) + try: + olm.ed25519_verify(key, signed_data, signature) + return True + except olm.OlmVerifyError: + return False diff --git a/mautrix/crypto/signature_test.py b/mautrix/crypto/signature_test.py new file mode 100644 index 00000000..115e4836 --- /dev/null +++ b/mautrix/crypto/signature_test.py @@ -0,0 +1,39 @@ +# Copyright (c) 2025 Tulir Asokan +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +from mautrix.types import SigningKey, UserID + +from .signature import verify_signature_json + + +def test_verify_signature_json() -> None: + assert verify_signature_json( + # This is actually a federation PDU rather than a device signature, + # but they're both 25519 curves so it doesn't make a difference. + { + "auth_events": [ + "$L8Ak6A939llTRIsZrytMlLDXQhI4uLEjx-wb1zSg-Bw", + "$QJmr7mmGeXGD4Tof0ZYSPW2oRGklseyHTKtZXnF-YNM", + "$7bkKK_Z-cGQ6Ae4HXWGBwXyZi3YjC6rIcQzGfVyl3Eo", + ], + "content": {}, + "depth": 3212, + "hashes": {"sha256": "K549YdTnv62Jn84Y7sS5ZN3+AdmhleZHbenbhUpR2R8"}, + "origin_server_ts": 1754242687127, + "prev_events": ["$DAhJg4jVsqk5FRatE2hbT1dSA8D2ASy5DbjEHIMSHwY"], + "room_id": "!offtopic-2:continuwuity.org", + "sender": "@tulir:maunium.net", + "type": "m.room.message", + "signatures": { + UserID("maunium.net"): { + "ed25519:a_xxeS": "SkzZdZ+rH22kzCBBIAErTdB0Vg6vkFmzvwjlOarGul72EnufgtE/tJcd3a8szAdK7f1ZovRyQxDgVm/Ib2u0Aw" + } + }, + "unsigned": {"age_ts": 1754242687146}, + }, + UserID("maunium.net"), + "a_xxeS", + SigningKey("lVt/CC3tv74OH6xTph2JrUmeRj/j+1q0HVa0Xf4QlCg"), + ) diff --git a/mautrix/crypto/ssss/util.py b/mautrix/crypto/ssss/util.py index 9ebf6dc9..bc7dfb81 100644 --- a/mautrix/crypto/ssss/util.py +++ b/mautrix/crypto/ssss/util.py @@ -5,7 +5,6 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. import hashlib import hmac -import struct import base58 import unpaddedbase64 From 71f4fae6cd1d32f12e09a04af7199da70ff4e4d1 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sat, 13 Sep 2025 20:33:14 +0300 Subject: [PATCH 424/456] Switch to pycryptodome for ed25519 verification --- mautrix/crypto/signature.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/mautrix/crypto/signature.py b/mautrix/crypto/signature.py index 32f4342a..ca5220f8 100644 --- a/mautrix/crypto/signature.py +++ b/mautrix/crypto/signature.py @@ -7,7 +7,7 @@ import functools import json -import olm +import unpaddedbase64 from mautrix.types import DeviceID, EncryptionKeyAlgorithm, KeyID, SigningKey, UserID @@ -37,11 +37,13 @@ def verify_signature_json( key_id = str(KeyID(EncryptionKeyAlgorithm.ED25519, key_name)) try: signature = signatures[user_id][key_id] - except KeyError: - return False - signed_data = canonical_json(data_copy) - try: - olm.ed25519_verify(key, signed_data, signature) + decoded_key = unpaddedbase64.decode_base64(key) + # pycryptodome doesn't accept raw keys, so wrap it in a DER structure + der_key = b"\x30\x2a\x30\x05\x06\x03\x2b\x65\x70\x03\x21\x00" + decoded_key + decoded_signature = unpaddedbase64.decode_base64(signature) + parsed_key = ECC.import_key(der_key) + verifier = eddsa.new(parsed_key, "rfc8032") + verifier.verify(canonical_json(data_copy).encode("utf-8"), decoded_signature) return True - except olm.OlmVerifyError: + except (KeyError, ValueError): return False From 0ea9586cc2a7317e1d679503137a2bc4c2c3cbef Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 15 Sep 2025 02:22:06 +0300 Subject: [PATCH 425/456] Use more dataclasses for uploading e2ee device keys --- mautrix/client/api/modules/crypto.py | 10 ++++++-- mautrix/crypto/account.py | 35 ++++++++++++++-------------- mautrix/crypto/machine.py | 1 + mautrix/crypto/signature.py | 20 +++++++++++++++- mautrix/types/event/encrypted.py | 14 ++++++++++- 5 files changed, 59 insertions(+), 21 deletions(-) diff --git a/mautrix/client/api/modules/crypto.py b/mautrix/client/api/modules/crypto.py index af656916..065bffda 100644 --- a/mautrix/client/api/modules/crypto.py +++ b/mautrix/client/api/modules/crypto.py @@ -5,13 +5,15 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import annotations -from typing import Any +from typing import Any, Union from mautrix.api import Method, Path from mautrix.errors import MatrixResponseError from mautrix.types import ( + JSON, ClaimKeysResponse, DeviceID, + DeviceKeys, EncryptionKeyAlgorithm, EventType, QueryKeysResponse, @@ -82,7 +84,7 @@ async def send_to_one_device( async def upload_keys( self, one_time_keys: dict[str, Any] | None = None, - device_keys: dict[str, Any] | None = None, + device_keys: DeviceKeys | dict[str, Any] | None = None, ) -> dict[EncryptionKeyAlgorithm, int]: """ Publishes end-to-end encryption keys for the device. @@ -102,8 +104,12 @@ async def upload_keys( """ data = {} if device_keys: + if isinstance(device_keys, Serializable): + device_keys = device_keys.serialize() data["device_keys"] = device_keys if one_time_keys: + if isinstance(one_time_keys, Serializable): + one_time_keys = one_time_keys.serialize() data["one_time_keys"] = one_time_keys resp = await self.api.request(Method.POST, Path.v3.keys.upload, data) try: diff --git a/mautrix/crypto/account.py b/mautrix/crypto/account.py index db508262..63f83f93 100644 --- a/mautrix/crypto/account.py +++ b/mautrix/crypto/account.py @@ -10,15 +10,17 @@ from mautrix.types import ( DeviceID, + DeviceKeys, EncryptionAlgorithm, EncryptionKeyAlgorithm, IdentityKey, + KeyID, SigningKey, UserID, ) -from . import base from .sessions import Session +from .signature import sign_olm class OlmAccount(olm.Account): @@ -74,19 +76,18 @@ def new_outbound_session(self, target_key: IdentityKey, one_time_key: IdentityKe session.pickle("roundtrip"), passphrase="roundtrip", creation_time=datetime.now() ) - def get_device_keys(self, user_id: UserID, device_id: DeviceID) -> Dict[str, Any]: - device_keys = { - "user_id": user_id, - "device_id": device_id, - "algorithms": [EncryptionAlgorithm.OLM_V1.value, EncryptionAlgorithm.MEGOLM_V1.value], - "keys": { - f"{algorithm}:{device_id}": key for algorithm, key in self.identity_keys.items() + def get_device_keys(self, user_id: UserID, device_id: DeviceID) -> DeviceKeys: + device_keys = DeviceKeys( + user_id=user_id, + device_id=device_id, + algorithms=[EncryptionAlgorithm.OLM_V1, EncryptionAlgorithm.MEGOLM_V1], + keys={ + KeyID(algorithm=EncryptionKeyAlgorithm(algorithm), key_id=key): key + for algorithm, key in self.identity_keys.items() }, - } - signature = self.sign(base.canonical_json(device_keys)) - device_keys["signatures"] = { - user_id: {f"{EncryptionKeyAlgorithm.ED25519}:{device_id}": signature} - } + signatures={}, + ) + device_keys.signatures[user_id] = {KeyID.ed25519(device_id): sign_olm(device_keys, self)} return device_keys def get_one_time_keys( @@ -97,12 +98,12 @@ def get_one_time_keys( self.generate_one_time_keys(new_count) keys = {} for key_id, key in self.one_time_keys.get("curve25519", {}).items(): - signature = self.sign(base.canonical_json({"key": key})) - keys[f"{EncryptionKeyAlgorithm.SIGNED_CURVE25519}:{key_id}"] = { + keys[str(KeyID.signed_curve25519(IdentityKey(key_id)))] = { "key": key, "signatures": { - user_id: {f"{EncryptionKeyAlgorithm.ED25519}:{device_id}": signature} + user_id: { + str(KeyID.ed25519(device_id)): sign_olm({"key": key}, self), + } }, } - self.mark_keys_as_published() return keys diff --git a/mautrix/crypto/machine.py b/mautrix/crypto/machine.py index adca95b2..9088662d 100644 --- a/mautrix/crypto/machine.py +++ b/mautrix/crypto/machine.py @@ -313,6 +313,7 @@ async def _share_keys(self, current_otk_count: int | None) -> None: self.log.debug(f"Uploading {len(one_time_keys)} one-time keys") resp = await self.client.upload_keys(one_time_keys=one_time_keys, device_keys=device_keys) self.account.shared = True + self.account.mark_keys_as_published() self._last_key_share = time.monotonic() await self.crypto_store.put_account(self.account) self.log.debug(f"Shared keys and saved account, new keys: {resp}") diff --git a/mautrix/crypto/signature.py b/mautrix/crypto/signature.py index ca5220f8..6dc13e65 100644 --- a/mautrix/crypto/signature.py +++ b/mautrix/crypto/signature.py @@ -7,9 +7,19 @@ import functools import json +import olm import unpaddedbase64 -from mautrix.types import DeviceID, EncryptionKeyAlgorithm, KeyID, SigningKey, UserID +from mautrix.types import ( + JSON, + DeviceID, + EncryptionKeyAlgorithm, + KeyID, + Serializable, + Signature, + SigningKey, + UserID, +) try: from Crypto.PublicKey import ECC @@ -28,6 +38,14 @@ class SignedObject(TypedDict): unsigned: Any +def sign_olm(data: dict[str, JSON] | Serializable, key: olm.PkSigning | olm.Account) -> Signature: + if isinstance(data, Serializable): + data = data.serialize() + data.pop("signatures", None) + data.pop("unsigned", None) + return Signature(key.sign(canonical_json(data))) + + def verify_signature_json( data: "SignedObject", user_id: UserID, key_name: DeviceID | str, key: SigningKey ) -> bool: diff --git a/mautrix/types/event/encrypted.py b/mautrix/types/event/encrypted.py index cd08fc94..735e481a 100644 --- a/mautrix/types/event/encrypted.py +++ b/mautrix/types/event/encrypted.py @@ -9,7 +9,7 @@ from attr import dataclass -from ..primitive import JSON, DeviceID, IdentityKey, SessionID +from ..primitive import JSON, DeviceID, IdentityKey, SessionID, SigningKey from ..util import ExtensibleEnum, Obj, Serializable, SerializableAttrs, deserializer, field from .base import BaseRoomEvent, BaseUnsigned from .message import RelatesTo @@ -43,6 +43,18 @@ def deserialize(cls, raw: JSON) -> "KeyID": def __str__(self) -> str: return f"{self.algorithm.value}:{self.key_id}" + @classmethod + def ed25519(cls, key_id: SigningKey | DeviceID) -> "KeyID": + return cls(EncryptionKeyAlgorithm.ED25519, key_id) + + @classmethod + def curve25519(cls, key_id: IdentityKey) -> "KeyID": + return cls(EncryptionKeyAlgorithm.CURVE25519, key_id) + + @classmethod + def signed_curve25519(cls, key_id: IdentityKey) -> "KeyID": + return cls(EncryptionKeyAlgorithm.SIGNED_CURVE25519, key_id) + class OlmMsgType(Serializable, IntEnum): PREKEY = 0 From e496c2f5a2bd74458758c1f214101364c9483f64 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 15 Sep 2025 02:22:52 +0300 Subject: [PATCH 426/456] Add utilities for generating and using recovery keys --- mautrix/client/api/modules/crypto.py | 39 +++++++ mautrix/crypto/cross_signing.py | 169 +++++++++++++++++++++++++++ mautrix/crypto/cross_signing_key.py | 52 +++++++++ mautrix/crypto/device_lists.py | 18 +++ mautrix/crypto/machine.py | 6 + mautrix/types/crypto.py | 6 +- 6 files changed, 287 insertions(+), 3 deletions(-) create mode 100644 mautrix/crypto/cross_signing.py create mode 100644 mautrix/crypto/cross_signing_key.py diff --git a/mautrix/client/api/modules/crypto.py b/mautrix/client/api/modules/crypto.py index 065bffda..2d879a63 100644 --- a/mautrix/client/api/modules/crypto.py +++ b/mautrix/client/api/modules/crypto.py @@ -12,6 +12,8 @@ from mautrix.types import ( JSON, ClaimKeysResponse, + CrossSigningKeys, + CrossSigningUsage, DeviceID, DeviceKeys, EncryptionKeyAlgorithm, @@ -122,6 +124,43 @@ async def upload_keys( except AttributeError as e: raise MatrixResponseError("Invalid `one_time_key_counts` field in response.") from e + async def upload_cross_signing_keys( + self, + keys: dict[CrossSigningUsage, CrossSigningKeys], + auth: dict[str, JSON] | None = None, + ) -> None: + await self.api.request( + Method.POST, + Path.v3.keys.device_signing.upload, + {f"{usage}_key": key.serialize() for usage, key in keys.items()} + | ({"auth": auth} if auth else {}), + ) + + async def upload_one_signature( + self, + user_id: UserID, + device_id: DeviceID, + keys: Union[DeviceKeys, CrossSigningKeys], + ) -> None: + await self.api.request( + Method.POST, Path.v3.keys.signatures.upload, {user_id: {device_id: keys.serialize()}} + ) + # TODO check failures + + async def upload_many_signatures( + self, + signatures: dict[UserID, dict[DeviceID, Union[DeviceKeys, CrossSigningKeys]]], + ) -> None: + await self.api.request( + Method.POST, + Path.v3.keys.signatures.upload, + { + user_id: {device_id: keys.serialize() for device_id, keys in devices.items()} + for user_id, devices in signatures.items() + }, + ) + # TODO check failures + async def query_keys( self, device_keys: list[UserID] | set[UserID] | dict[UserID, list[DeviceID]], diff --git a/mautrix/crypto/cross_signing.py b/mautrix/crypto/cross_signing.py new file mode 100644 index 00000000..b768bc6f --- /dev/null +++ b/mautrix/crypto/cross_signing.py @@ -0,0 +1,169 @@ +# Copyright (c) 2025 Tulir Asokan +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +from ..types import ( + JSON, + CrossSigner, + CrossSigningKeys, + CrossSigningUsage, + DeviceIdentity, + EventType, + KeyID, + UserID, +) +from .cross_signing_key import CrossSigningPrivateKeys, CrossSigningPublicKeys, CrossSigningSeeds +from .device_lists import DeviceListMachine +from .signature import sign_olm +from .ssss import Key as SSSSKey + + +class CrossSigningMachine(DeviceListMachine): + _cross_signing_public_keys: CrossSigningPublicKeys | None + _cross_signing_public_keys_fetched: bool + _cross_signing_private_keys: CrossSigningPrivateKeys | None + + async def verify_with_recovery_key(self, recovery_key: str) -> None: + key_id, key_data = await self.ssss.get_default_key_data() + ssss_key = key_data.verify_recovery_key(key_id, recovery_key) + seeds = await self._fetch_cross_signing_keys_from_ssss(ssss_key) + self._import_cross_signing_keys(seeds) + await self.sign_own_device(self.own_identity) + + def _import_cross_signing_keys(self, seeds: CrossSigningSeeds) -> None: + self._cross_signing_private_keys = seeds.to_keys() + self._cross_signing_public_keys = self._cross_signing_private_keys.public_keys + + async def generate_recovery_key( + self, passphrase: str | None = None, seeds: CrossSigningSeeds | None = None + ) -> str: + seeds = seeds or CrossSigningSeeds.generate() + ssss_key = await self.ssss.generate_and_upload_key(passphrase) + await self._upload_cross_signing_keys_to_ssss(ssss_key, seeds) + await self._publish_cross_signing_keys(seeds.to_keys()) + await self.ssss.set_default_key_id(ssss_key.id) + await self.sign_own_device(self.own_identity) + return ssss_key.recovery_key + + async def _fetch_cross_signing_keys_from_ssss(self, key: SSSSKey) -> CrossSigningSeeds: + return CrossSigningSeeds( + master_key=await self.ssss.get_decrypted_account_data( + EventType.CROSS_SIGNING_MASTER, key + ), + user_signing_key=await self.ssss.get_decrypted_account_data( + EventType.CROSS_SIGNING_USER_SIGNING, key + ), + self_signing_key=await self.ssss.get_decrypted_account_data( + EventType.CROSS_SIGNING_SELF_SIGNING, key + ), + ) + + async def _upload_cross_signing_keys_to_ssss( + self, key: SSSSKey, seeds: CrossSigningSeeds + ) -> None: + await self.ssss.set_encrypted_account_data( + EventType.CROSS_SIGNING_MASTER, seeds.master_key, key + ) + await self.ssss.set_encrypted_account_data( + EventType.CROSS_SIGNING_USER_SIGNING, seeds.user_signing_key, key + ) + await self.ssss.set_encrypted_account_data( + EventType.CROSS_SIGNING_SELF_SIGNING, seeds.self_signing_key, key + ) + + async def get_own_cross_signing_public_keys(self) -> CrossSigningPublicKeys | None: + if self._cross_signing_public_keys or self._cross_signing_public_keys_fetched: + return self._cross_signing_public_keys + keys = await self.get_cross_signing_public_keys(self.client.mxid) + self._cross_signing_public_keys_fetched = True + if keys: + self._cross_signing_public_keys = keys + return keys + + async def get_cross_signing_public_keys( + self, user_id: UserID + ) -> CrossSigningPublicKeys | None: + db_keys = await self.crypto_store.get_cross_signing_keys(user_id) + if CrossSigningUsage.MASTER not in db_keys: + await self._fetch_keys([user_id], include_untracked=True) + db_keys = await self.crypto_store.get_cross_signing_keys(user_id) + if CrossSigningUsage.MASTER not in db_keys: + return None + return CrossSigningPublicKeys( + master_key=db_keys[CrossSigningUsage.MASTER].key, + self_signing_key=( + db_keys[CrossSigningUsage.SELF].key if CrossSigningUsage.SELF in db_keys else None + ), + user_signing_key=( + db_keys[CrossSigningUsage.USER].key if CrossSigningUsage.USER in db_keys else None + ), + ) + + async def sign_own_device(self, device: DeviceIdentity) -> None: + full_keys = await self._get_full_device_keys(device) + ssk = self._cross_signing_private_keys.self_signing_key + signature = sign_olm(full_keys, ssk) + full_keys.signatures = {self.client.mxid: {KeyID.ed25519(ssk.public_key): signature}} + await self.client.upload_one_signature(device.user_id, device.device_id, full_keys) + await self.crypto_store.put_signature( + CrossSigner(device.user_id, device.signing_key), + CrossSigner(self.client.mxid, ssk.public_key), + signature, + ) + + async def _publish_cross_signing_keys( + self, + keys: CrossSigningPrivateKeys, + auth: dict[str, JSON] | None = None, + ) -> None: + public = keys.public_keys + master_key = CrossSigningKeys( + user_id=self.client.mxid, + usage=[CrossSigningUsage.MASTER], + keys={KeyID.ed25519(public.master_key): public.master_key}, + ) + master_key.signatures = { + self.client.mxid: { + KeyID.ed25519(self.client.device_id): sign_olm(master_key, self.account), + } + } + self_key = CrossSigningKeys( + user_id=self.client.mxid, + usage=[CrossSigningUsage.SELF], + keys={KeyID.ed25519(public.self_signing_key): public.self_signing_key}, + ) + self_key.signatures = { + self.client.mxid: { + KeyID.ed25519(public.master_key): sign_olm(self_key, keys.master_key), + } + } + user_key = CrossSigningKeys( + user_id=self.client.mxid, + usage=[CrossSigningUsage.USER], + keys={KeyID.ed25519(public.user_signing_key): public.user_signing_key}, + ) + user_key.signatures = { + self.client.mxid: { + KeyID.ed25519(public.master_key): sign_olm(user_key, keys.master_key), + } + } + await self.client.upload_cross_signing_keys( + keys={ + CrossSigningUsage.MASTER: master_key, + CrossSigningUsage.SELF: self_key, + CrossSigningUsage.USER: user_key, + }, + auth=auth, + ) + await self.crypto_store.put_cross_signing_key( + self.client.mxid, CrossSigningUsage.MASTER, public.master_key + ) + await self.crypto_store.put_cross_signing_key( + self.client.mxid, CrossSigningUsage.SELF, public.self_signing_key + ) + await self.crypto_store.put_cross_signing_key( + self.client.mxid, CrossSigningUsage.USER, public.user_signing_key + ) + self._cross_signing_private_keys = keys + self._cross_signing_public_keys = public diff --git a/mautrix/crypto/cross_signing_key.py b/mautrix/crypto/cross_signing_key.py new file mode 100644 index 00000000..f4e3c1c1 --- /dev/null +++ b/mautrix/crypto/cross_signing_key.py @@ -0,0 +1,52 @@ +# Copyright (c) 2025 Tulir Asokan +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +from typing import NamedTuple + +import olm + +from mautrix.crypto.ssss.util import cryptorand +from mautrix.types import SigningKey + + +class CrossSigningPublicKeys(NamedTuple): + master_key: SigningKey + self_signing_key: SigningKey + user_signing_key: SigningKey + + +class CrossSigningPrivateKeys(NamedTuple): + master_key: olm.PkSigning + self_signing_key: olm.PkSigning + user_signing_key: olm.PkSigning + + @property + def public_keys(self) -> CrossSigningPublicKeys: + return CrossSigningPublicKeys( + master_key=self.master_key.public_key, + self_signing_key=self.self_signing_key.public_key, + user_signing_key=self.user_signing_key.public_key, + ) + + +class CrossSigningSeeds(NamedTuple): + master_key: bytes + self_signing_key: bytes + user_signing_key: bytes + + def to_keys(self) -> CrossSigningPrivateKeys: + return CrossSigningPrivateKeys( + master_key=olm.PkSigning(self.master_key), + self_signing_key=olm.PkSigning(self.self_signing_key), + user_signing_key=olm.PkSigning(self.user_signing_key), + ) + + @classmethod + def generate(cls) -> "CrossSigningSeeds": + return cls( + master_key=cryptorand.read(32), + self_signing_key=cryptorand.read(32), + user_signing_key=cryptorand.read(32), + ) diff --git a/mautrix/crypto/device_lists.py b/mautrix/crypto/device_lists.py index 93648889..96b4a64a 100644 --- a/mautrix/crypto/device_lists.py +++ b/mautrix/crypto/device_lists.py @@ -28,6 +28,18 @@ class DeviceListMachine(BaseOlmMachine): + @property + def own_identity(self) -> DeviceIdentity: + return DeviceIdentity( + user_id=self.client.mxid, + device_id=self.client.device_id, + identity_key=self.account.identity_key, + signing_key=self.account.signing_key, + trust=TrustState.VERIFIED, + deleted=False, + name="", + ) + async def _fetch_keys( self, users: list[UserID], since: SyncToken = "", include_untracked: bool = False ) -> dict[UserID, dict[DeviceID, DeviceIdentity]]: @@ -220,6 +232,12 @@ async def _store_cross_signing_keys(self, resp: QueryKeysResponse, user_id: User else: self.log.warning(f"Invalid signature from {signing_key_log} for {key_id}") + async def _get_full_device_keys(self, device: DeviceIdentity) -> DeviceKeys: + resp = await self.client.query_keys({device.user_id: [device.device_id]}) + keys = resp.device_keys[device.user_id][device.device_id] + await self._validate_device(device.user_id, device.device_id, keys, device) + return keys + async def get_or_fetch_device( self, user_id: UserID, device_id: DeviceID ) -> DeviceIdentity | None: diff --git a/mautrix/crypto/machine.py b/mautrix/crypto/machine.py index 9088662d..7585776d 100644 --- a/mautrix/crypto/machine.py +++ b/mautrix/crypto/machine.py @@ -32,6 +32,7 @@ from mautrix.util.logging import TraceLogger from .account import OlmAccount +from .cross_signing import CrossSigningMachine from .decrypt_megolm import MegolmDecryptionMachine from .encrypt_megolm import MegolmEncryptionMachine from .key_request import KeyRequestingMachine @@ -47,6 +48,7 @@ class OlmMachine( OlmUnwedgingMachine, KeySharingMachine, KeyRequestingMachine, + CrossSigningMachine, ): """ OlmMachine is the main class for handling things related to Matrix end-to-end encryption with @@ -99,6 +101,10 @@ def __init__( self._prev_unwedge = {} self._cs_fetch_attempted = set() + self._cross_signing_public_keys = None + self._cross_signing_public_keys_fetched = False + self._cross_signing_private_keys = None + self.client.add_event_handler( cli.InternalEventType.DEVICE_OTK_COUNT, self.handle_otk_count, wait_sync=True ) diff --git a/mautrix/types/crypto.py b/mautrix/types/crypto.py index 3bf7e96b..fe4ab742 100644 --- a/mautrix/types/crypto.py +++ b/mautrix/types/crypto.py @@ -47,9 +47,9 @@ def curve25519(self) -> Optional[IdentityKey]: class CrossSigningUsage(ExtensibleEnum): - MASTER = "master" - SELF = "self_signing" - USER = "user_signing" + MASTER: "CrossSigningUsage" = "master" + SELF: "CrossSigningUsage" = "self_signing" + USER: "CrossSigningUsage" = "user_signing" @dataclass From eccb1c5943d360185a255b69337f382940963e90 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 15 Sep 2025 03:07:46 +0300 Subject: [PATCH 427/456] Adjust cross-signing things --- mautrix/crypto/cross_signing.py | 8 ++++++-- mautrix/crypto/device_lists.py | 18 ++++++++++++------ 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/mautrix/crypto/cross_signing.py b/mautrix/crypto/cross_signing.py index b768bc6f..e3e3ead1 100644 --- a/mautrix/crypto/cross_signing.py +++ b/mautrix/crypto/cross_signing.py @@ -85,8 +85,12 @@ async def get_cross_signing_public_keys( self, user_id: UserID ) -> CrossSigningPublicKeys | None: db_keys = await self.crypto_store.get_cross_signing_keys(user_id) - if CrossSigningUsage.MASTER not in db_keys: - await self._fetch_keys([user_id], include_untracked=True) + if CrossSigningUsage.MASTER not in db_keys and user_id not in self._cs_fetch_attempted: + self.log.debug(f"Didn't find any cross-signing keys for {user_id}, fetching...") + async with self._fetch_keys_lock: + if user_id not in self._cs_fetch_attempted: + self._cs_fetch_attempted.add(user_id) + await self._fetch_keys([user_id], include_untracked=True) db_keys = await self.crypto_store.get_cross_signing_keys(user_id) if CrossSigningUsage.MASTER not in db_keys: return None diff --git a/mautrix/crypto/device_lists.py b/mautrix/crypto/device_lists.py index 96b4a64a..e2ff411e 100644 --- a/mautrix/crypto/device_lists.py +++ b/mautrix/crypto/device_lists.py @@ -315,18 +315,23 @@ async def _validate_device( deleted=False, ) - async def resolve_trust(self, device: DeviceIdentity) -> TrustState: + async def resolve_trust(self, device: DeviceIdentity, allow_fetch: bool = True) -> TrustState: try: - return await self._try_resolve_trust(device) + return await self._try_resolve_trust(device, allow_fetch) except Exception: self.log.exception(f"Failed to resolve trust of {device.user_id}/{device.device_id}") return TrustState.UNVERIFIED - async def _try_resolve_trust(self, device: DeviceIdentity) -> TrustState: - if device.trust in (TrustState.VERIFIED, TrustState.BLACKLISTED): + async def _try_resolve_trust( + self, device: DeviceIdentity, allow_fetch: bool = True + ) -> TrustState: + if device.device_id != self.client.device_id and device.trust in ( + TrustState.VERIFIED, + TrustState.BLACKLISTED, + ): return device.trust their_keys = await self.crypto_store.get_cross_signing_keys(device.user_id) - if len(their_keys) == 0 and device.user_id not in self._cs_fetch_attempted: + if len(their_keys) == 0 and allow_fetch and device.user_id not in self._cs_fetch_attempted: self.log.debug(f"Didn't find any cross-signing keys for {device.user_id}, fetching...") async with self._fetch_keys_lock: if device.user_id not in self._cs_fetch_attempted: @@ -337,7 +342,8 @@ async def _try_resolve_trust(self, device: DeviceIdentity) -> TrustState: msk = their_keys[CrossSigningUsage.MASTER] ssk = their_keys[CrossSigningUsage.SELF] except KeyError as e: - self.log.error(f"Didn't find cross-signing key {e.args[0]} of {device.user_id}") + if allow_fetch: + self.log.error(f"Didn't find cross-signing key {e.args[0]} of {device.user_id}") return TrustState.UNVERIFIED ssk_signed = await self.crypto_store.is_key_signed_by( target=CrossSigner(device.user_id, ssk.key), From afc780408d6e4d3a06577467bf0a0a5e603c4437 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 15 Sep 2025 03:13:31 +0300 Subject: [PATCH 428/456] Bump version to 0.21.0b1 --- CHANGELOG.md | 4 +++- mautrix/__init__.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6bbb268b..7e663680 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,8 @@ -## v0.20.9 (unreleased) +## v0.21.0 (unreleased) * *(event)* Added support for creator power in room v12+. +* *(crypto)* Added support for generating and using recovery keys for verifying + the active device. * *(bridge)* Removed check for login flows when using MSC4190 (thanks to [@meson800] in [#178]). diff --git a/mautrix/__init__.py b/mautrix/__init__.py index beb96400..8a95f01e 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.20.9rc3" +__version__ = "0.21.0b1" __author__ = "Tulir Asokan " __all__ = [ "api", From 9cb168138eafc9c2047df1edf1e02cf0c6f034de Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 15 Sep 2025 03:16:03 +0300 Subject: [PATCH 429/456] Update doc builder requirements --- docs/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/requirements.txt b/docs/requirements.txt index 798e0f1b..4b0fdff7 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -17,3 +17,4 @@ prometheus_client python-olm unpaddedbase64 pycryptodome +base58 From 8a201445fcc3b0b3d3b332e61e00e468994025c3 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 15 Sep 2025 03:18:49 +0300 Subject: [PATCH 430/456] Fix sqlalchemy version --- docs/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 4b0fdff7..7269d3a5 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -9,7 +9,7 @@ yarl # that aren't used for anything that's in the docs python-magic ruamel.yaml -SQLAlchemy +SQLAlchemy<2 commonmark asyncpg aiosqlite From 7aadbc53ae254179d88472dc7d06275bc969b9ea Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 15 Sep 2025 03:19:36 +0300 Subject: [PATCH 431/456] Update test requirements --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d5c58eb0..20b51098 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from mautrix import __version__ -encryption_dependencies = ["python-olm", "unpaddedbase64", "pycryptodome"] +encryption_dependencies = ["python-olm", "unpaddedbase64", "pycryptodome", "base58"] test_dependencies = ["aiosqlite", "asyncpg", "ruamel.yaml", *encryption_dependencies] setuptools.setup( From bcb597733f0421fdbf0808389a857880aa29a10c Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 15 Sep 2025 03:25:16 +0300 Subject: [PATCH 432/456] Fix Python 3.10 compatibility --- mautrix/crypto/ssss/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/crypto/ssss/util.py b/mautrix/crypto/ssss/util.py index bc7dfb81..b58c941a 100644 --- a/mautrix/crypto/ssss/util.py +++ b/mautrix/crypto/ssss/util.py @@ -67,7 +67,7 @@ def prepare_aes(key: bytes, iv: str | bytes) -> AES: iv = unpaddedbase64.decode_base64(iv) # initial_value = struct.unpack(">Q", iv[8:])[0] # counter = Counter.new(64, prefix=iv[:8], initial_value=initial_value) - counter = Counter.new(128, initial_value=int.from_bytes(iv)) + counter = Counter.new(128, initial_value=int.from_bytes(iv, byteorder="big")) return AES.new(key=key, mode=AES.MODE_CTR, counter=counter) From 481cb8330739e93df1dfc0b239bb3bbc875ab149 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 15 Sep 2025 03:26:58 +0300 Subject: [PATCH 433/456] Bump version to 0.21.0b2 --- mautrix/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 8a95f01e..15596f17 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.21.0b1" +__version__ = "0.21.0b2" __author__ = "Tulir Asokan " __all__ = [ "api", From 03258b1c511722aed6a74649a0fb1d56e8ebe461 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 15 Sep 2025 15:18:43 +0300 Subject: [PATCH 434/456] Throw error if recovery key is malformed --- mautrix/__init__.py | 2 +- mautrix/crypto/ssss/key.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 15596f17..9f500b4c 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.21.0b2" +__version__ = "0.21.0b3" __author__ = "Tulir Asokan " __all__ = [ "api", diff --git a/mautrix/crypto/ssss/key.py b/mautrix/crypto/ssss/key.py index 0a159b3e..691ded71 100644 --- a/mautrix/crypto/ssss/key.py +++ b/mautrix/crypto/ssss/key.py @@ -66,7 +66,10 @@ def verify_passphrase(self, key_id: str, phrase: str) -> "Key": return self.verify_raw_key(key_id, self.passphrase.get_key(phrase)) def verify_recovery_key(self, key_id: str, recovery_key: str) -> "Key": - return self.verify_raw_key(key_id, decode_base58_recovery_key(recovery_key)) + decoded_key = decode_base58_recovery_key(recovery_key) + if not decoded_key: + raise ValueError("Invalid recovery key syntax") + return self.verify_raw_key(key_id, decoded_key) def verify_raw_key(self, key_id: str, key: bytes) -> "Key": if self.mac.rstrip("=") != calculate_hash(key, self.iv): From c4028be095c092623068819f174671f40ccb8a1a Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 15 Sep 2025 15:24:50 +0300 Subject: [PATCH 435/456] Don't allow verifying before sharing account keys --- mautrix/crypto/cross_signing.py | 4 ++++ mautrix/crypto/device_lists.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/mautrix/crypto/cross_signing.py b/mautrix/crypto/cross_signing.py index e3e3ead1..7577cd5d 100644 --- a/mautrix/crypto/cross_signing.py +++ b/mautrix/crypto/cross_signing.py @@ -25,6 +25,8 @@ class CrossSigningMachine(DeviceListMachine): _cross_signing_private_keys: CrossSigningPrivateKeys | None async def verify_with_recovery_key(self, recovery_key: str) -> None: + if not self.account.shared: + raise ValueError("Device keys must be shared before verifying with recovery key") key_id, key_data = await self.ssss.get_default_key_data() ssss_key = key_data.verify_recovery_key(key_id, recovery_key) seeds = await self._fetch_cross_signing_keys_from_ssss(ssss_key) @@ -38,6 +40,8 @@ def _import_cross_signing_keys(self, seeds: CrossSigningSeeds) -> None: async def generate_recovery_key( self, passphrase: str | None = None, seeds: CrossSigningSeeds | None = None ) -> str: + if not self.account.shared: + raise ValueError("Device keys must be shared before generating recovery key") seeds = seeds or CrossSigningSeeds.generate() ssss_key = await self.ssss.generate_and_upload_key(passphrase) await self._upload_cross_signing_keys_to_ssss(ssss_key, seeds) diff --git a/mautrix/crypto/device_lists.py b/mautrix/crypto/device_lists.py index e2ff411e..4314501a 100644 --- a/mautrix/crypto/device_lists.py +++ b/mautrix/crypto/device_lists.py @@ -206,7 +206,7 @@ async def _store_cross_signing_keys(self, resp: QueryKeysResponse, user_id: User signing_key = device.ed25519 except KeyError: pass - if len(signing_key) != 43: + if not signing_key or len(signing_key) != 43: self.log.debug( f"Cross-signing key {user_id}/{actual_key} has a signature from " f"an unknown key {key_id}" From f5492167d9b832390a9869a9c98f65d2949e7071 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 15 Sep 2025 15:26:01 +0300 Subject: [PATCH 436/456] Fix sharing device keys --- mautrix/crypto/account.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/crypto/account.py b/mautrix/crypto/account.py index 63f83f93..a00ada71 100644 --- a/mautrix/crypto/account.py +++ b/mautrix/crypto/account.py @@ -82,7 +82,7 @@ def get_device_keys(self, user_id: UserID, device_id: DeviceID) -> DeviceKeys: device_id=device_id, algorithms=[EncryptionAlgorithm.OLM_V1, EncryptionAlgorithm.MEGOLM_V1], keys={ - KeyID(algorithm=EncryptionKeyAlgorithm(algorithm), key_id=key): key + KeyID(algorithm=EncryptionKeyAlgorithm(algorithm), key_id=device_id): key for algorithm, key in self.identity_keys.items() }, signatures={}, From 96b5f20b44ef1258741f8ba193021d56066c1fc0 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 15 Sep 2025 15:26:41 +0300 Subject: [PATCH 437/456] Bump version to 0.21.0b4 --- mautrix/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 9f500b4c..41283ebb 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.21.0b3" +__version__ = "0.21.0b4" __author__ = "Tulir Asokan " __all__ = [ "api", From d122a7a9ba3be32c5fb9d782fea6e509a3952317 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 23 Sep 2025 23:33:28 +0300 Subject: [PATCH 438/456] Fix CI image --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8f3efe97..b0d7ab3f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,6 +1,6 @@ build docs builder: stage: build - image: docker:stable + image: docker:latest tags: - amd64 only: From 8adeea8c576f9db03aab8dd00619e99fc1a06ea0 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 23 Sep 2025 23:33:50 +0300 Subject: [PATCH 439/456] Store acquired connection in context variable --- mautrix/util/async_db/aiosqlite.py | 2 +- mautrix/util/async_db/asyncpg.py | 2 +- mautrix/util/async_db/database.py | 19 ++++++++++++++++++- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/mautrix/util/async_db/aiosqlite.py b/mautrix/util/async_db/aiosqlite.py index 80fa3cf8..4a2ff826 100644 --- a/mautrix/util/async_db/aiosqlite.py +++ b/mautrix/util/async_db/aiosqlite.py @@ -181,7 +181,7 @@ async def stop(self) -> None: self._conns -= 1 await conn.close() - def acquire(self) -> AsyncContextManager[LoggingConnection]: + def acquire_direct(self) -> AsyncContextManager[LoggingConnection]: if self._parent: return self._parent.acquire() return self._acquire() diff --git a/mautrix/util/async_db/asyncpg.py b/mautrix/util/async_db/asyncpg.py index 07ef7ad7..97b49f6c 100644 --- a/mautrix/util/async_db/asyncpg.py +++ b/mautrix/util/async_db/asyncpg.py @@ -96,7 +96,7 @@ async def _handle_exception(self, err: Exception) -> None: sys.exit(26) @asynccontextmanager - async def acquire(self) -> LoggingConnection: + async def acquire_direct(self) -> LoggingConnection: async with self.pool.acquire() as conn: yield LoggingConnection( self.scheme, conn, self.log, handle_exception=self._handle_exception diff --git a/mautrix/util/async_db/database.py b/mautrix/util/async_db/database.py index 0f23b02d..ff73a8b2 100644 --- a/mautrix/util/async_db/database.py +++ b/mautrix/util/async_db/database.py @@ -7,6 +7,8 @@ from typing import Any, AsyncContextManager, Type from abc import ABC, abstractmethod +from contextlib import asynccontextmanager +from contextvars import ContextVar import logging from yarl import URL @@ -23,6 +25,8 @@ from aiosqlite import Cursor from asyncpg import Record +conn_var: ContextVar[LoggingConnection | None] = ContextVar("db_connection", default=None) + class Database(ABC): schemes: dict[str, Type[Database]] = {} @@ -128,9 +132,22 @@ async def stop(self) -> None: pass @abstractmethod - def acquire(self) -> AsyncContextManager[LoggingConnection]: + def acquire_direct(self) -> AsyncContextManager[LoggingConnection]: pass + @asynccontextmanager + async def acquire(self) -> LoggingConnection: + conn = conn_var.get(None) + if conn is not None: + yield conn + return + async with self.acquire_direct() as conn: + token = conn_var.set(conn) + try: + yield conn + finally: + conn_var.reset(token) + async def execute(self, query: str, *args: Any, timeout: float | None = None) -> str | Cursor: async with self.acquire() as conn: return await conn.execute(query, *args, timeout=timeout) From d675cf3f9729b20bc024737b0bd1285cd847dd6e Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 23 Sep 2025 23:50:01 +0300 Subject: [PATCH 440/456] Make nested transactions no-ops on SQLite --- mautrix/util/async_db/aiosqlite.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/mautrix/util/async_db/aiosqlite.py b/mautrix/util/async_db/aiosqlite.py index 4a2ff826..934379a8 100644 --- a/mautrix/util/async_db/aiosqlite.py +++ b/mautrix/util/async_db/aiosqlite.py @@ -7,6 +7,7 @@ from typing import Any, AsyncContextManager from contextlib import asynccontextmanager +from contextvars import ContextVar import asyncio import logging import os @@ -24,6 +25,9 @@ POSITIONAL_PARAM_PATTERN = re.compile(r"\$(\d+)") +in_transaction = ContextVar("in_transaction", default=False) + + class TxnConnection(aiosqlite.Connection): def __init__(self, path: str, **kwargs) -> None: def connector() -> sqlite3.Connection: @@ -35,7 +39,11 @@ def connector() -> sqlite3.Connection: @asynccontextmanager async def transaction(self) -> None: + if in_transaction.get(): + yield + return await self.execute("BEGIN TRANSACTION") + token = in_transaction.set(True) try: yield except Exception: @@ -43,6 +51,8 @@ async def transaction(self) -> None: raise else: await self.commit() + finally: + in_transaction.reset(token) def __execute(self, query: str, *args: Any): query = POSITIONAL_PARAM_PATTERN.sub(r"?\1", query) From 513e9256f4c11c2b24a648bab79c244835f5cbf7 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 23 Sep 2025 23:56:01 +0300 Subject: [PATCH 441/456] Use transaction for entire fetch keys step --- mautrix/crypto/device_lists.py | 106 ++++++++++++++------------ mautrix/crypto/store/abstract.py | 6 +- mautrix/crypto/store/asyncpg/store.py | 6 ++ 3 files changed, 67 insertions(+), 51 deletions(-) diff --git a/mautrix/crypto/device_lists.py b/mautrix/crypto/device_lists.py index 4314501a..b00b104c 100644 --- a/mautrix/crypto/device_lists.py +++ b/mautrix/crypto/device_lists.py @@ -59,61 +59,67 @@ async def _fetch_keys( data = {} for user_id, devices in resp.device_keys.items(): missing_users.remove(user_id) - - new_devices = {} - existing_devices = (await self.crypto_store.get_devices(user_id)) or {} - - self.log.trace( - f"Updating devices for {user_id}, got {len(devices)}, " - f"have {len(existing_devices)} in store" - ) - changed = False - ssks = resp.self_signing_keys.get(user_id) - ssk = ssks.first_ed25519_key if ssks else None - for device_id, device_keys in devices.items(): - try: - existing = existing_devices[device_id] - except KeyError: - existing = None - changed = True - self.log.trace(f"Validating device {device_keys} of {user_id}") - try: - new_device = await self._validate_device( - user_id, device_id, device_keys, existing - ) - except DeviceValidationError as e: - self.log.warning(f"Failed to validate device {device_id} of {user_id}: {e}") - else: - if new_device: - new_devices[device_id] = new_device - await self._store_device_self_signatures(device_keys, ssk) - self.log.debug( - f"Storing new device list for {user_id} containing {len(new_devices)} devices" - ) - await self.crypto_store.put_devices(user_id, new_devices) - data[user_id] = new_devices - - if changed or len(new_devices) != len(existing_devices): - if self.delete_keys_on_device_delete: - for device_id in existing_devices.keys() - new_devices.keys(): - device = existing_devices[device_id] - removed_ids = await self.crypto_store.redact_group_sessions( - room_id=None, sender_key=device.identity_key, reason="device removed" - ) - self.log.info( - "Redacted megolm sessions sent by removed device " - f"{device.user_id}/{device.device_id}: {removed_ids}" - ) - await self.on_devices_changed(user_id) + async with self.crypto_store.transaction(): + data[user_id] = await self._process_fetched_keys(user_id, devices, resp) for user_id in missing_users: self.log.warning(f"Didn't get any devices for user {user_id}") - for user_id in users: - await self._store_cross_signing_keys(resp, user_id) - return data + async def _process_fetched_keys( + self, + user_id: UserID, + devices: dict[DeviceID, DeviceKeys], + resp: QueryKeysResponse, + ) -> dict[DeviceID, DeviceIdentity]: + new_devices = {} + existing_devices = (await self.crypto_store.get_devices(user_id)) or {} + + self.log.trace( + f"Updating devices for {user_id}, got {len(devices)}, " + f"have {len(existing_devices)} in store" + ) + changed = False + ssks = resp.self_signing_keys.get(user_id) + ssk = ssks.first_ed25519_key if ssks else None + for device_id, device_keys in devices.items(): + try: + existing = existing_devices[device_id] + except KeyError: + existing = None + changed = True + self.log.trace(f"Validating device {device_keys} of {user_id}") + try: + new_device = await self._validate_device(user_id, device_id, device_keys, existing) + except DeviceValidationError as e: + self.log.warning(f"Failed to validate device {device_id} of {user_id}: {e}") + else: + if new_device: + new_devices[device_id] = new_device + await self._store_device_self_signatures(device_keys, ssk) + self.log.debug( + f"Storing new device list for {user_id} containing {len(new_devices)} devices" + ) + await self.crypto_store.put_devices(user_id, new_devices) + + if changed or len(new_devices) != len(existing_devices): + if self.delete_keys_on_device_delete: + for device_id in existing_devices.keys() - new_devices.keys(): + device = existing_devices[device_id] + removed_ids = await self.crypto_store.redact_group_sessions( + room_id=None, sender_key=device.identity_key, reason="device removed" + ) + self.log.info( + "Redacted megolm sessions sent by removed device " + f"{device.user_id}/{device.device_id}: {removed_ids}" + ) + await self.on_devices_changed(user_id) + + await self._store_cross_signing_keys(resp, user_id) + + return new_devices + async def _store_device_self_signatures( self, device_keys: DeviceKeys, self_signing_key: SigningKey | None ) -> None: @@ -343,7 +349,7 @@ async def _try_resolve_trust( ssk = their_keys[CrossSigningUsage.SELF] except KeyError as e: if allow_fetch: - self.log.error(f"Didn't find cross-signing key {e.args[0]} of {device.user_id}") + self.log.warning(f"Didn't find cross-signing key {e.args[0]} of {device.user_id}") return TrustState.UNVERIFIED ssk_signed = await self.crypto_store.is_key_signed_by( target=CrossSigner(device.user_id, ssk.key), diff --git a/mautrix/crypto/store/abstract.py b/mautrix/crypto/store/abstract.py index 787d5225..1bae832f 100644 --- a/mautrix/crypto/store/abstract.py +++ b/mautrix/crypto/store/abstract.py @@ -5,7 +5,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import annotations -from typing import NamedTuple +from typing import AsyncContextManager, NamedTuple from abc import ABC, abstractmethod from mautrix.types import ( @@ -87,6 +87,10 @@ async def close(self) -> None: async def flush(self) -> None: """Flush the store. If all the methods persist data immediately, this can be a no-op.""" + async def transaction(self) -> AsyncContextManager[None]: + """Run a database transaction. If the store doesn't support transactions, this can be a no-op.""" + pass + @abstractmethod async def delete(self) -> None: """Delete the data in the store.""" diff --git a/mautrix/crypto/store/asyncpg/store.py b/mautrix/crypto/store/asyncpg/store.py index a29f7737..bdc37ddd 100644 --- a/mautrix/crypto/store/asyncpg/store.py +++ b/mautrix/crypto/store/asyncpg/store.py @@ -6,6 +6,7 @@ from __future__ import annotations from collections import defaultdict +from contextlib import asynccontextmanager from datetime import timedelta from asyncpg import UniqueViolationError @@ -79,6 +80,11 @@ def __init__(self, account_id: str, pickle_key: str, db: Database) -> None: self._account = None self._olm_cache = defaultdict(lambda: {}) + @asynccontextmanager + async def transaction(self) -> None: + async with self.db.acquire() as conn, conn.transaction(): + yield + async def delete(self) -> None: tables = ("crypto_account", "crypto_olm_session", "crypto_megolm_outbound_session") async with self.db.acquire() as conn, conn.transaction(): From 1443f53ef58ee3dc5b50d377d31b845aee989c41 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Tue, 23 Sep 2025 23:58:50 +0300 Subject: [PATCH 442/456] Add option to self-sign bridge bot device --- mautrix/bridge/config.py | 1 + mautrix/bridge/e2ee.py | 12 ++++++++++++ 2 files changed, 13 insertions(+) diff --git a/mautrix/bridge/config.py b/mautrix/bridge/config.py index 46fb88d2..defed222 100644 --- a/mautrix/bridge/config.py +++ b/mautrix/bridge/config.py @@ -144,6 +144,7 @@ def do_update(self, helper: ConfigUpdateHelper) -> None: copy("bridge.encryption.require") copy("bridge.encryption.appservice") copy("bridge.encryption.msc4190") + copy("bridge.encryption.self_sign") copy("bridge.encryption.delete_keys.delete_outbound_on_ack") copy("bridge.encryption.delete_keys.dont_store_outbound") copy("bridge.encryption.delete_keys.ratchet_on_decrypt") diff --git a/mautrix/bridge/e2ee.py b/mautrix/bridge/e2ee.py index 56270942..1525b388 100644 --- a/mautrix/bridge/e2ee.py +++ b/mautrix/bridge/e2ee.py @@ -58,6 +58,7 @@ class EncryptionManager: periodically_delete_expired_keys: bool delete_outdated_inbound: bool msc4190: bool + self_sign: bool bridge: br.Bridge az: AppService @@ -110,6 +111,7 @@ def __init__( self.key_sharing_enabled = bridge.config["bridge.encryption.allow_key_sharing"] self.appservice_mode = bridge.config["bridge.encryption.appservice"] self.msc4190 = bridge.config["bridge.encryption.msc4190"] + self.self_sign = bridge.config["bridge.encryption.self_sign"] if self.appservice_mode: self.az.otk_handler = self.crypto.handle_as_otk_counts self.az.device_list_handler = self.crypto.handle_as_device_lists @@ -288,8 +290,18 @@ async def start(self) -> None: if not device_id: await self.crypto_store.put_device_id(self.client.device_id) self.log.debug(f"Logged in with new device ID {self.client.device_id}") + await self.crypto.share_keys() elif self.crypto.account.shared: await self._verify_keys_are_on_server() + else: + await self.crypto.share_keys() + if self.self_sign: + trust_state = await self.crypto.resolve_trust(self.crypto.own_identity) + if trust_state < TrustState.CROSS_SIGNED_UNTRUSTED: + recovery_key = await self.crypto.generate_recovery_key() + self.log.info(f"Generated recovery key and signed own device: {recovery_key}") + else: + self.log.debug(f"Own device is already verified ({trust_state})") if self.appservice_mode: self.log.info("End-to-bridge encryption support is enabled (appservice mode)") else: From 36b29aff83e479c657f161f4fb65c0cacdcc7e78 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 24 Sep 2025 00:07:23 +0300 Subject: [PATCH 443/456] Bump version to 0.21.0b5 --- mautrix/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 41283ebb..0fb5e1e6 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.21.0b4" +__version__ = "0.21.0b5" __author__ = "Tulir Asokan " __all__ = [ "api", From 9aced5042aac119e7204f9c7ac6077c1ebef5143 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 24 Sep 2025 00:20:35 +0300 Subject: [PATCH 444/456] Fix inserting database owner --- mautrix/util/async_db/database.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mautrix/util/async_db/database.py b/mautrix/util/async_db/database.py index ff73a8b2..6a17dc68 100644 --- a/mautrix/util/async_db/database.py +++ b/mautrix/util/async_db/database.py @@ -123,7 +123,9 @@ async def _check_owner(self) -> None: ) owner = await self.fetchval("SELECT owner FROM database_owner WHERE key=0") if not owner: - await self.execute("INSERT INTO database_owner (owner) VALUES ($1)", self.owner_name) + await self.execute( + "INSERT INTO database_owner (key, owner) VALUES (0, $1)", self.owner_name + ) elif owner != self.owner_name: raise DatabaseNotOwned(owner) From 1dd7d49f3cdf24c0b96c39859e412b4145b7de8d Mon Sep 17 00:00:00 2001 From: L0ric0 Date: Fri, 24 Oct 2025 13:54:11 +0200 Subject: [PATCH 445/456] Add option to change listening network interface of prometheus client (#181) --- mautrix/util/program.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mautrix/util/program.py b/mautrix/util/program.py index aa37b374..74752ca5 100644 --- a/mautrix/util/program.py +++ b/mautrix/util/program.py @@ -191,6 +191,7 @@ def start_prometheus(self) -> None: try: enabled = self.config["metrics.enabled"] listen_port = self.config["metrics.listen_port"] + hostname = self.config.get("metrics.hostname", "0.0.0.0") except KeyError: return if not enabled: @@ -200,7 +201,7 @@ def start_prometheus(self) -> None: "Metrics are enabled in config, but prometheus_client is not installed" ) return - prometheus.start_http_server(listen_port) + prometheus.start_http_server(listen_port, addr=hostname) def _run(self) -> None: signal.signal(signal.SIGINT, signal.default_int_handler) From e4ab162eb3848c7b522b715298d5ba249b130923 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 17 Nov 2025 00:23:09 +0200 Subject: [PATCH 446/456] Update variation selectors to Unicode 17 --- mautrix/util/variation_selector.json | 17 +++++++++++++++++ mautrix/util/variation_selector.py | 2 +- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/mautrix/util/variation_selector.json b/mautrix/util/variation_selector.json index 0205ce94..f27acb7b 100644 --- a/mautrix/util/variation_selector.json +++ b/mautrix/util/variation_selector.json @@ -31,9 +31,12 @@ "23CF": "⏏", "23E9": "⏩", "23EA": "⏪", + "23EB": "⏫", + "23EC": "⏬", "23ED": "⏭", "23EE": "⏮", "23EF": "⏯", + "23F0": "⏰", "23F1": "⏱", "23F2": "⏲", "23F3": "⏳", @@ -114,6 +117,7 @@ "26C4": "⛄", "26C5": "⛅", "26C8": "⛈", + "26CE": "⛎", "26CF": "⛏", "26D1": "⛑", "26D3": "⛓", @@ -132,8 +136,11 @@ "26FA": "⛺", "26FD": "⛽", "2702": "✂", + "2705": "✅", "2708": "✈", "2709": "✉", + "270A": "✊", + "270B": "✋", "270C": "✌", "270D": "✍", "270F": "✏", @@ -142,15 +149,25 @@ "2716": "✖", "271D": "✝", "2721": "✡", + "2728": "✨", "2733": "✳", "2734": "✴", "2744": "❄", "2747": "❇", + "274C": "❌", + "274E": "❎", "2753": "❓", + "2754": "❔", + "2755": "❕", "2757": "❗", "2763": "❣", "2764": "❤", + "2795": "➕", + "2796": "➖", + "2797": "➗", "27A1": "➡", + "27B0": "➰", + "27BF": "➿", "2934": "⤴", "2935": "⤵", "2B05": "⬅", diff --git a/mautrix/util/variation_selector.py b/mautrix/util/variation_selector.py index dec0554d..7ef2ad24 100644 --- a/mautrix/util/variation_selector.py +++ b/mautrix/util/variation_selector.py @@ -10,7 +10,7 @@ import aiohttp -EMOJI_VAR_URL = "https://www.unicode.org/Public/14.0.0/ucd/emoji/emoji-variation-sequences.txt" +EMOJI_VAR_URL = "https://www.unicode.org/Public/17.0.0/ucd/emoji/emoji-variation-sequences.txt" def read_data() -> dict[str, str]: From 8bfde09694b7ffa7fd61ccb2bc2031370d01ab72 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 17 Nov 2025 00:23:30 +0200 Subject: [PATCH 447/456] Update variation selector generator to use importlib.resources --- mautrix/util/variation_selector.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/mautrix/util/variation_selector.py b/mautrix/util/variation_selector.py index 7ef2ad24..e1430293 100644 --- a/mautrix/util/variation_selector.py +++ b/mautrix/util/variation_selector.py @@ -43,11 +43,12 @@ async def fetch_data() -> dict[str, str]: if __name__ == "__main__": import asyncio + import importlib.resources + import pathlib import sys - import pkg_resources - - path = pkg_resources.resource_filename("mautrix.util", "variation_selector.json") + path = importlib.resources.files("mautrix.util").joinpath("variation_selector.json") + assert isinstance(path, pathlib.Path) emojis = asyncio.run(fetch_data()) with open(path, "w") as file: json.dump(emojis, file, indent=" ", ensure_ascii=False) From a4198a6ec46a5ca049a88214b799f65e8cb7f04e Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 17 Nov 2025 00:23:50 +0200 Subject: [PATCH 448/456] Update linters, CI and python version --- .github/workflows/python-package.yml | 14 +++++++------- .pre-commit-config.yaml | 6 +++--- setup.py | 3 ++- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index aef7eb74..e6f77c9f 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -8,12 +8,12 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.10", "3.11", "3.12", "3.13"] + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} - name: Install libolm @@ -46,17 +46,17 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@v5 + - uses: actions/setup-python@v6 with: - python-version: "3.13" + python-version: "3.14" - uses: isort/isort-action@master with: sortPaths: "./mautrix" - uses: psf/black@stable with: src: "./mautrix" - version: "24.10.0" + version: "25.11.0" - name: pre-commit run: | pip install pre-commit diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 72cce792..b3bd2047 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v5.0.0 + rev: v6.0.0 hooks: - id: trailing-whitespace exclude_types: [markdown] @@ -8,13 +8,13 @@ repos: - id: check-yaml - id: check-added-large-files - repo: https://github.com/psf/black - rev: 24.10.0 + rev: 25.11.0 hooks: - id: black language_version: python3 files: ^mautrix/.*\.pyi?$ - repo: https://github.com/PyCQA/isort - rev: 5.13.2 + rev: 7.0.0 hooks: - id: isort files: ^mautrix/.*\.pyi?$ diff --git a/setup.py b/setup.py index 20b51098..23ac1cb1 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ ], extras_require={ "detect_mimetype": ["python-magic>=0.4.15,<0.5"], - "lint": ["black~=24.1", "isort"], + "lint": ["black~=25.1", "isort"], "test": ["pytest", "pytest-asyncio", *test_dependencies], "encryption": encryption_dependencies, }, @@ -46,6 +46,7 @@ "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", ], package_data={ From ada10a3f762f6a04fbccbf88554a1ff561cc332f Mon Sep 17 00:00:00 2001 From: Joe Groocock Date: Sun, 16 Nov 2025 23:49:42 +0100 Subject: [PATCH 449/456] Avoid setting avatar_url to "" (#171) Fixes an edge-case bug where calling `set_avatar_url("")` when the user doesn't have an avatar(`get_avatar_url()` returns `None`) caused a redundant PUT request to be made to nullify the already-empty avatar_url field in the user profile. This also fixes the errant `$user made no change` state events appearing in every room they are in when `set_avatar_url("")` is called. Signed-off-by: Joe Groocock --- mautrix/client/api/user_data.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/mautrix/client/api/user_data.py b/mautrix/client/api/user_data.py index 4c3d437a..9c380335 100644 --- a/mautrix/client/api/user_data.py +++ b/mautrix/client/api/user_data.py @@ -71,7 +71,7 @@ async def search_users(self, search_query: str, limit: int | None = 10) -> UserS # region 10.2 Profiles # API reference: https://matrix.org/docs/spec/client_server/r0.4.0.html#profiles - async def set_displayname(self, displayname: str, check_current: bool = True) -> None: + async def set_displayname(self, displayname: str | None, check_current: bool = True) -> None: """ Set the display name of the current user. @@ -81,7 +81,9 @@ async def set_displayname(self, displayname: str, check_current: bool = True) -> displayname: The new display name for the user. check_current: Whether or not to check if the displayname is already set. """ - if check_current and await self.get_displayname(self.mxid) == displayname: + if check_current and str_or_none(await self.get_displayname(self.mxid)) == str_or_none( + displayname + ): return await self.api.request( Method.PUT, @@ -112,7 +114,9 @@ async def get_displayname(self, user_id: UserID) -> str | None: except KeyError: return None - async def set_avatar_url(self, avatar_url: ContentURI, check_current: bool = True) -> None: + async def set_avatar_url( + self, avatar_url: ContentURI | None, check_current: bool = True + ) -> None: """ Set the avatar of the current user. @@ -122,7 +126,9 @@ async def set_avatar_url(self, avatar_url: ContentURI, check_current: bool = Tru avatar_url: The ``mxc://`` URI to the new avatar. check_current: Whether or not to check if the avatar is already set. """ - if check_current and await self.get_avatar_url(self.mxid) == avatar_url: + if check_current and str_or_none(await self.get_avatar_url(self.mxid)) == str_or_none( + avatar_url + ): return await self.api.request( Method.PUT, @@ -185,3 +191,10 @@ async def beeper_update_profile(self, custom_fields: dict[str, Any]) -> None: await self.api.request(Method.PATCH, Path.v3.profile[self.mxid], custom_fields) # endregion + + +def str_or_none(v: str | None) -> str | None: + """ + str_or_none empty string values to None + """ + return None if v == "" else v From db075d702a4ee21664e1a4098ef040159fefb9e4 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 17 Nov 2025 15:51:55 +0200 Subject: [PATCH 450/456] Bump version to 0.21.0 --- CHANGELOG.md | 7 ++++++- mautrix/__init__.py | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e663680..c7179c86 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,12 +1,17 @@ -## v0.21.0 (unreleased) +## v0.21.0 (2025-11-17) * *(event)* Added support for creator power in room v12+. * *(crypto)* Added support for generating and using recovery keys for verifying the active device. +* *(bridge)* Added config option for self-signing bot device. * *(bridge)* Removed check for login flows when using MSC4190 (thanks to [@meson800] in [#178]). +* *(client)* Changed `set_displayname` and `set_avatar_url` to avoid setting + empty strings if the value is already unset (thanks to [@frebib] in [#171]). +[@frebib]: https://github.com/frebib [@meson800]: https://github.com/meson800 +[#171]: https://github.com/mautrix/python/pull/171 [#178]: https://github.com/mautrix/python/pull/178 ## v0.20.8 (2025-06-01) diff --git a/mautrix/__init__.py b/mautrix/__init__.py index 0fb5e1e6..c94995f9 100644 --- a/mautrix/__init__.py +++ b/mautrix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.21.0b5" +__version__ = "0.21.0" __author__ = "Tulir Asokan " __all__ = [ "api", From 93dea5a78864f91de07a84c5df60b67c012429f4 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Wed, 4 Feb 2026 21:43:02 +0200 Subject: [PATCH 451/456] Dispatch unknown decrypted to-device events to event handlers --- mautrix/crypto/machine.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/mautrix/crypto/machine.py b/mautrix/crypto/machine.py index 7585776d..60c65677 100644 --- a/mautrix/crypto/machine.py +++ b/mautrix/crypto/machine.py @@ -222,6 +222,11 @@ async def handle_to_device_event(self, evt: ToDeviceEvent) -> None: passed to the OlmMachine is syncing. You shouldn't need to call this yourself unless you do syncing in some manual way. """ + if isinstance(evt, DecryptedOlmEvent): + self.log.warning( + f"Dropping unexpected nested encrypted to-device event from {evt.sender}" + ) + return self.log.trace( f"Handling encrypted to-device event from {evt.content.sender_key} ({evt.sender})" ) @@ -230,6 +235,8 @@ async def handle_to_device_event(self, evt: ToDeviceEvent) -> None: await self._receive_room_key(decrypted_evt) elif decrypted_evt.type == EventType.FORWARDED_ROOM_KEY: await self._receive_forwarded_room_key(decrypted_evt) + else: + self.client.dispatch_event(decrypted_evt, source=evt.source) async def _receive_room_key(self, evt: DecryptedOlmEvent) -> None: # TODO nio had a comment saying "handle this better" From d7cb10b3ae62534fc4f3eb38052b858fc625efc2 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 15 Feb 2026 15:40:26 +0200 Subject: [PATCH 452/456] Catch invalid invites in sync --- mautrix/client/syncer.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/mautrix/client/syncer.py b/mautrix/client/syncer.py index 8183e142..4b8661a2 100644 --- a/mautrix/client/syncer.py +++ b/mautrix/client/syncer.py @@ -369,12 +369,18 @@ def handle_sync(self, data: JSON) -> list[asyncio.Task]: events: list[dict[str, JSON]] = room_data.get("invite_state", {}).get("events", []) for raw_event in events: raw_event["room_id"] = room_id - raw_invite = next( - raw_event - for raw_event in events - if raw_event.get("type", "") == "m.room.member" - and raw_event.get("state_key", "") == self.mxid - ) + try: + raw_invite = next( + raw_event + for raw_event in events + if raw_event.get("type", "") == "m.room.member" + and raw_event.get("state_key", "") == self.mxid + ) + except StopIteration: + self.log.warning( + f"Corrupted invite section in sync: no invite event present for {room_id}" + ) + continue # These aren't required by the spec, so make sure they're set raw_invite.setdefault("event_id", None) raw_invite.setdefault("origin_server_ts", int(time.time() * 1000)) From 2296ac74a77940b2ceeaf4dbd3d82cfe3ca61731 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 13 Mar 2026 14:03:10 +0200 Subject: [PATCH 453/456] Fix join rule types Closes #182 --- mautrix/types/event/state.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/mautrix/types/event/state.py b/mautrix/types/event/state.py index ef51cff4..5ffc855f 100644 --- a/mautrix/types/event/state.py +++ b/mautrix/types/event/state.py @@ -9,7 +9,7 @@ import attr from ..primitive import JSON, ContentURI, EventID, RoomAlias, RoomID, UserID -from ..util import Obj, SerializableAttrs, SerializableEnum, deserializer, field +from ..util import ExtensibleEnum, Obj, SerializableAttrs, SerializableEnum, deserializer, field from .base import BaseRoomEvent, BaseUnsigned from .encrypted import EncryptionAlgorithm from .type import EventType, RoomType @@ -159,16 +159,15 @@ class RoomAvatarStateEventContent(SerializableAttrs): url: Optional[ContentURI] = None -class JoinRule(SerializableEnum): +class JoinRule(ExtensibleEnum): PUBLIC = "public" KNOCK = "knock" RESTRICTED = "restricted" INVITE = "invite" - PRIVATE = "private" KNOCK_RESTRICTED = "knock_restricted" -class JoinRestrictionType(SerializableEnum): +class JoinRestrictionType(ExtensibleEnum): ROOM_MEMBERSHIP = "m.room_membership" From 89ebd5fff50ca22a8c6e4f554ad9aae393c96760 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 23 Mar 2026 14:48:30 +0200 Subject: [PATCH 454/456] Fix parsing /messages response --- mautrix/client/api/events.py | 6 ++---- mautrix/types/misc.py | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/mautrix/client/api/events.py b/mautrix/client/api/events.py index fd6b2c9d..bd415b9b 100644 --- a/mautrix/client/api/events.py +++ b/mautrix/client/api/events.py @@ -395,15 +395,13 @@ async def get_messages( try: return PaginatedMessages( content["start"], - content["end"], + content.get("end"), [Event.deserialize(event) for event in content["chunk"]], ) except KeyError: if "start" not in content: raise MatrixResponseError("`start` not in response.") - elif "end" not in content: - raise MatrixResponseError("`start` not in response.") - raise MatrixResponseError("`content` not in response.") + raise MatrixResponseError("`chunk` not in response.") except SerializerError as e: raise MatrixResponseError("Invalid events in response") from e diff --git a/mautrix/types/misc.py b/mautrix/types/misc.py index cf576c2b..5a07699c 100644 --- a/mautrix/types/misc.py +++ b/mautrix/types/misc.py @@ -106,7 +106,7 @@ class RoomDirectoryResponse(SerializableAttrs): PaginatedMessages = NamedTuple( - "PaginatedMessages", start=SyncToken, end=SyncToken, events=List[Event] + "PaginatedMessages", start=SyncToken, end=Optional[SyncToken], events=List[Event] ) From 8e0dcefb9f4b68f30b58031272f1048e36980422 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 12 Apr 2026 14:04:31 +0300 Subject: [PATCH 455/456] Fix transaction in abstract crypto store --- mautrix/crypto/store/abstract.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mautrix/crypto/store/abstract.py b/mautrix/crypto/store/abstract.py index 1bae832f..0916a2d7 100644 --- a/mautrix/crypto/store/abstract.py +++ b/mautrix/crypto/store/abstract.py @@ -7,6 +7,7 @@ from typing import AsyncContextManager, NamedTuple from abc import ABC, abstractmethod +from contextlib import asynccontextmanager from mautrix.types import ( CrossSigner, @@ -87,9 +88,10 @@ async def close(self) -> None: async def flush(self) -> None: """Flush the store. If all the methods persist data immediately, this can be a no-op.""" - async def transaction(self) -> AsyncContextManager[None]: + @asynccontextmanager + async def transaction(self) -> None: """Run a database transaction. If the store doesn't support transactions, this can be a no-op.""" - pass + yield @abstractmethod async def delete(self) -> None: From bea9e63142c1fc89820be3344941cb57808456b6 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Sun, 12 Apr 2026 14:10:39 +0300 Subject: [PATCH 456/456] Update black and isort --- .github/workflows/python-package.yml | 6 +- .pre-commit-config.yaml | 4 +- dev-requirements.txt | 4 +- mautrix/client/state_store/asyncpg/upgrade.py | 22 +-- mautrix/crypto/store/asyncpg/upgrade.py | 176 +++++++++--------- mautrix/util/async_db/database.py | 8 +- mautrix/util/async_db/upgrade.py | 8 +- 7 files changed, 109 insertions(+), 119 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index e6f77c9f..3a845672 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -11,7 +11,7 @@ jobs: python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v6 with: @@ -46,7 +46,7 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - uses: actions/setup-python@v6 with: python-version: "3.14" @@ -56,7 +56,7 @@ jobs: - uses: psf/black@stable with: src: "./mautrix" - version: "25.11.0" + version: "26.3.1" - name: pre-commit run: | pip install pre-commit diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b3bd2047..66065033 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,13 +8,13 @@ repos: - id: check-yaml - id: check-added-large-files - repo: https://github.com/psf/black - rev: 25.11.0 + rev: 26.3.1 hooks: - id: black language_version: python3 files: ^mautrix/.*\.pyi?$ - repo: https://github.com/PyCQA/isort - rev: 7.0.0 + rev: 8.0.1 hooks: - id: isort files: ^mautrix/.*\.pyi?$ diff --git a/dev-requirements.txt b/dev-requirements.txt index bb8c2a0a..dff6ee9d 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,3 +1,3 @@ pre-commit>=2.10.1,<3 -isort>=5.10.1,<6 -black>=24,<25 +isort>=8,<9 +black>=26,<27 diff --git a/mautrix/client/state_store/asyncpg/upgrade.py b/mautrix/client/state_store/asyncpg/upgrade.py index c1f32664..20b2e5b2 100644 --- a/mautrix/client/state_store/asyncpg/upgrade.py +++ b/mautrix/client/state_store/asyncpg/upgrade.py @@ -16,16 +16,16 @@ @upgrade_table.register(description="Latest revision", upgrades_to=4) async def upgrade_blank_to_v4(conn: Connection, scheme: Scheme) -> None: - await conn.execute( - """CREATE TABLE mx_room_state ( + await conn.execute(""" + CREATE TABLE mx_room_state ( room_id TEXT PRIMARY KEY, is_encrypted BOOLEAN, has_full_member_list BOOLEAN, encryption TEXT, power_levels TEXT, create_event TEXT - )""" - ) + ) + """) membership_check = "" if scheme != Scheme.SQLITE: await conn.execute( @@ -33,16 +33,16 @@ async def upgrade_blank_to_v4(conn: Connection, scheme: Scheme) -> None: ) else: membership_check = "CHECK (membership IN ('join', 'leave', 'invite', 'ban', 'knock'))" - await conn.execute( - f"""CREATE TABLE mx_user_profile ( + await conn.execute(f""" + CREATE TABLE mx_user_profile ( room_id TEXT, user_id TEXT, membership membership NOT NULL {membership_check}, displayname TEXT, avatar_url TEXT, PRIMARY KEY (room_id, user_id) - )""" - ) + ) + """) @upgrade_table.register(description="Stop using size-limited string fields") @@ -60,16 +60,14 @@ async def upgrade_v2(conn: Connection, scheme: Scheme) -> None: @upgrade_table.register(description="Mark rooms that need crypto state event resynced") async def upgrade_v3(conn: Connection) -> None: if await conn.table_exists("portal"): - await conn.execute( - """ + await conn.execute(""" INSERT INTO mx_room_state (room_id, encryption) SELECT portal.mxid, '{"resync":true}' FROM portal WHERE portal.encrypted=true AND portal.mxid IS NOT NULL ON CONFLICT (room_id) DO UPDATE SET encryption=excluded.encryption WHERE mx_room_state.encryption IS NULL - """ - ) + """) @upgrade_table.register(description="Add create event to room state cache") diff --git a/mautrix/crypto/store/asyncpg/upgrade.py b/mautrix/crypto/store/asyncpg/upgrade.py index e097c5d9..8d413858 100644 --- a/mautrix/crypto/store/asyncpg/upgrade.py +++ b/mautrix/crypto/store/asyncpg/upgrade.py @@ -18,32 +18,32 @@ @upgrade_table.register(description="Latest revision", upgrades_to=10) async def upgrade_blank_to_latest(conn: Connection) -> None: - await conn.execute( - """CREATE TABLE IF NOT EXISTS crypto_account ( + await conn.execute(""" + CREATE TABLE IF NOT EXISTS crypto_account ( account_id TEXT PRIMARY KEY, device_id TEXT NOT NULL, shared BOOLEAN NOT NULL, sync_token TEXT NOT NULL, account bytea NOT NULL - )""" - ) - await conn.execute( - """CREATE TABLE IF NOT EXISTS crypto_message_index ( + ) + """) + await conn.execute(""" + CREATE TABLE IF NOT EXISTS crypto_message_index ( sender_key CHAR(43), session_id CHAR(43), "index" INTEGER, event_id TEXT NOT NULL, timestamp BIGINT NOT NULL, PRIMARY KEY (sender_key, session_id, "index") - )""" - ) - await conn.execute( - """CREATE TABLE IF NOT EXISTS crypto_tracked_user ( + ) + """) + await conn.execute(""" + CREATE TABLE IF NOT EXISTS crypto_tracked_user ( user_id TEXT PRIMARY KEY - )""" - ) - await conn.execute( - """CREATE TABLE IF NOT EXISTS crypto_device ( + ) + """) + await conn.execute(""" + CREATE TABLE IF NOT EXISTS crypto_device ( user_id TEXT, device_id TEXT, identity_key CHAR(43) NOT NULL, @@ -52,10 +52,10 @@ async def upgrade_blank_to_latest(conn: Connection) -> None: deleted BOOLEAN NOT NULL, name TEXT NOT NULL, PRIMARY KEY (user_id, device_id) - )""" - ) - await conn.execute( - """CREATE TABLE IF NOT EXISTS crypto_olm_session ( + ) + """) + await conn.execute(""" + CREATE TABLE IF NOT EXISTS crypto_olm_session ( account_id TEXT, session_id CHAR(43), sender_key CHAR(43) NOT NULL, @@ -64,10 +64,10 @@ async def upgrade_blank_to_latest(conn: Connection) -> None: last_decrypted timestamp NOT NULL, last_encrypted timestamp NOT NULL, PRIMARY KEY (account_id, session_id) - )""" - ) - await conn.execute( - """CREATE TABLE IF NOT EXISTS crypto_megolm_inbound_session ( + ) + """) + await conn.execute(""" + CREATE TABLE IF NOT EXISTS crypto_megolm_inbound_session ( account_id TEXT, session_id CHAR(43), sender_key CHAR(43) NOT NULL, @@ -83,10 +83,10 @@ async def upgrade_blank_to_latest(conn: Connection) -> None: max_messages INTEGER, is_scheduled BOOLEAN NOT NULL DEFAULT false, PRIMARY KEY (account_id, session_id) - )""" - ) - await conn.execute( - """CREATE TABLE IF NOT EXISTS crypto_megolm_outbound_session ( + ) + """) + await conn.execute(""" + CREATE TABLE IF NOT EXISTS crypto_megolm_outbound_session ( account_id TEXT, room_id TEXT, session_id CHAR(43) NOT NULL UNIQUE, @@ -98,10 +98,10 @@ async def upgrade_blank_to_latest(conn: Connection) -> None: created_at timestamp NOT NULL, last_used timestamp NOT NULL, PRIMARY KEY (account_id, room_id) - )""" - ) - await conn.execute( - """CREATE TABLE crypto_cross_signing_keys ( + ) + """) + await conn.execute(""" + CREATE TABLE crypto_cross_signing_keys ( user_id TEXT, usage TEXT, key CHAR(43) NOT NULL, @@ -109,18 +109,18 @@ async def upgrade_blank_to_latest(conn: Connection) -> None: first_seen_key CHAR(43) NOT NULL, PRIMARY KEY (user_id, usage) - )""" - ) - await conn.execute( - """CREATE TABLE crypto_cross_signing_signatures ( + ) + """) + await conn.execute(""" + CREATE TABLE crypto_cross_signing_signatures ( signed_user_id TEXT, signed_key TEXT, signer_user_id TEXT, signer_key TEXT, signature CHAR(88) NOT NULL, PRIMARY KEY (signed_user_id, signed_key, signer_user_id, signer_key) - )""" - ) + ) + """) @upgrade_table.register(description="Add account_id primary key column") @@ -130,17 +130,17 @@ async def upgrade_v2(conn: Connection, scheme: Scheme) -> None: await conn.execute("DROP TABLE crypto_olm_session") await conn.execute("DROP TABLE crypto_megolm_inbound_session") await conn.execute("DROP TABLE crypto_megolm_outbound_session") - await conn.execute( - """CREATE TABLE crypto_account ( + await conn.execute(""" + CREATE TABLE crypto_account ( account_id VARCHAR(255) PRIMARY KEY, device_id VARCHAR(255) NOT NULL, shared BOOLEAN NOT NULL, sync_token TEXT NOT NULL, account bytea NOT NULL - )""" - ) - await conn.execute( - """CREATE TABLE crypto_olm_session ( + ) + """) + await conn.execute(""" + CREATE TABLE crypto_olm_session ( account_id VARCHAR(255), session_id CHAR(43), sender_key CHAR(43) NOT NULL, @@ -148,10 +148,10 @@ async def upgrade_v2(conn: Connection, scheme: Scheme) -> None: created_at timestamp NOT NULL, last_used timestamp NOT NULL, PRIMARY KEY (account_id, session_id) - )""" - ) - await conn.execute( - """CREATE TABLE crypto_megolm_inbound_session ( + ) + """) + await conn.execute(""" + CREATE TABLE crypto_megolm_inbound_session ( account_id VARCHAR(255), session_id CHAR(43), sender_key CHAR(43) NOT NULL, @@ -160,10 +160,10 @@ async def upgrade_v2(conn: Connection, scheme: Scheme) -> None: session bytea NOT NULL, forwarding_chains TEXT NOT NULL, PRIMARY KEY (account_id, session_id) - )""" - ) - await conn.execute( - """CREATE TABLE crypto_megolm_outbound_session ( + ) + """) + await conn.execute(""" + CREATE TABLE crypto_megolm_outbound_session ( account_id VARCHAR(255), room_id VARCHAR(255), session_id CHAR(43) NOT NULL UNIQUE, @@ -175,8 +175,8 @@ async def upgrade_v2(conn: Connection, scheme: Scheme) -> None: created_at timestamp NOT NULL, last_used timestamp NOT NULL, PRIMARY KEY (account_id, room_id) - )""" - ) + ) + """) else: async def add_account_id_column(table: str, pkey_columns: list[str]) -> None: @@ -233,25 +233,25 @@ async def upgrade_v4(conn: Connection, scheme: Scheme) -> None: @upgrade_table.register(description="Add cross-signing key and signature caches") async def upgrade_v5(conn: Connection) -> None: - await conn.execute( - """CREATE TABLE crypto_cross_signing_keys ( + await conn.execute(""" + CREATE TABLE crypto_cross_signing_keys ( user_id TEXT, usage TEXT, key CHAR(43), first_seen_key CHAR(43), PRIMARY KEY (user_id, usage) - )""" - ) - await conn.execute( - """CREATE TABLE crypto_cross_signing_signatures ( + ) + """) + await conn.execute(""" + CREATE TABLE crypto_cross_signing_signatures ( signed_user_id TEXT, signed_key TEXT, signer_user_id TEXT, signer_key TEXT, signature TEXT, PRIMARY KEY (signed_user_id, signed_key, signer_user_id, signer_key) - )""" - ) + ) + """) @upgrade_table.register(description="Update trust state values") @@ -322,27 +322,25 @@ async def upgrade_v9_postgres(conn: Connection) -> None: async def upgrade_v9_sqlite(conn: Connection) -> None: await conn.execute("PRAGMA foreign_keys = OFF") async with conn.transaction(): - await conn.execute( - """CREATE TABLE new_crypto_account ( + await conn.execute(""" + CREATE TABLE new_crypto_account ( account_id TEXT PRIMARY KEY, device_id TEXT NOT NULL, shared BOOLEAN NOT NULL, sync_token TEXT NOT NULL, account bytea NOT NULL - )""" - ) - await conn.execute( - """ + ) + """) + await conn.execute(""" INSERT INTO new_crypto_account (account_id, device_id, shared, sync_token, account) SELECT account_id, COALESCE(device_id, ''), shared, sync_token, account FROM crypto_account - """ - ) + """) await conn.execute("DROP TABLE crypto_account") await conn.execute("ALTER TABLE new_crypto_account RENAME TO crypto_account") - await conn.execute( - """CREATE TABLE new_crypto_megolm_inbound_session ( + await conn.execute(""" + CREATE TABLE new_crypto_megolm_inbound_session ( account_id TEXT, session_id CHAR(43), sender_key CHAR(43) NOT NULL, @@ -353,10 +351,9 @@ async def upgrade_v9_sqlite(conn: Connection) -> None: withheld_code TEXT, withheld_reason TEXT, PRIMARY KEY (account_id, session_id) - )""" - ) - await conn.execute( - """ + ) + """) + await conn.execute(""" INSERT INTO new_crypto_megolm_inbound_session ( account_id, session_id, sender_key, signing_key, room_id, session, forwarding_chains @@ -364,8 +361,7 @@ async def upgrade_v9_sqlite(conn: Connection) -> None: SELECT account_id, session_id, sender_key, signing_key, room_id, session, forwarding_chains FROM crypto_megolm_inbound_session - """ - ) + """) await conn.execute("DROP TABLE crypto_megolm_inbound_session") await conn.execute( "ALTER TABLE new_crypto_megolm_inbound_session RENAME TO crypto_megolm_inbound_session" @@ -373,8 +369,8 @@ async def upgrade_v9_sqlite(conn: Connection) -> None: await conn.execute("UPDATE crypto_megolm_outbound_session SET max_age=max_age*1000") - await conn.execute( - """CREATE TABLE new_crypto_cross_signing_keys ( + await conn.execute(""" + CREATE TABLE new_crypto_cross_signing_keys ( user_id TEXT, usage TEXT, key CHAR(43) NOT NULL, @@ -382,41 +378,37 @@ async def upgrade_v9_sqlite(conn: Connection) -> None: first_seen_key CHAR(43) NOT NULL, PRIMARY KEY (user_id, usage) - )""" - ) - await conn.execute( - """ + ) + """) + await conn.execute(""" INSERT INTO new_crypto_cross_signing_keys (user_id, usage, key, first_seen_key) SELECT user_id, usage, key, COALESCE(first_seen_key, key) FROM crypto_cross_signing_keys WHERE key IS NOT NULL - """ - ) + """) await conn.execute("DROP TABLE crypto_cross_signing_keys") await conn.execute( "ALTER TABLE new_crypto_cross_signing_keys RENAME TO crypto_cross_signing_keys" ) - await conn.execute( - """CREATE TABLE new_crypto_cross_signing_signatures ( + await conn.execute(""" + CREATE TABLE new_crypto_cross_signing_signatures ( signed_user_id TEXT, signed_key TEXT, signer_user_id TEXT, signer_key TEXT, signature CHAR(88) NOT NULL, PRIMARY KEY (signed_user_id, signed_key, signer_user_id, signer_key) - )""" - ) - await conn.execute( - """ + ) + """) + await conn.execute(""" INSERT INTO new_crypto_cross_signing_signatures ( signed_user_id, signed_key, signer_user_id, signer_key, signature ) SELECT signed_user_id, signed_key, signer_user_id, signer_key, signature FROM crypto_cross_signing_signatures WHERE signature IS NOT NULL - """ - ) + """) await conn.execute("DROP TABLE crypto_cross_signing_signatures") await conn.execute( "ALTER TABLE new_crypto_cross_signing_signatures " diff --git a/mautrix/util/async_db/database.py b/mautrix/util/async_db/database.py index 6a17dc68..b5128b74 100644 --- a/mautrix/util/async_db/database.py +++ b/mautrix/util/async_db/database.py @@ -115,12 +115,12 @@ async def _check_foreign_tables(self) -> None: raise ForeignTablesFound("found roomserver_rooms likely belonging to Dendrite") async def _check_owner(self) -> None: - await self.execute( - """CREATE TABLE IF NOT EXISTS database_owner ( + await self.execute(""" + CREATE TABLE IF NOT EXISTS database_owner ( key INTEGER PRIMARY KEY DEFAULT 0, owner TEXT NOT NULL - )""" - ) + ) + """) owner = await self.fetchval("SELECT owner FROM database_owner WHERE key=0") if not owner: await self.execute( diff --git a/mautrix/util/async_db/upgrade.py b/mautrix/util/async_db/upgrade.py index 9e593ece..c084d28b 100644 --- a/mautrix/util/async_db/upgrade.py +++ b/mautrix/util/async_db/upgrade.py @@ -97,11 +97,11 @@ async def _save_version(self, conn: LoggingConnection, version: int) -> None: await conn.execute(f"INSERT INTO {self.version_table_name} (version) VALUES ($1)", version) async def upgrade(self, db: async_db.Database) -> None: - await db.execute( - f"""CREATE TABLE IF NOT EXISTS {self.version_table_name} ( + await db.execute(f""" + CREATE TABLE IF NOT EXISTS {self.version_table_name} ( version INTEGER PRIMARY KEY - )""" - ) + ) + """) row = await db.fetchrow(f"SELECT version FROM {self.version_table_name} LIMIT 1") version = row["version"] if row else 0