From cf2f2934ad6c94a269e750684d1d8170b1173b7a Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Mon, 3 Apr 2023 13:20:32 -0400 Subject: [PATCH 1/9] Call appservices on modern paths, falling back to legacy paths. (#15317) This uses the specced /_matrix/app/v1/... paths instead of the "legacy" paths. If the homeserver receives an error it will retry using the legacy path. --- changelog.d/15317.bugfix | 1 + docs/upgrade.md | 16 +++++ synapse/appservice/api.py | 133 ++++++++++++++++++++++++----------- synapse/http/client.py | 13 ++-- tests/appservice/test_api.py | 57 ++++++++++++++- 5 files changed, 172 insertions(+), 48 deletions(-) create mode 100644 changelog.d/15317.bugfix diff --git a/changelog.d/15317.bugfix b/changelog.d/15317.bugfix new file mode 100644 index 000000000000..194e4c46c64c --- /dev/null +++ b/changelog.d/15317.bugfix @@ -0,0 +1 @@ +Fix a long-standing bug that Synpase only used the [legacy appservice routes](https://spec.matrix.org/v1.6/application-service-api/#legacy-routes). diff --git a/docs/upgrade.md b/docs/upgrade.md index f14444a400ea..1ddfc31ff603 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -88,6 +88,22 @@ process, for example: dpkg -i matrix-synapse-py3_1.3.0+stretch1_amd64.deb ``` +# Upgrading to v1.81.0 + +## Application service path & authentication deprecations + +Synapse now attempts the versioned appservice paths before falling back to the +[legacy paths](https://spec.matrix.org/v1.6/application-service-api/#legacy-routes). +Usage of the legacy routes should be considered deprecated. + +Additionally, Synapse has supported sending the application service access token +via [the `Authorization` header](https://spec.matrix.org/v1.6/application-service-api/#authorization) +since v1.70.0. For backwards compatibility it is *also* sent as the `access_token` +query parameter. This is insecure and should be considered deprecated. + +A future version of Synapse (v1.88.0 or later) will remove support for legacy +application service routes and query parameter authorization. + # Upgrading to v1.80.0 ## Reporting events error code change diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py index b27eedef99f2..86ddb1bb289e 100644 --- a/synapse/appservice/api.py +++ b/synapse/appservice/api.py @@ -17,6 +17,8 @@ from typing import ( TYPE_CHECKING, Any, + Awaitable, + Callable, Dict, Iterable, List, @@ -24,10 +26,11 @@ Optional, Sequence, Tuple, + TypeVar, ) from prometheus_client import Counter -from typing_extensions import TypeGuard +from typing_extensions import Concatenate, ParamSpec, TypeGuard from synapse.api.constants import EventTypes, Membership, ThirdPartyEntityKind from synapse.api.errors import CodeMessageException, HttpResponseException @@ -78,7 +81,11 @@ HOUR_IN_MS = 60 * 60 * 1000 -APP_SERVICE_PREFIX = "/_matrix/app/unstable" +APP_SERVICE_PREFIX = "/_matrix/app/v1" +APP_SERVICE_UNSTABLE_PREFIX = "/_matrix/app/unstable" + +P = ParamSpec("P") +R = TypeVar("R") def _is_valid_3pe_metadata(info: JsonDict) -> bool: @@ -121,6 +128,47 @@ def __init__(self, hs: "HomeServer"): hs.get_clock(), "as_protocol_meta", timeout_ms=HOUR_IN_MS ) + async def _send_with_fallbacks( + self, + service: "ApplicationService", + prefixes: List[str], + path: str, + func: Callable[Concatenate[str, P], Awaitable[R]], + *args: P.args, + **kwargs: P.kwargs, + ) -> R: + """ + Attempt to call an application service with multiple paths, falling back + until one succeeds. + + Args: + service: The appliacation service, this provides the base URL. + prefixes: A last of paths to try in order for the requests. + path: A suffix to append to each prefix. + func: The function to call, the first argument will be the full + endpoint to fetch. Other arguments are provided by args/kwargs. + + Returns: + The return value of func. + """ + for i, prefix in enumerate(prefixes, start=1): + uri = f"{service.url}{prefix}{path}" + try: + return await func(uri, *args, **kwargs) + except HttpResponseException as e: + # If an error is received that is due to an unrecognised path, + # fallback to next path (if one exists). Otherwise, consider it + # a legitimate error and raise. + if i < len(prefixes) and is_unknown_endpoint(e): + continue + raise + except Exception: + # Unexpected exceptions get sent to the caller. + raise + + # The function should always exit via the return or raise above this. + raise RuntimeError("Unexpected fallback behaviour. This should never be seen.") + async def query_user(self, service: "ApplicationService", user_id: str) -> bool: if service.url is None: return False @@ -128,10 +176,12 @@ async def query_user(self, service: "ApplicationService", user_id: str) -> bool: # This is required by the configuration. assert service.hs_token is not None - uri = service.url + ("/users/%s" % urllib.parse.quote(user_id)) try: - response = await self.get_json( - uri, + response = await self._send_with_fallbacks( + service, + [APP_SERVICE_PREFIX, ""], + f"/users/{urllib.parse.quote(user_id)}", + self.get_json, {"access_token": service.hs_token}, headers={"Authorization": [f"Bearer {service.hs_token}"]}, ) @@ -140,9 +190,9 @@ async def query_user(self, service: "ApplicationService", user_id: str) -> bool: except CodeMessageException as e: if e.code == 404: return False - logger.warning("query_user to %s received %s", uri, e.code) + logger.warning("query_user to %s received %s", service.url, e.code) except Exception as ex: - logger.warning("query_user to %s threw exception %s", uri, ex) + logger.warning("query_user to %s threw exception %s", service.url, ex) return False async def query_alias(self, service: "ApplicationService", alias: str) -> bool: @@ -152,21 +202,23 @@ async def query_alias(self, service: "ApplicationService", alias: str) -> bool: # This is required by the configuration. assert service.hs_token is not None - uri = service.url + ("/rooms/%s" % urllib.parse.quote(alias)) try: - response = await self.get_json( - uri, + response = await self._send_with_fallbacks( + service, + [APP_SERVICE_PREFIX, ""], + f"/rooms/{urllib.parse.quote(alias)}", + self.get_json, {"access_token": service.hs_token}, headers={"Authorization": [f"Bearer {service.hs_token}"]}, ) if response is not None: # just an empty json object return True except CodeMessageException as e: - logger.warning("query_alias to %s received %s", uri, e.code) + logger.warning("query_alias to %s received %s", service.url, e.code) if e.code == 404: return False except Exception as ex: - logger.warning("query_alias to %s threw exception %s", uri, ex) + logger.warning("query_alias to %s threw exception %s", service.url, ex) return False async def query_3pe( @@ -188,25 +240,24 @@ async def query_3pe( # This is required by the configuration. assert service.hs_token is not None - uri = "%s%s/thirdparty/%s/%s" % ( - service.url, - APP_SERVICE_PREFIX, - kind, - urllib.parse.quote(protocol), - ) try: args: Mapping[Any, Any] = { **fields, b"access_token": service.hs_token, } - response = await self.get_json( - uri, + response = await self._send_with_fallbacks( + service, + [APP_SERVICE_PREFIX, APP_SERVICE_UNSTABLE_PREFIX], + f"/thirdparty/{kind}/{urllib.parse.quote(protocol)}", + self.get_json, args=args, headers={"Authorization": [f"Bearer {service.hs_token}"]}, ) if not isinstance(response, list): logger.warning( - "query_3pe to %s returned an invalid response %r", uri, response + "query_3pe to %s returned an invalid response %r", + service.url, + response, ) return [] @@ -216,12 +267,12 @@ async def query_3pe( ret.append(r) else: logger.warning( - "query_3pe to %s returned an invalid result %r", uri, r + "query_3pe to %s returned an invalid result %r", service.url, r ) return ret except Exception as ex: - logger.warning("query_3pe to %s threw exception %s", uri, ex) + logger.warning("query_3pe to %s threw exception %s", service.url, ex) return [] async def get_3pe_protocol( @@ -233,21 +284,20 @@ async def get_3pe_protocol( async def _get() -> Optional[JsonDict]: # This is required by the configuration. assert service.hs_token is not None - uri = "%s%s/thirdparty/protocol/%s" % ( - service.url, - APP_SERVICE_PREFIX, - urllib.parse.quote(protocol), - ) try: - info = await self.get_json( - uri, + info = await self._send_with_fallbacks( + service, + [APP_SERVICE_PREFIX, APP_SERVICE_UNSTABLE_PREFIX], + f"/thirdparty/protocol/{urllib.parse.quote(protocol)}", + self.get_json, {"access_token": service.hs_token}, headers={"Authorization": [f"Bearer {service.hs_token}"]}, ) if not _is_valid_3pe_metadata(info): logger.warning( - "query_3pe_protocol to %s did not return a valid result", uri + "query_3pe_protocol to %s did not return a valid result", + service.url, ) return None @@ -260,7 +310,9 @@ async def _get() -> Optional[JsonDict]: return info except Exception as ex: - logger.warning("query_3pe_protocol to %s threw exception %s", uri, ex) + logger.warning( + "query_3pe_protocol to %s threw exception %s", service.url, ex + ) return None key = (service.id, protocol) @@ -274,7 +326,7 @@ async def ping(self, service: "ApplicationService", txn_id: Optional[str]) -> No assert service.hs_token is not None await self.post_json_get_json( - uri=service.url + "/_matrix/app/unstable/fi.mau.msc2659/ping", + uri=f"{service.url}{APP_SERVICE_UNSTABLE_PREFIX}/fi.mau.msc2659/ping", post_json={"transaction_id": txn_id}, headers={"Authorization": [f"Bearer {service.hs_token}"]}, ) @@ -318,8 +370,6 @@ async def push_bulk( ) txn_id = 0 - uri = service.url + ("/transactions/%s" % urllib.parse.quote(str(txn_id))) - # Never send ephemeral events to appservices that do not support it body: JsonDict = {"events": serialized_events} if service.supports_ephemeral: @@ -351,8 +401,11 @@ async def push_bulk( } try: - await self.put_json( - uri=uri, + await self._send_with_fallbacks( + service, + [APP_SERVICE_PREFIX, ""], + f"/transactions/{urllib.parse.quote(str(txn_id))}", + self.put_json, json_body=body, args={"access_token": service.hs_token}, headers={"Authorization": [f"Bearer {service.hs_token}"]}, @@ -360,7 +413,7 @@ async def push_bulk( if logger.isEnabledFor(logging.DEBUG): logger.debug( "push_bulk to %s succeeded! events=%s", - uri, + service.url, [event.get("event_id") for event in events], ) sent_transactions_counter.labels(service.id).inc() @@ -371,7 +424,7 @@ async def push_bulk( except CodeMessageException as e: logger.warning( "push_bulk to %s received code=%s msg=%s", - uri, + service.url, e.code, e.msg, exc_info=logger.isEnabledFor(logging.DEBUG), @@ -379,7 +432,7 @@ async def push_bulk( except Exception as ex: logger.warning( "push_bulk to %s threw exception(%s) %s args=%s", - uri, + service.url, type(ex).__name__, ex, ex.args, diff --git a/synapse/http/client.py b/synapse/http/client.py index 5ee55981d9f8..b5cf8123ce84 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -982,20 +982,21 @@ def is_unknown_endpoint( """ if synapse_error is None: synapse_error = e.to_synapse_error() - # MSC3743 specifies that servers should return a 404 or 405 with an errcode + + # Matrix v1.6 specifies that servers should return a 404 or 405 with an errcode # of M_UNRECOGNIZED when they receive a request to an unknown endpoint or # to an unknown method, respectively. # - # Older versions of servers don't properly handle this. This needs to be - # rather specific as some endpoints truly do return 404 errors. + # Older versions of servers don't return proper errors, so be graceful. But, + # also handle that some endpoints truly do return 404 errors. return ( # 404 is an unknown endpoint, 405 is a known endpoint, but unknown method. (e.code == 404 or e.code == 405) and ( - # Older Dendrites returned a text body or empty body. - # Older Conduit returned an empty body. + # Consider empty body or non-JSON bodies to be unrecognised (matches + # older Dendrites & Conduits). not e.response - or e.response == b"404 page not found" + or not e.response.startswith(b"{") # The proper response JSON with M_UNRECOGNIZED errcode. or synapse_error.errcode == Codes.UNRECOGNIZED ) diff --git a/tests/appservice/test_api.py b/tests/appservice/test_api.py index 0dd02b7d5839..7deb923a280d 100644 --- a/tests/appservice/test_api.py +++ b/tests/appservice/test_api.py @@ -16,6 +16,7 @@ from twisted.test.proto_helpers import MemoryReactor +from synapse.api.errors import HttpResponseException from synapse.appservice import ApplicationService from synapse.server import HomeServer from synapse.types import JsonDict @@ -64,8 +65,8 @@ def test_query_3pe_authenticates_token(self) -> None: } ] - URL_USER = f"{URL}/_matrix/app/unstable/thirdparty/user/{PROTOCOL}" - URL_LOCATION = f"{URL}/_matrix/app/unstable/thirdparty/location/{PROTOCOL}" + URL_USER = f"{URL}/_matrix/app/v1/thirdparty/user/{PROTOCOL}" + URL_LOCATION = f"{URL}/_matrix/app/v1/thirdparty/location/{PROTOCOL}" self.request_url = None @@ -106,6 +107,58 @@ async def get_json( self.assertEqual(self.request_url, URL_LOCATION) self.assertEqual(result, SUCCESS_RESULT_LOCATION) + def test_fallback(self) -> None: + """ + Tests that the fallback to legacy URLs works. + """ + SUCCESS_RESULT_USER = [ + { + "protocol": PROTOCOL, + "userid": "@a:user", + "fields": { + "more": "fields", + }, + } + ] + + URL_USER = f"{URL}/_matrix/app/v1/thirdparty/user/{PROTOCOL}" + FALLBACK_URL_USER = f"{URL}/_matrix/app/unstable/thirdparty/user/{PROTOCOL}" + + self.request_url = None + self.v1_seen = False + + async def get_json( + url: str, + args: Mapping[Any, Any], + headers: Mapping[Union[str, bytes], Sequence[Union[str, bytes]]], + ) -> List[JsonDict]: + # Ensure the access token is passed as both a header and query arg. + if not headers.get("Authorization") or not args.get(b"access_token"): + raise RuntimeError("Access token not provided") + + self.assertEqual(headers.get("Authorization"), [f"Bearer {TOKEN}"]) + self.assertEqual(args.get(b"access_token"), TOKEN) + self.request_url = url + if url == URL_USER: + self.v1_seen = True + raise HttpResponseException(404, "NOT_FOUND", b"NOT_FOUND") + elif url == FALLBACK_URL_USER: + return SUCCESS_RESULT_USER + else: + raise RuntimeError( + "URL provided was invalid. This should never be seen." + ) + + # We assign to a method, which mypy doesn't like. + self.api.get_json = Mock(side_effect=get_json) # type: ignore[assignment] + + result = self.get_success( + self.api.query_3pe(self.service, "user", PROTOCOL, {b"some": [b"field"]}) + ) + self.assertTrue(self.v1_seen) + self.assertEqual(self.request_url, FALLBACK_URL_USER) + self.assertEqual(result, SUCCESS_RESULT_USER) + def test_claim_keys(self) -> None: """ Tests that the /keys/claim response is properly parsed for missing From 8aa121c2be699af439f48af785c23cda189c9198 Mon Sep 17 00:00:00 2001 From: Sean Quah <8349537+squahtx@users.noreply.github.com> Date: Mon, 3 Apr 2023 19:37:26 +0100 Subject: [PATCH 2/9] Build Debian packages for Ubuntu 23.04 Lunar Lobster (#15381) Signed-off-by: Sean Quah --- changelog.d/15381.feature | 1 + scripts-dev/build_debian_packages.py | 1 + 2 files changed, 2 insertions(+) create mode 100644 changelog.d/15381.feature diff --git a/changelog.d/15381.feature b/changelog.d/15381.feature new file mode 100644 index 000000000000..ba5b578564e9 --- /dev/null +++ b/changelog.d/15381.feature @@ -0,0 +1 @@ +Build Debian packages for Ubuntu 23.04 (Lunar Lobster). diff --git a/scripts-dev/build_debian_packages.py b/scripts-dev/build_debian_packages.py index 744230019610..ede766501100 100755 --- a/scripts-dev/build_debian_packages.py +++ b/scripts-dev/build_debian_packages.py @@ -28,6 +28,7 @@ "ubuntu:focal", # 20.04 LTS (our EOL forced by Py38 on 2024-10-14) "ubuntu:jammy", # 22.04 LTS (EOL 2027-04) "ubuntu:kinetic", # 22.10 (EOL 2023-07-20) + "ubuntu:lunar", # 23.04 (EOL 2024-01) ) DESC = """\ From c0772b4461ff396b6a09b4a17368d61a2810c452 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 4 Apr 2023 13:08:17 +0100 Subject: [PATCH 3/9] Note that Synapse 1.74 queued a user dir rebuild (#15386) * Note that Synapse 1.74 queued a user dir rebuild * Changelog --- changelog.d/15386.doc | 1 + docs/upgrade.md | 11 +++++++++++ 2 files changed, 12 insertions(+) create mode 100644 changelog.d/15386.doc diff --git a/changelog.d/15386.doc b/changelog.d/15386.doc new file mode 100644 index 000000000000..5c032193aeef --- /dev/null +++ b/changelog.d/15386.doc @@ -0,0 +1 @@ +Note that Synapse 1.74 queued a rebuild of the user directory tables. diff --git a/docs/upgrade.md b/docs/upgrade.md index 1ddfc31ff603..0886b0311571 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -199,6 +199,17 @@ Docker images and Debian packages need nothing specific as they already include or specify ICU as an explicit dependency. +## User directory rebuild + +Synapse 1.74 queues a background update +[to rebuild the user directory](https://github.com/matrix-org/synapse/pull/14643), +in order to fix missing or erroneous entries. + +When this update begins, the user directory will be cleared out and rebuilt from +scratch. User directory lookups will be incomplete until the rebuild completes. +Admins can monitor the rebuild's progress by using the +[Background update Admin API](usage/administration/admin_api/background_updates.md#status). + # Upgrading to v1.73.0 ## Legacy Prometheus metric names have now been removed From 89a71e73905ffa1c97ae8be27d521cd2ef3f3a0c Mon Sep 17 00:00:00 2001 From: Sean Quah <8349537+squahtx@users.noreply.github.com> Date: Tue, 4 Apr 2023 13:10:25 +0100 Subject: [PATCH 4/9] Fix a rare bug where initial /syncs would fail (#15383) This change fixes a rare bug where initial /syncs would fail with a `KeyError` under the following circumstances: 1. A user fast joins a remote room. 2. The user is kicked from the room before the room's full state has been synced. 3. A second local user fast joins the room. 4. Events are backfilled into the room with a higher topological ordering than the original user's leave. They are assigned a negative stream ordering. It's not clear how backfill happened here, since it is expected to be equivalent to syncing the full state. 5. The second local user leaves the room before the room's full state has been synced. The homeserver does not complete the sync. 6. The original user performs an initial /sync with lazy_load_members enabled. * Because they were kicked from the room, the room is included in the /sync response even though the include_leave option is not specified. * To populate the room's timeline, `_load_filtered_recents` / `get_recent_events_for_room` fetches events with a lower stream ordering than the leave event and picks the ones with the highest topological orderings (which are most recent). This captures the backfilled events after the leave, since they have a negative stream ordering. These events are filtered out of the timeline, since the user was not in the room at the time and cannot view them. The sync code ends up with an empty timeline for the room that notably does not include the user's leave event. This seems buggy, but at least we don't disclose events the user isn't allowed to see. * Normally, `compute_state_delta` would fetch the state at the start and end of the room's timeline to generate the sync response. Since the timeline is empty, it fetches the state at `min(now, last event in the room)`, which corresponds with the second user's leave. The state during the entirety of the second user's membership does not include the membership for the first user because of partial state. This part is also questionable, since we are fetching state from outside the bounds of the user's membership. * `compute_state_delta` then tries and fails to find the user's membership in the auth events of timeline events. Because there is no timeline event whose auth events are expected to contain the user's membership, a `KeyError` is raised. Also contains a drive-by fix for a separate unlikely race condition. Signed-off-by: Sean Quah --- changelog.d/15383.bugfix | 1 + synapse/handlers/sync.py | 24 +++++++++++++++++++----- 2 files changed, 20 insertions(+), 5 deletions(-) create mode 100644 changelog.d/15383.bugfix diff --git a/changelog.d/15383.bugfix b/changelog.d/15383.bugfix new file mode 100644 index 000000000000..28c66ef45461 --- /dev/null +++ b/changelog.d/15383.bugfix @@ -0,0 +1 @@ +Fix a rare bug introduced in Synapse 1.66.0 where initial syncs would fail when the user had been kicked from a faster joined room that had not finished syncing. diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 9f5b83ed5492..64d298408d21 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -943,6 +943,8 @@ async def compute_state_delta( timeline_state = {} + # Membership events to fetch that can be found in the room state, or in + # the case of partial state rooms, the auth events of timeline events. members_to_fetch = set() first_event_by_sender_map = {} for event in batch.events: @@ -964,9 +966,19 @@ async def compute_state_delta( # (if we are) to fix https://github.com/vector-im/riot-web/issues/7209 # We only need apply this on full state syncs given we disabled # LL for incr syncs in #3840. - members_to_fetch.add(sync_config.user.to_string()) - - state_filter = StateFilter.from_lazy_load_member_list(members_to_fetch) + # We don't insert ourselves into `members_to_fetch`, because in some + # rare cases (an empty event batch with a now_token after the user's + # leave in a partial state room which another local user has + # joined), the room state will be missing our membership and there + # is no guarantee that our membership will be in the auth events of + # timeline events when the room is partial stated. + state_filter = StateFilter.from_lazy_load_member_list( + members_to_fetch.union((sync_config.user.to_string(),)) + ) + else: + state_filter = StateFilter.from_lazy_load_member_list( + members_to_fetch + ) # We are happy to use partial state to compute the `/sync` response. # Since partial state may not include the lazy-loaded memberships we @@ -988,7 +1000,9 @@ async def compute_state_delta( # sync's timeline and the start of the current sync's timeline. # See the docstring above for details. state_ids: StateMap[str] - + # We need to know whether the state we fetch may be partial, so check + # whether the room is partial stated *before* fetching it. + is_partial_state_room = await self.store.is_partial_state_room(room_id) if full_state: if batch: state_at_timeline_end = ( @@ -1119,7 +1133,7 @@ async def compute_state_delta( # If we only have partial state for the room, `state_ids` may be missing the # memberships we wanted. We attempt to find some by digging through the auth # events of timeline events. - if lazy_load_members and await self.store.is_partial_state_room(room_id): + if lazy_load_members and is_partial_state_room: assert members_to_fetch is not None assert first_event_by_sender_map is not None From 79d2e2e79c97b21a4b3b786594d0d9ebebd33964 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 4 Apr 2023 14:11:34 +0100 Subject: [PATCH 5/9] Speed up membership queries for users with forgotten rooms (#15385) --- changelog.d/15385.misc | 1 + synapse/storage/databases/main/roommember.py | 12 +++++++++++- .../delta/74/03_room_membership_index.sql | 19 +++++++++++++++++++ 3 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 changelog.d/15385.misc create mode 100644 synapse/storage/schema/main/delta/74/03_room_membership_index.sql diff --git a/changelog.d/15385.misc b/changelog.d/15385.misc new file mode 100644 index 000000000000..76350c398d6f --- /dev/null +++ b/changelog.d/15385.misc @@ -0,0 +1 @@ +Speed up membership queries for users with forgotten rooms. diff --git a/synapse/storage/databases/main/roommember.py b/synapse/storage/databases/main/roommember.py index 694a5b802c7c..daad58291a8b 100644 --- a/synapse/storage/databases/main/roommember.py +++ b/synapse/storage/databases/main/roommember.py @@ -419,7 +419,11 @@ async def get_rooms_for_local_user_where_membership_is( ) # Now we filter out forgotten and excluded rooms - rooms_to_exclude = await self.get_forgotten_rooms_for_user(user_id) + rooms_to_exclude: AbstractSet[str] = set() + + # Users can't forget joined/invited rooms, so we skip the check for such look ups. + if any(m not in (Membership.JOIN, Membership.INVITE) for m in membership_list): + rooms_to_exclude = await self.get_forgotten_rooms_for_user(user_id) if excluded_rooms is not None: # Take a copy to avoid mutating the in-cache set @@ -1391,6 +1395,12 @@ def __init__( columns=["user_id", "room_id"], where_clause="forgotten = 1", ) + self.db_pool.updates.register_background_index_update( + "room_membership_user_room_index", + index_name="room_membership_user_room_idx", + table="room_memberships", + columns=["user_id", "room_id"], + ) async def _background_add_membership_profile( self, progress: JsonDict, batch_size: int diff --git a/synapse/storage/schema/main/delta/74/03_room_membership_index.sql b/synapse/storage/schema/main/delta/74/03_room_membership_index.sql new file mode 100644 index 000000000000..81a7d9ff9cad --- /dev/null +++ b/synapse/storage/schema/main/delta/74/03_room_membership_index.sql @@ -0,0 +1,19 @@ +/* Copyright 2023 The Matrix.org Foundation C.I.C + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +-- Add an index to `room_membership(user_id, room_id)` to make querying for +-- forgotten rooms faster. +INSERT INTO background_updates (ordering, update_name, progress_json) VALUES + (7403, 'room_membership_user_room_index', '{}'); From 735e4d1f9d064ab69aa642dd989ada0cda9985f9 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 4 Apr 2023 14:29:08 +0100 Subject: [PATCH 6/9] 1.81.0rc1 --- CHANGES.md | 69 +++++++++++++++++++++++++++++++++++++++ changelog.d/14978.feature | 1 - changelog.d/15113.misc | 1 - changelog.d/15243.feature | 1 - changelog.d/15265.misc | 1 - changelog.d/15280.misc | 1 - changelog.d/15285.misc | 1 - changelog.d/15295.bugfix | 1 - changelog.d/15297.bugfix | 1 - changelog.d/15302.misc | 1 - changelog.d/15303.misc | 1 - changelog.d/15304.misc | 1 - changelog.d/15306.bugfix | 2 -- changelog.d/15309.bugfix | 1 - changelog.d/15311.misc | 1 - changelog.d/15314.feature | 1 - changelog.d/15316.misc | 1 - changelog.d/15317.bugfix | 1 - changelog.d/15319.misc | 1 - changelog.d/15321.feature | 1 - changelog.d/15323.bugfix | 1 - changelog.d/15324.misc | 1 - changelog.d/15325.misc | 1 - changelog.d/15326.misc | 1 - changelog.d/15327.misc | 1 - changelog.d/15328.misc | 1 - changelog.d/15329.misc | 1 - changelog.d/15330.misc | 1 - changelog.d/15331.feature | 1 - changelog.d/15332.bugfix | 1 - changelog.d/15334.misc | 1 - changelog.d/15336.misc | 1 - changelog.d/15339.misc | 1 - changelog.d/15340.misc | 1 - changelog.d/15341.doc | 1 - changelog.d/15349.bugfix | 1 - changelog.d/15351.bugfix | 1 - changelog.d/15352.bugfix | 1 - changelog.d/15353.misc | 1 - changelog.d/15354.misc | 1 - changelog.d/15358.misc | 1 - changelog.d/15369.misc | 1 - changelog.d/15370.misc | 1 - changelog.d/15371.misc | 1 - changelog.d/15381.feature | 1 - changelog.d/15383.bugfix | 1 - changelog.d/15385.misc | 1 - changelog.d/15386.doc | 1 - debian/changelog | 6 ++++ pyproject.toml | 2 +- 50 files changed, 76 insertions(+), 49 deletions(-) delete mode 100644 changelog.d/14978.feature delete mode 100644 changelog.d/15113.misc delete mode 100644 changelog.d/15243.feature delete mode 100644 changelog.d/15265.misc delete mode 100644 changelog.d/15280.misc delete mode 100644 changelog.d/15285.misc delete mode 100644 changelog.d/15295.bugfix delete mode 100644 changelog.d/15297.bugfix delete mode 100644 changelog.d/15302.misc delete mode 100644 changelog.d/15303.misc delete mode 100644 changelog.d/15304.misc delete mode 100644 changelog.d/15306.bugfix delete mode 100644 changelog.d/15309.bugfix delete mode 100644 changelog.d/15311.misc delete mode 100644 changelog.d/15314.feature delete mode 100644 changelog.d/15316.misc delete mode 100644 changelog.d/15317.bugfix delete mode 100644 changelog.d/15319.misc delete mode 100644 changelog.d/15321.feature delete mode 100644 changelog.d/15323.bugfix delete mode 100644 changelog.d/15324.misc delete mode 100644 changelog.d/15325.misc delete mode 100644 changelog.d/15326.misc delete mode 100644 changelog.d/15327.misc delete mode 100644 changelog.d/15328.misc delete mode 100644 changelog.d/15329.misc delete mode 100644 changelog.d/15330.misc delete mode 100644 changelog.d/15331.feature delete mode 100644 changelog.d/15332.bugfix delete mode 100644 changelog.d/15334.misc delete mode 100644 changelog.d/15336.misc delete mode 100644 changelog.d/15339.misc delete mode 100644 changelog.d/15340.misc delete mode 100644 changelog.d/15341.doc delete mode 100644 changelog.d/15349.bugfix delete mode 100644 changelog.d/15351.bugfix delete mode 100644 changelog.d/15352.bugfix delete mode 100644 changelog.d/15353.misc delete mode 100644 changelog.d/15354.misc delete mode 100644 changelog.d/15358.misc delete mode 100644 changelog.d/15369.misc delete mode 100644 changelog.d/15370.misc delete mode 100644 changelog.d/15371.misc delete mode 100644 changelog.d/15381.feature delete mode 100644 changelog.d/15383.bugfix delete mode 100644 changelog.d/15385.misc delete mode 100644 changelog.d/15386.doc diff --git a/CHANGES.md b/CHANGES.md index 5f2a4a41eb42..1f77fab96078 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,72 @@ +Synapse 1.81.0rc1 (2023-04-04) +============================== + +Features +-------- + +- Add the ability to enable/disable registrations when in the OIDC flow. ([\#14978](https://github.com/matrix-org/synapse/issues/14978)) +- Add a primitive helper script for listing worker endpoints. ([\#15243](https://github.com/matrix-org/synapse/issues/15243)) +- Experimental support for passing One Time Key and device key requests to application services ([MSC3983](https://github.com/matrix-org/matrix-spec-proposals/pull/3983) and [MSC3984](https://github.com/matrix-org/matrix-spec-proposals/pull/3984)). ([\#15314](https://github.com/matrix-org/synapse/issues/15314), [\#15321](https://github.com/matrix-org/synapse/issues/15321)) +- Allow loading `/password_policy` endpoint on workers. ([\#15331](https://github.com/matrix-org/synapse/issues/15331)) +- Build Debian packages for Ubuntu 23.04 (Lunar Lobster). ([\#15381](https://github.com/matrix-org/synapse/issues/15381)) + + +Bugfixes +-------- + +- Fix a long-standing bug where edits of non-`m.room.message` events would not be correctly bundled. ([\#15295](https://github.com/matrix-org/synapse/issues/15295)) +- Fix a bug introduced in Synapse v1.55.0 which could delay remote homeservers being able to decrypt encrypted messages sent by local users. ([\#15297](https://github.com/matrix-org/synapse/issues/15297)) +- Add a check to [SQLite port_db script](https://matrix-org.github.io/synapse/latest/postgres.html#porting-from-sqlite) + to ensure that the sqlite database passed to the script exists before trying to port from it. ([\#15306](https://github.com/matrix-org/synapse/issues/15306)) +- Fix a bug introduced in Synapse 1.76.0 where responses from worker deployments could include an internal `_INT_STREAM_POS` key. ([\#15309](https://github.com/matrix-org/synapse/issues/15309)) +- Fix a long-standing bug that Synpase only used the [legacy appservice routes](https://spec.matrix.org/v1.6/application-service-api/#legacy-routes). ([\#15317](https://github.com/matrix-org/synapse/issues/15317)) +- Fix a long-standing bug preventing users from joining rooms, that they had been unbanned from, over federation. Contributed by Nico. ([\#15323](https://github.com/matrix-org/synapse/issues/15323)) +- Fix bug in worker mode where on a rolling restart of workers the "typing" worker would consume 100% CPU until it got restarted. ([\#15332](https://github.com/matrix-org/synapse/issues/15332)) +- Fix a long-standing bug where some to_device messages could be dropped when using workers. ([\#15349](https://github.com/matrix-org/synapse/issues/15349)) +- Fix a bug introduced in Synapse 1.70.0 where the background sync from a faster join could spin for hours when one of the events involved had been marked for backoff. ([\#15351](https://github.com/matrix-org/synapse/issues/15351)) +- Fix missing app variable in mail subject for password resets. Contributed by Cyberes. ([\#15352](https://github.com/matrix-org/synapse/issues/15352)) +- Fix a rare bug introduced in Synapse 1.66.0 where initial syncs would fail when the user had been kicked from a faster joined room that had not finished syncing. ([\#15383](https://github.com/matrix-org/synapse/issues/15383)) + + +Improved Documentation +---------------------- + +- Fix a typo in login requests ratelimit defaults. ([\#15341](https://github.com/matrix-org/synapse/issues/15341)) +- Note that Synapse 1.74 queued a rebuild of the user directory tables. ([\#15386](https://github.com/matrix-org/synapse/issues/15386)) + + +Internal Changes +---------------- + +- Use `immutabledict` instead of `frozendict`. ([\#15113](https://github.com/matrix-org/synapse/issues/15113)) +- Add developer documentation for the Federation Sender and add a documentation mechanism using Sphinx. ([\#15265](https://github.com/matrix-org/synapse/issues/15265), [\#15336](https://github.com/matrix-org/synapse/issues/15336)) +- Make the pushers rely on the `device_id` instead of the `access_token_id` for various operations. ([\#15280](https://github.com/matrix-org/synapse/issues/15280)) +- Bump sentry-sdk from 1.15.0 to 1.17.0. ([\#15285](https://github.com/matrix-org/synapse/issues/15285)) +- Allow running the Twisted trunk job against other branches. ([\#15302](https://github.com/matrix-org/synapse/issues/15302)) +- Remind the releaser to ask for changelog feedback in [#synapse-dev](https://matrix.to/#/#synapse-dev:matrix.org). ([\#15303](https://github.com/matrix-org/synapse/issues/15303)) +- Bump dtolnay/rust-toolchain from e12eda571dc9a5ee5d58eecf4738ec291c66f295 to fc3253060d0c959bea12a59f10f8391454a0b02d. ([\#15304](https://github.com/matrix-org/synapse/issues/15304)) +- Reject events with an invalid "mentions" property pert [MSC3952](https://github.com/matrix-org/matrix-spec-proposals/pull/3952). ([\#15311](https://github.com/matrix-org/synapse/issues/15311)) +- As an optimisation, use `TRUNCATE` on Postgres when clearing the user directory tables. ([\#15316](https://github.com/matrix-org/synapse/issues/15316)) +- Fix `.gitignore` rule for the Complement source tarball downloaded automatically by `complement.sh`. ([\#15319](https://github.com/matrix-org/synapse/issues/15319)) +- Bump serde from 1.0.157 to 1.0.158. ([\#15324](https://github.com/matrix-org/synapse/issues/15324)) +- Bump regex from 1.7.1 to 1.7.3. ([\#15325](https://github.com/matrix-org/synapse/issues/15325)) +- Bump types-pyopenssl from 23.0.0.4 to 23.1.0.0. ([\#15326](https://github.com/matrix-org/synapse/issues/15326)) +- Bump furo from 2022.12.7 to 2023.3.23. ([\#15327](https://github.com/matrix-org/synapse/issues/15327)) +- Bump ruff from 0.0.252 to 0.0.259. ([\#15328](https://github.com/matrix-org/synapse/issues/15328)) +- Bump cryptography from 40.0.0 to 40.0.1. ([\#15329](https://github.com/matrix-org/synapse/issues/15329)) +- Bump mypy-zope from 0.9.0 to 0.9.1. ([\#15330](https://github.com/matrix-org/synapse/issues/15330)) +- Speed up unit tests when using SQLite3. ([\#15334](https://github.com/matrix-org/synapse/issues/15334)) +- Speed up pydantic CI job. ([\#15339](https://github.com/matrix-org/synapse/issues/15339)) +- Speed up sample config CI job. ([\#15340](https://github.com/matrix-org/synapse/issues/15340)) +- Add experimental support for Unix sockets. Contributed by Jason Little. ([\#15353](https://github.com/matrix-org/synapse/issues/15353)) +- Add some clarification to the doc/comments regarding TCP replication. ([\#15354](https://github.com/matrix-org/synapse/issues/15354)) +- Fix copyright year in SSO footer template. ([\#15358](https://github.com/matrix-org/synapse/issues/15358)) +- Bump peaceiris/actions-gh-pages from 3.9.2 to 3.9.3. ([\#15369](https://github.com/matrix-org/synapse/issues/15369)) +- Bump serde from 1.0.158 to 1.0.159. ([\#15370](https://github.com/matrix-org/synapse/issues/15370)) +- Bump serde_json from 1.0.94 to 1.0.95. ([\#15371](https://github.com/matrix-org/synapse/issues/15371)) +- Speed up membership queries for users with forgotten rooms. ([\#15385](https://github.com/matrix-org/synapse/issues/15385)) + + Synapse 1.80.0 (2023-03-28) =========================== diff --git a/changelog.d/14978.feature b/changelog.d/14978.feature deleted file mode 100644 index 14f6fee65896..000000000000 --- a/changelog.d/14978.feature +++ /dev/null @@ -1 +0,0 @@ -Add the ability to enable/disable registrations when in the OIDC flow. \ No newline at end of file diff --git a/changelog.d/15113.misc b/changelog.d/15113.misc deleted file mode 100644 index 6917dd56528b..000000000000 --- a/changelog.d/15113.misc +++ /dev/null @@ -1 +0,0 @@ -Use `immutabledict` instead of `frozendict`. diff --git a/changelog.d/15243.feature b/changelog.d/15243.feature deleted file mode 100644 index c45e974c4c7b..000000000000 --- a/changelog.d/15243.feature +++ /dev/null @@ -1 +0,0 @@ -Add a primitive helper script for listing worker endpoints. \ No newline at end of file diff --git a/changelog.d/15265.misc b/changelog.d/15265.misc deleted file mode 100644 index 355c3cae2b7f..000000000000 --- a/changelog.d/15265.misc +++ /dev/null @@ -1 +0,0 @@ -Add developer documentation for the Federation Sender and add a documentation mechanism using Sphinx. \ No newline at end of file diff --git a/changelog.d/15280.misc b/changelog.d/15280.misc deleted file mode 100644 index 41d56b0cf0b4..000000000000 --- a/changelog.d/15280.misc +++ /dev/null @@ -1 +0,0 @@ -Make the pushers rely on the `device_id` instead of the `access_token_id` for various operations. diff --git a/changelog.d/15285.misc b/changelog.d/15285.misc deleted file mode 100644 index ad635ef3f29d..000000000000 --- a/changelog.d/15285.misc +++ /dev/null @@ -1 +0,0 @@ -Bump sentry-sdk from 1.15.0 to 1.17.0. diff --git a/changelog.d/15295.bugfix b/changelog.d/15295.bugfix deleted file mode 100644 index af43035383bf..000000000000 --- a/changelog.d/15295.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a long-standing bug where edits of non-`m.room.message` events would not be correctly bundled. diff --git a/changelog.d/15297.bugfix b/changelog.d/15297.bugfix deleted file mode 100644 index b5735fe4da85..000000000000 --- a/changelog.d/15297.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug introduced in Synapse v1.55.0 which could delay remote homeservers being able to decrypt encrypted messages sent by local users. diff --git a/changelog.d/15302.misc b/changelog.d/15302.misc deleted file mode 100644 index aadadf4b43ed..000000000000 --- a/changelog.d/15302.misc +++ /dev/null @@ -1 +0,0 @@ -Allow running the Twisted trunk job against other branches. diff --git a/changelog.d/15303.misc b/changelog.d/15303.misc deleted file mode 100644 index 977b9dcd02f8..000000000000 --- a/changelog.d/15303.misc +++ /dev/null @@ -1 +0,0 @@ -Remind the releaser to ask for changelog feedback in [#synapse-dev](https://matrix.to/#/#synapse-dev:matrix.org). diff --git a/changelog.d/15304.misc b/changelog.d/15304.misc deleted file mode 100644 index 38abb87a88bf..000000000000 --- a/changelog.d/15304.misc +++ /dev/null @@ -1 +0,0 @@ -Bump dtolnay/rust-toolchain from e12eda571dc9a5ee5d58eecf4738ec291c66f295 to fc3253060d0c959bea12a59f10f8391454a0b02d. diff --git a/changelog.d/15306.bugfix b/changelog.d/15306.bugfix deleted file mode 100644 index f5eb716f12c5..000000000000 --- a/changelog.d/15306.bugfix +++ /dev/null @@ -1,2 +0,0 @@ -Add a check to [SQLite port_db script](https://matrix-org.github.io/synapse/latest/postgres.html#porting-from-sqlite) -to ensure that the sqlite database passed to the script exists before trying to port from it. diff --git a/changelog.d/15309.bugfix b/changelog.d/15309.bugfix deleted file mode 100644 index 4d3fe4e4b10b..000000000000 --- a/changelog.d/15309.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug introduced in Synapse 1.76.0 where responses from worker deployments could include an internal `_INT_STREAM_POS` key. diff --git a/changelog.d/15311.misc b/changelog.d/15311.misc deleted file mode 100644 index ce03cb95238f..000000000000 --- a/changelog.d/15311.misc +++ /dev/null @@ -1 +0,0 @@ -Reject events with an invalid "mentions" property pert [MSC3952](https://github.com/matrix-org/matrix-spec-proposals/pull/3952). diff --git a/changelog.d/15314.feature b/changelog.d/15314.feature deleted file mode 100644 index 5ce0c029cee2..000000000000 --- a/changelog.d/15314.feature +++ /dev/null @@ -1 +0,0 @@ -Experimental support for passing One Time Key and device key requests to application services ([MSC3983](https://github.com/matrix-org/matrix-spec-proposals/pull/3983) and [MSC3984](https://github.com/matrix-org/matrix-spec-proposals/pull/3984)). diff --git a/changelog.d/15316.misc b/changelog.d/15316.misc deleted file mode 100644 index 1f408739f0bc..000000000000 --- a/changelog.d/15316.misc +++ /dev/null @@ -1 +0,0 @@ -As an optimisation, use `TRUNCATE` on Postgres when clearing the user directory tables. \ No newline at end of file diff --git a/changelog.d/15317.bugfix b/changelog.d/15317.bugfix deleted file mode 100644 index 194e4c46c64c..000000000000 --- a/changelog.d/15317.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a long-standing bug that Synpase only used the [legacy appservice routes](https://spec.matrix.org/v1.6/application-service-api/#legacy-routes). diff --git a/changelog.d/15319.misc b/changelog.d/15319.misc deleted file mode 100644 index 339e5b347dcf..000000000000 --- a/changelog.d/15319.misc +++ /dev/null @@ -1 +0,0 @@ -Fix `.gitignore` rule for the Complement source tarball downloaded automatically by `complement.sh`. \ No newline at end of file diff --git a/changelog.d/15321.feature b/changelog.d/15321.feature deleted file mode 100644 index 5ce0c029cee2..000000000000 --- a/changelog.d/15321.feature +++ /dev/null @@ -1 +0,0 @@ -Experimental support for passing One Time Key and device key requests to application services ([MSC3983](https://github.com/matrix-org/matrix-spec-proposals/pull/3983) and [MSC3984](https://github.com/matrix-org/matrix-spec-proposals/pull/3984)). diff --git a/changelog.d/15323.bugfix b/changelog.d/15323.bugfix deleted file mode 100644 index bc1ab35532fd..000000000000 --- a/changelog.d/15323.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a long-standing bug preventing users from joining rooms, that they had been unbanned from, over federation. Contributed by Nico. diff --git a/changelog.d/15324.misc b/changelog.d/15324.misc deleted file mode 100644 index 2908c82a2a4f..000000000000 --- a/changelog.d/15324.misc +++ /dev/null @@ -1 +0,0 @@ -Bump serde from 1.0.157 to 1.0.158. diff --git a/changelog.d/15325.misc b/changelog.d/15325.misc deleted file mode 100644 index 51af16f7e897..000000000000 --- a/changelog.d/15325.misc +++ /dev/null @@ -1 +0,0 @@ -Bump regex from 1.7.1 to 1.7.3. diff --git a/changelog.d/15326.misc b/changelog.d/15326.misc deleted file mode 100644 index 6238b7ff804d..000000000000 --- a/changelog.d/15326.misc +++ /dev/null @@ -1 +0,0 @@ -Bump types-pyopenssl from 23.0.0.4 to 23.1.0.0. diff --git a/changelog.d/15327.misc b/changelog.d/15327.misc deleted file mode 100644 index e5813f1aa985..000000000000 --- a/changelog.d/15327.misc +++ /dev/null @@ -1 +0,0 @@ -Bump furo from 2022.12.7 to 2023.3.23. diff --git a/changelog.d/15328.misc b/changelog.d/15328.misc deleted file mode 100644 index e3e595333229..000000000000 --- a/changelog.d/15328.misc +++ /dev/null @@ -1 +0,0 @@ -Bump ruff from 0.0.252 to 0.0.259. diff --git a/changelog.d/15329.misc b/changelog.d/15329.misc deleted file mode 100644 index eea38b7a42da..000000000000 --- a/changelog.d/15329.misc +++ /dev/null @@ -1 +0,0 @@ -Bump cryptography from 40.0.0 to 40.0.1. diff --git a/changelog.d/15330.misc b/changelog.d/15330.misc deleted file mode 100644 index d60e63483e1e..000000000000 --- a/changelog.d/15330.misc +++ /dev/null @@ -1 +0,0 @@ -Bump mypy-zope from 0.9.0 to 0.9.1. diff --git a/changelog.d/15331.feature b/changelog.d/15331.feature deleted file mode 100644 index b4c2eddc48ce..000000000000 --- a/changelog.d/15331.feature +++ /dev/null @@ -1 +0,0 @@ -Allow loading `/password_policy` endpoint on workers. \ No newline at end of file diff --git a/changelog.d/15332.bugfix b/changelog.d/15332.bugfix deleted file mode 100644 index ca6fb1d2fd79..000000000000 --- a/changelog.d/15332.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix bug in worker mode where on a rolling restart of workers the "typing" worker would consume 100% CPU until it got restarted. diff --git a/changelog.d/15334.misc b/changelog.d/15334.misc deleted file mode 100644 index 0c30818ed015..000000000000 --- a/changelog.d/15334.misc +++ /dev/null @@ -1 +0,0 @@ -Speed up unit tests when using SQLite3. diff --git a/changelog.d/15336.misc b/changelog.d/15336.misc deleted file mode 100644 index 39c9fc82e919..000000000000 --- a/changelog.d/15336.misc +++ /dev/null @@ -1 +0,0 @@ -Add developer documentation for the Federation Sender and add a documentation mechanism using Sphinx. diff --git a/changelog.d/15339.misc b/changelog.d/15339.misc deleted file mode 100644 index 626f7ef5c007..000000000000 --- a/changelog.d/15339.misc +++ /dev/null @@ -1 +0,0 @@ -Speed up pydantic CI job. diff --git a/changelog.d/15340.misc b/changelog.d/15340.misc deleted file mode 100644 index e2fe35d7c591..000000000000 --- a/changelog.d/15340.misc +++ /dev/null @@ -1 +0,0 @@ -Speed up sample config CI job. diff --git a/changelog.d/15341.doc b/changelog.d/15341.doc deleted file mode 100644 index d85c0fd2c490..000000000000 --- a/changelog.d/15341.doc +++ /dev/null @@ -1 +0,0 @@ -Fix a typo in login requests ratelimit defaults. diff --git a/changelog.d/15349.bugfix b/changelog.d/15349.bugfix deleted file mode 100644 index 65ea7ae7eb1e..000000000000 --- a/changelog.d/15349.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a long-standing bug where some to_device messages could be dropped when using workers. diff --git a/changelog.d/15351.bugfix b/changelog.d/15351.bugfix deleted file mode 100644 index e68023c6716f..000000000000 --- a/changelog.d/15351.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug introduced in Synapse 1.70.0 where the background sync from a faster join could spin for hours when one of the events involved had been marked for backoff. diff --git a/changelog.d/15352.bugfix b/changelog.d/15352.bugfix deleted file mode 100644 index 36d6615cac3c..000000000000 --- a/changelog.d/15352.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix missing app variable in mail subject for password resets. Contributed by Cyberes. diff --git a/changelog.d/15353.misc b/changelog.d/15353.misc deleted file mode 100644 index 23927fea8ff2..000000000000 --- a/changelog.d/15353.misc +++ /dev/null @@ -1 +0,0 @@ -Add experimental support for Unix sockets. Contributed by Jason Little. diff --git a/changelog.d/15354.misc b/changelog.d/15354.misc deleted file mode 100644 index 862444edfbaf..000000000000 --- a/changelog.d/15354.misc +++ /dev/null @@ -1 +0,0 @@ -Add some clarification to the doc/comments regarding TCP replication. diff --git a/changelog.d/15358.misc b/changelog.d/15358.misc deleted file mode 100644 index daf261a9f16d..000000000000 --- a/changelog.d/15358.misc +++ /dev/null @@ -1 +0,0 @@ -Fix copyright year in SSO footer template. diff --git a/changelog.d/15369.misc b/changelog.d/15369.misc deleted file mode 100644 index da990285cbab..000000000000 --- a/changelog.d/15369.misc +++ /dev/null @@ -1 +0,0 @@ -Bump peaceiris/actions-gh-pages from 3.9.2 to 3.9.3. diff --git a/changelog.d/15370.misc b/changelog.d/15370.misc deleted file mode 100644 index 23a18eaf07b8..000000000000 --- a/changelog.d/15370.misc +++ /dev/null @@ -1 +0,0 @@ -Bump serde from 1.0.158 to 1.0.159. diff --git a/changelog.d/15371.misc b/changelog.d/15371.misc deleted file mode 100644 index c21deede05a6..000000000000 --- a/changelog.d/15371.misc +++ /dev/null @@ -1 +0,0 @@ -Bump serde_json from 1.0.94 to 1.0.95. diff --git a/changelog.d/15381.feature b/changelog.d/15381.feature deleted file mode 100644 index ba5b578564e9..000000000000 --- a/changelog.d/15381.feature +++ /dev/null @@ -1 +0,0 @@ -Build Debian packages for Ubuntu 23.04 (Lunar Lobster). diff --git a/changelog.d/15383.bugfix b/changelog.d/15383.bugfix deleted file mode 100644 index 28c66ef45461..000000000000 --- a/changelog.d/15383.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a rare bug introduced in Synapse 1.66.0 where initial syncs would fail when the user had been kicked from a faster joined room that had not finished syncing. diff --git a/changelog.d/15385.misc b/changelog.d/15385.misc deleted file mode 100644 index 76350c398d6f..000000000000 --- a/changelog.d/15385.misc +++ /dev/null @@ -1 +0,0 @@ -Speed up membership queries for users with forgotten rooms. diff --git a/changelog.d/15386.doc b/changelog.d/15386.doc deleted file mode 100644 index 5c032193aeef..000000000000 --- a/changelog.d/15386.doc +++ /dev/null @@ -1 +0,0 @@ -Note that Synapse 1.74 queued a rebuild of the user directory tables. diff --git a/debian/changelog b/debian/changelog index 98366d49166b..c3bea01c050a 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.81.0~rc1) stable; urgency=medium + + * New Synapse release 1.81.0rc1. + + -- Synapse Packaging team Tue, 04 Apr 2023 14:29:03 +0100 + matrix-synapse-py3 (1.80.0) stable; urgency=medium * New Synapse release 1.80.0. diff --git a/pyproject.toml b/pyproject.toml index dc5522c4b516..3b38d8370c84 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -89,7 +89,7 @@ manifest-path = "rust/Cargo.toml" [tool.poetry] name = "matrix-synapse" -version = "1.80.0" +version = "1.81.0rc1" description = "Homeserver for the Matrix decentralised comms protocol" authors = ["Matrix.org Team and Contributors "] license = "Apache-2.0" From 6d103373e277b428389585af165fdccc34c4760f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 4 Apr 2023 14:38:33 +0100 Subject: [PATCH 7/9] Update changelog --- CHANGES.md | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 1f77fab96078..2a6ee1490d74 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,19 @@ Synapse 1.81.0rc1 (2023-04-04) ============================== +Synapse now attempts the versioned appservice paths before falling back to the +[legacy paths](https://spec.matrix.org/v1.6/application-service-api/#legacy-routes). +Usage of the legacy routes should be considered deprecated. + +Additionally, Synapse has supported sending the application service access token +via [the `Authorization` header](https://spec.matrix.org/v1.6/application-service-api/#authorization) +since v1.70.0. For backwards compatibility it is *also* sent as the `access_token` +query parameter. This is insecure and should be considered deprecated. + +A future version of Synapse (v1.88.0 or later) will remove support for legacy +application service routes and query parameter authorization. + + Features -------- @@ -8,6 +21,7 @@ Features - Add a primitive helper script for listing worker endpoints. ([\#15243](https://github.com/matrix-org/synapse/issues/15243)) - Experimental support for passing One Time Key and device key requests to application services ([MSC3983](https://github.com/matrix-org/matrix-spec-proposals/pull/3983) and [MSC3984](https://github.com/matrix-org/matrix-spec-proposals/pull/3984)). ([\#15314](https://github.com/matrix-org/synapse/issues/15314), [\#15321](https://github.com/matrix-org/synapse/issues/15321)) - Allow loading `/password_policy` endpoint on workers. ([\#15331](https://github.com/matrix-org/synapse/issues/15331)) +- Add experimental support for Unix sockets. Contributed by Jason Little. ([\#15353](https://github.com/matrix-org/synapse/issues/15353)) - Build Debian packages for Ubuntu 23.04 (Lunar Lobster). ([\#15381](https://github.com/matrix-org/synapse/issues/15381)) @@ -20,7 +34,7 @@ Bugfixes to ensure that the sqlite database passed to the script exists before trying to port from it. ([\#15306](https://github.com/matrix-org/synapse/issues/15306)) - Fix a bug introduced in Synapse 1.76.0 where responses from worker deployments could include an internal `_INT_STREAM_POS` key. ([\#15309](https://github.com/matrix-org/synapse/issues/15309)) - Fix a long-standing bug that Synpase only used the [legacy appservice routes](https://spec.matrix.org/v1.6/application-service-api/#legacy-routes). ([\#15317](https://github.com/matrix-org/synapse/issues/15317)) -- Fix a long-standing bug preventing users from joining rooms, that they had been unbanned from, over federation. Contributed by Nico. ([\#15323](https://github.com/matrix-org/synapse/issues/15323)) +- Fix a long-standing bug preventing users from rejoining rooms after being banned and unbanned over federation. Contributed by Nico. ([\#15323](https://github.com/matrix-org/synapse/issues/15323)) - Fix bug in worker mode where on a rolling restart of workers the "typing" worker would consume 100% CPU until it got restarted. ([\#15332](https://github.com/matrix-org/synapse/issues/15332)) - Fix a long-standing bug where some to_device messages could be dropped when using workers. ([\#15349](https://github.com/matrix-org/synapse/issues/15349)) - Fix a bug introduced in Synapse 1.70.0 where the background sync from a faster join could spin for hours when one of the events involved had been marked for backoff. ([\#15351](https://github.com/matrix-org/synapse/issues/15351)) @@ -32,6 +46,7 @@ Improved Documentation ---------------------- - Fix a typo in login requests ratelimit defaults. ([\#15341](https://github.com/matrix-org/synapse/issues/15341)) +- Add some clarification to the doc/comments regarding TCP replication. ([\#15354](https://github.com/matrix-org/synapse/issues/15354)) - Note that Synapse 1.74 queued a rebuild of the user directory tables. ([\#15386](https://github.com/matrix-org/synapse/issues/15386)) @@ -45,7 +60,7 @@ Internal Changes - Allow running the Twisted trunk job against other branches. ([\#15302](https://github.com/matrix-org/synapse/issues/15302)) - Remind the releaser to ask for changelog feedback in [#synapse-dev](https://matrix.to/#/#synapse-dev:matrix.org). ([\#15303](https://github.com/matrix-org/synapse/issues/15303)) - Bump dtolnay/rust-toolchain from e12eda571dc9a5ee5d58eecf4738ec291c66f295 to fc3253060d0c959bea12a59f10f8391454a0b02d. ([\#15304](https://github.com/matrix-org/synapse/issues/15304)) -- Reject events with an invalid "mentions" property pert [MSC3952](https://github.com/matrix-org/matrix-spec-proposals/pull/3952). ([\#15311](https://github.com/matrix-org/synapse/issues/15311)) +- Reject events with an invalid "mentions" property per [MSC3952](https://github.com/matrix-org/matrix-spec-proposals/pull/3952). ([\#15311](https://github.com/matrix-org/synapse/issues/15311)) - As an optimisation, use `TRUNCATE` on Postgres when clearing the user directory tables. ([\#15316](https://github.com/matrix-org/synapse/issues/15316)) - Fix `.gitignore` rule for the Complement source tarball downloaded automatically by `complement.sh`. ([\#15319](https://github.com/matrix-org/synapse/issues/15319)) - Bump serde from 1.0.157 to 1.0.158. ([\#15324](https://github.com/matrix-org/synapse/issues/15324)) @@ -58,8 +73,6 @@ Internal Changes - Speed up unit tests when using SQLite3. ([\#15334](https://github.com/matrix-org/synapse/issues/15334)) - Speed up pydantic CI job. ([\#15339](https://github.com/matrix-org/synapse/issues/15339)) - Speed up sample config CI job. ([\#15340](https://github.com/matrix-org/synapse/issues/15340)) -- Add experimental support for Unix sockets. Contributed by Jason Little. ([\#15353](https://github.com/matrix-org/synapse/issues/15353)) -- Add some clarification to the doc/comments regarding TCP replication. ([\#15354](https://github.com/matrix-org/synapse/issues/15354)) - Fix copyright year in SSO footer template. ([\#15358](https://github.com/matrix-org/synapse/issues/15358)) - Bump peaceiris/actions-gh-pages from 3.9.2 to 3.9.3. ([\#15369](https://github.com/matrix-org/synapse/issues/15369)) - Bump serde from 1.0.158 to 1.0.159. ([\#15370](https://github.com/matrix-org/synapse/issues/15370)) @@ -463,7 +476,7 @@ Those who are `poetry install`ing from source using our lockfile should ensure t Notes on faster joins --------------------- -The faster joins project sees the most benefit when joining a room with a large number of members (joined or historical). We expect it to be particularly useful for joining large public rooms like the [Matrix HQ](https://matrix.to/#/#matrix:matrix.org) or [Synapse Admins](https://matrix.to/#/#synapse:matrix.org) rooms. +The faster joins project sees the most benefit when joining a room with a large number of members (joined or historical). We expect it to be particularly useful for joining large public rooms like the [Matrix HQ](https://matrix.to/#/#matrix:matrix.org) or [Synapse Admins](https://matrix.to/#/#synapse:matrix.org) rooms. After a faster join, Synapse considers that room "partially joined". In this state, you should be able to From 6b23d74ad160d96e06bcc5b62acad56ade06bf6e Mon Sep 17 00:00:00 2001 From: Shay Date: Tue, 4 Apr 2023 13:16:08 -0700 Subject: [PATCH 8/9] Delete server-side backup keys when deactivating an account. (#15181) --- changelog.d/15181.bugfix | 1 + synapse/_scripts/synapse_port_db.py | 2 + synapse/handlers/deactivate_account.py | 3 + .../storage/databases/main/e2e_room_keys.py | 114 ++++++++++++- ..._e2e_backup_keys_for_deactivated_users.sql | 17 ++ tests/rest/client/test_account.py | 157 ++++++++++++++++++ 6 files changed, 291 insertions(+), 3 deletions(-) create mode 100644 changelog.d/15181.bugfix create mode 100644 synapse/storage/schema/main/delta/74/04_delete_e2e_backup_keys_for_deactivated_users.sql diff --git a/changelog.d/15181.bugfix b/changelog.d/15181.bugfix new file mode 100644 index 000000000000..191bb6f611a7 --- /dev/null +++ b/changelog.d/15181.bugfix @@ -0,0 +1 @@ +Delete server-side backup keys when deactivating an account. \ No newline at end of file diff --git a/synapse/_scripts/synapse_port_db.py b/synapse/_scripts/synapse_port_db.py index 1dcb397ba456..a58ae2a308c6 100755 --- a/synapse/_scripts/synapse_port_db.py +++ b/synapse/_scripts/synapse_port_db.py @@ -59,6 +59,7 @@ from synapse.storage.databases.main.client_ips import ClientIpBackgroundUpdateStore from synapse.storage.databases.main.deviceinbox import DeviceInboxBackgroundUpdateStore from synapse.storage.databases.main.devices import DeviceBackgroundUpdateStore +from synapse.storage.databases.main.e2e_room_keys import EndToEndRoomKeyBackgroundStore from synapse.storage.databases.main.end_to_end_keys import EndToEndKeyBackgroundStore from synapse.storage.databases.main.event_push_actions import EventPushActionsStore from synapse.storage.databases.main.events_bg_updates import ( @@ -225,6 +226,7 @@ class Store( MainStateBackgroundUpdateStore, UserDirectoryBackgroundUpdateStore, EndToEndKeyBackgroundStore, + EndToEndRoomKeyBackgroundStore, StatsStore, AccountDataWorkerStore, PushRuleStore, diff --git a/synapse/handlers/deactivate_account.py b/synapse/handlers/deactivate_account.py index d31263c717b5..bd5867491baf 100644 --- a/synapse/handlers/deactivate_account.py +++ b/synapse/handlers/deactivate_account.py @@ -176,6 +176,9 @@ async def deactivate_account( # Remove account data (including ignored users and push rules). await self.store.purge_account_data_for_user(user_id) + # Delete any server-side backup keys + await self.store.bulk_delete_backup_keys_and_versions_for_user(user_id) + # Let modules know the user has been deactivated. await self._third_party_rules.on_user_deactivation_status_changed( user_id, diff --git a/synapse/storage/databases/main/e2e_room_keys.py b/synapse/storage/databases/main/e2e_room_keys.py index 9f8d2e4bea65..d01f28cc803f 100644 --- a/synapse/storage/databases/main/e2e_room_keys.py +++ b/synapse/storage/databases/main/e2e_room_keys.py @@ -13,17 +13,24 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Dict, Iterable, Mapping, Optional, Tuple, cast +from typing import TYPE_CHECKING, Dict, Iterable, Mapping, Optional, Tuple, cast from typing_extensions import Literal, TypedDict from synapse.api.errors import StoreError from synapse.logging.opentracing import log_kv, trace from synapse.storage._base import SQLBaseStore, db_to_json -from synapse.storage.database import LoggingTransaction +from synapse.storage.database import ( + DatabasePool, + LoggingDatabaseConnection, + LoggingTransaction, +) from synapse.types import JsonDict, JsonSerializable, StreamKeyType from synapse.util import json_encoder +if TYPE_CHECKING: + from synapse.server import HomeServer + class RoomKey(TypedDict): """`KeyBackupData` in the Matrix spec. @@ -37,7 +44,82 @@ class RoomKey(TypedDict): session_data: JsonSerializable -class EndToEndRoomKeyStore(SQLBaseStore): +class EndToEndRoomKeyBackgroundStore(SQLBaseStore): + def __init__( + self, + database: DatabasePool, + db_conn: LoggingDatabaseConnection, + hs: "HomeServer", + ): + super().__init__(database, db_conn, hs) + + self.db_pool.updates.register_background_update_handler( + "delete_e2e_backup_keys_for_deactivated_users", + self._delete_e2e_backup_keys_for_deactivated_users, + ) + + def _delete_keys_txn(self, txn: LoggingTransaction, user_id: str) -> None: + self.db_pool.simple_delete_txn( + txn, + table="e2e_room_keys", + keyvalues={"user_id": user_id}, + ) + + self.db_pool.simple_delete_txn( + txn, + table="e2e_room_keys_versions", + keyvalues={"user_id": user_id}, + ) + + async def _delete_e2e_backup_keys_for_deactivated_users( + self, progress: JsonDict, batch_size: int + ) -> int: + """ + Retroactively purges account data for users that have already been deactivated. + Gets run as a background update caused by a schema delta. + """ + + last_user: str = progress.get("last_user", "") + + def _delete_backup_keys_for_deactivated_users_txn( + txn: LoggingTransaction, + ) -> int: + sql = """ + SELECT name FROM users + WHERE deactivated = ? and name > ? + ORDER BY name ASC + LIMIT ? + """ + + txn.execute(sql, (1, last_user, batch_size)) + users = [row[0] for row in txn] + + for user in users: + self._delete_keys_txn(txn, user) + + if users: + self.db_pool.updates._background_update_progress_txn( + txn, + "delete_e2e_backup_keys_for_deactivated_users", + {"last_user": users[-1]}, + ) + + return len(users) + + number_deleted = await self.db_pool.runInteraction( + "_delete_backup_keys_for_deactivated_users", + _delete_backup_keys_for_deactivated_users_txn, + ) + + if number_deleted < batch_size: + await self.db_pool.updates._end_background_update( + "delete_e2e_backup_keys_for_deactivated_users" + ) + + return number_deleted + + +class EndToEndRoomKeyStore(EndToEndRoomKeyBackgroundStore): """The store for end to end room key backups. See https://spec.matrix.org/v1.1/client-server-api/#server-side-key-backups @@ -550,3 +632,29 @@ def _delete_e2e_room_keys_version_txn(txn: LoggingTransaction) -> None: await self.db_pool.runInteraction( "delete_e2e_room_keys_version", _delete_e2e_room_keys_version_txn ) + + async def bulk_delete_backup_keys_and_versions_for_user(self, user_id: str) -> None: + """ + Bulk deletes all backup room keys and versions for a given user. + + Args: + user_id: the user whose backup keys and versions we're deleting + """ + + def _delete_all_e2e_room_keys_and_versions_txn(txn: LoggingTransaction) -> None: + self.db_pool.simple_delete_txn( + txn, + table="e2e_room_keys", + keyvalues={"user_id": user_id}, + ) + + self.db_pool.simple_delete_txn( + txn, + table="e2e_room_keys_versions", + keyvalues={"user_id": user_id}, + ) + + await self.db_pool.runInteraction( + "delete_all_e2e_room_keys_and_versions", + _delete_all_e2e_room_keys_and_versions_txn, + ) diff --git a/synapse/storage/schema/main/delta/74/04_delete_e2e_backup_keys_for_deactivated_users.sql b/synapse/storage/schema/main/delta/74/04_delete_e2e_backup_keys_for_deactivated_users.sql new file mode 100644 index 000000000000..a194f4cecef3 --- /dev/null +++ b/synapse/storage/schema/main/delta/74/04_delete_e2e_backup_keys_for_deactivated_users.sql @@ -0,0 +1,17 @@ +/* Copyright 2023 The Matrix.org Foundation C.I.C + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +INSERT INTO background_updates (ordering, update_name, progress_json) VALUES + (7404, 'delete_e2e_backup_keys_for_deactivated_users', '{}'); \ No newline at end of file diff --git a/tests/rest/client/test_account.py b/tests/rest/client/test_account.py index 7f675c44a278..ac19f3c6daef 100644 --- a/tests/rest/client/test_account.py +++ b/tests/rest/client/test_account.py @@ -474,6 +474,163 @@ def test_pending_invites(self) -> None: self.assertEqual(len(memberships), 1, memberships) self.assertEqual(memberships[0].room_id, room_id, memberships) + def test_deactivate_account_deletes_server_side_backup_keys(self) -> None: + key_handler = self.hs.get_e2e_room_keys_handler() + room_keys = { + "rooms": { + "!abc:matrix.org": { + "sessions": { + "c0ff33": { + "first_message_index": 1, + "forwarded_count": 1, + "is_verified": False, + "session_data": "SSBBTSBBIEZJU0gK", + } + } + } + } + } + + user_id = self.register_user("missPiggy", "test") + tok = self.login("missPiggy", "test") + + # add some backup keys/versions + version = self.get_success( + key_handler.create_version( + user_id, + { + "algorithm": "m.megolm_backup.v1", + "auth_data": "first_version_auth_data", + }, + ) + ) + + self.get_success(key_handler.upload_room_keys(user_id, version, room_keys)) + + version2 = self.get_success( + key_handler.create_version( + user_id, + { + "algorithm": "m.megolm_backup.v1", + "auth_data": "second_version_auth_data", + }, + ) + ) + + self.get_success(key_handler.upload_room_keys(user_id, version2, room_keys)) + + self.deactivate(user_id, tok) + store = self.hs.get_datastores().main + + # Check that the user has been marked as deactivated. + self.assertTrue(self.get_success(store.get_user_deactivated_status(user_id))) + + # Check that there are no entries in 'e2e_room_keys` and `e2e_room_keys_versions` + res = self.get_success( + self.hs.get_datastores().main.db_pool.simple_select_list( + "e2e_room_keys", {"user_id": user_id}, "*", "simple_select" + ) + ) + self.assertEqual(len(res), 0) + + res2 = self.get_success( + self.hs.get_datastores().main.db_pool.simple_select_list( + "e2e_room_keys_versions", {"user_id": user_id}, "*", "simple_select" + ) + ) + self.assertEqual(len(res2), 0) + + def test_background_update_deletes_deactivated_users_server_side_backup_keys( + self, + ) -> None: + key_handler = self.hs.get_e2e_room_keys_handler() + room_keys = { + "rooms": { + "!abc:matrix.org": { + "sessions": { + "c0ff33": { + "first_message_index": 1, + "forwarded_count": 1, + "is_verified": False, + "session_data": "SSBBTSBBIEZJU0gK", + } + } + } + } + } + self.store = self.hs.get_datastores().main + + # create a bunch of users and add keys for them + users = [] + for i in range(0, 20): + user_id = self.register_user("missPiggy" + str(i), "test") + users.append((user_id,)) + + # add some backup keys/versions + version = self.get_success( + key_handler.create_version( + user_id, + { + "algorithm": "m.megolm_backup.v1", + "auth_data": str(i) + "_version_auth_data", + }, + ) + ) + + self.get_success(key_handler.upload_room_keys(user_id, version, room_keys)) + + version2 = self.get_success( + key_handler.create_version( + user_id, + { + "algorithm": "m.megolm_backup.v1", + "auth_data": str(i) + "_version_auth_data", + }, + ) + ) + + self.get_success(key_handler.upload_room_keys(user_id, version2, room_keys)) + + # deactivate most of the users by editing DB + self.get_success( + self.store.db_pool.simple_update_many( + table="users", + key_names=("name",), + key_values=users[0:18], + value_names=("deactivated",), + value_values=[(1,) for i in range(1, 19)], + desc="", + ) + ) + + # run background update + self.get_success( + self.store.db_pool.simple_insert( + "background_updates", + { + "update_name": "delete_e2e_backup_keys_for_deactivated_users", + "progress_json": "{}", + }, + ) + ) + self.store.db_pool.updates._all_done = False + self.wait_for_background_updates() + + # check that keys are deleted for the deactivated users but not the others + res = self.get_success( + self.hs.get_datastores().main.db_pool.simple_select_list( + "e2e_room_keys", None, ("user_id",), "simple_select" + ) + ) + self.assertEqual(len(res), 4) + + res2 = self.get_success( + self.hs.get_datastores().main.db_pool.simple_select_list( + "e2e_room_keys_versions", None, ("user_id",), "simple_select" + ) + ) + self.assertEqual(len(res2), 4) + def deactivate(self, user_id: str, tok: str) -> None: request_data = { "auth": { From 83649b891db5fcca7b96c606293925bc47358730 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Wed, 5 Apr 2023 14:42:46 -0400 Subject: [PATCH 9/9] Implement MSC3989 to redact the origin field. (#15393) This will be done in a future room version, for now an unstable room version is added which redacts the origin field. --- changelog.d/15393.misc | 1 + synapse/api/room_versions.py | 37 ++++++++++++++++++++++++++++++++++++ synapse/events/utils.py | 5 ++++- tests/events/test_utils.py | 7 +++++++ 4 files changed, 49 insertions(+), 1 deletion(-) create mode 100644 changelog.d/15393.misc diff --git a/changelog.d/15393.misc b/changelog.d/15393.misc new file mode 100644 index 000000000000..24483c8d785c --- /dev/null +++ b/changelog.d/15393.misc @@ -0,0 +1 @@ +Implement [MSC3989](https://github.com/matrix-org/matrix-spec-proposals/pull/3989) redaction algorithm. diff --git a/synapse/api/room_versions.py b/synapse/api/room_versions.py index c397920fe54e..bc15f2d0631f 100644 --- a/synapse/api/room_versions.py +++ b/synapse/api/room_versions.py @@ -104,6 +104,8 @@ class RoomVersion: # support the flag. Unknown flags are ignored by the evaluator, making conditions # fail if used. msc3931_push_features: Tuple[str, ...] # values from PushRuleRoomFlag + # MSC3989: Redact the origin field. + msc3989_redaction_rules: bool class RoomVersions: @@ -125,6 +127,7 @@ class RoomVersions: msc3787_knock_restricted_join_rule=False, msc3667_int_only_power_levels=False, msc3931_push_features=(), + msc3989_redaction_rules=False, ) V2 = RoomVersion( "2", @@ -144,6 +147,7 @@ class RoomVersions: msc3787_knock_restricted_join_rule=False, msc3667_int_only_power_levels=False, msc3931_push_features=(), + msc3989_redaction_rules=False, ) V3 = RoomVersion( "3", @@ -163,6 +167,7 @@ class RoomVersions: msc3787_knock_restricted_join_rule=False, msc3667_int_only_power_levels=False, msc3931_push_features=(), + msc3989_redaction_rules=False, ) V4 = RoomVersion( "4", @@ -182,6 +187,7 @@ class RoomVersions: msc3787_knock_restricted_join_rule=False, msc3667_int_only_power_levels=False, msc3931_push_features=(), + msc3989_redaction_rules=False, ) V5 = RoomVersion( "5", @@ -201,6 +207,7 @@ class RoomVersions: msc3787_knock_restricted_join_rule=False, msc3667_int_only_power_levels=False, msc3931_push_features=(), + msc3989_redaction_rules=False, ) V6 = RoomVersion( "6", @@ -220,6 +227,7 @@ class RoomVersions: msc3787_knock_restricted_join_rule=False, msc3667_int_only_power_levels=False, msc3931_push_features=(), + msc3989_redaction_rules=False, ) MSC2176 = RoomVersion( "org.matrix.msc2176", @@ -239,6 +247,7 @@ class RoomVersions: msc3787_knock_restricted_join_rule=False, msc3667_int_only_power_levels=False, msc3931_push_features=(), + msc3989_redaction_rules=False, ) V7 = RoomVersion( "7", @@ -258,6 +267,7 @@ class RoomVersions: msc3787_knock_restricted_join_rule=False, msc3667_int_only_power_levels=False, msc3931_push_features=(), + msc3989_redaction_rules=False, ) V8 = RoomVersion( "8", @@ -277,6 +287,7 @@ class RoomVersions: msc3787_knock_restricted_join_rule=False, msc3667_int_only_power_levels=False, msc3931_push_features=(), + msc3989_redaction_rules=False, ) V9 = RoomVersion( "9", @@ -296,6 +307,7 @@ class RoomVersions: msc3787_knock_restricted_join_rule=False, msc3667_int_only_power_levels=False, msc3931_push_features=(), + msc3989_redaction_rules=False, ) MSC3787 = RoomVersion( "org.matrix.msc3787", @@ -315,6 +327,7 @@ class RoomVersions: msc3787_knock_restricted_join_rule=True, msc3667_int_only_power_levels=False, msc3931_push_features=(), + msc3989_redaction_rules=False, ) V10 = RoomVersion( "10", @@ -334,6 +347,7 @@ class RoomVersions: msc3787_knock_restricted_join_rule=True, msc3667_int_only_power_levels=True, msc3931_push_features=(), + msc3989_redaction_rules=False, ) MSC2716v4 = RoomVersion( "org.matrix.msc2716v4", @@ -353,6 +367,7 @@ class RoomVersions: msc3787_knock_restricted_join_rule=False, msc3667_int_only_power_levels=False, msc3931_push_features=(), + msc3989_redaction_rules=False, ) MSC1767v10 = RoomVersion( # MSC1767 (Extensible Events) based on room version "10" @@ -373,6 +388,27 @@ class RoomVersions: msc3787_knock_restricted_join_rule=True, msc3667_int_only_power_levels=True, msc3931_push_features=(PushRuleRoomFlag.EXTENSIBLE_EVENTS,), + msc3989_redaction_rules=False, + ) + MSC3989 = RoomVersion( + "org.matrix.msc3989", + RoomDisposition.UNSTABLE, + EventFormatVersions.ROOM_V4_PLUS, + StateResolutionVersions.V2, + enforce_key_validity=True, + special_case_aliases_auth=False, + strict_canonicaljson=True, + limit_notifications_power_levels=True, + msc2176_redaction_rules=False, + msc3083_join_rules=True, + msc3375_redaction_rules=True, + msc2403_knocking=True, + msc2716_historical=False, + msc2716_redactions=False, + msc3787_knock_restricted_join_rule=True, + msc3667_int_only_power_levels=True, + msc3931_push_features=(), + msc3989_redaction_rules=True, ) @@ -392,6 +428,7 @@ class RoomVersions: RoomVersions.MSC3787, RoomVersions.V10, RoomVersions.MSC2716v4, + RoomVersions.MSC3989, ) } diff --git a/synapse/events/utils.py b/synapse/events/utils.py index c14c7791db27..1d5d7491cd83 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -106,7 +106,6 @@ def prune_event_dict(room_version: RoomVersion, event_dict: JsonDict) -> JsonDic "depth", "prev_events", "auth_events", - "origin", "origin_server_ts", ] @@ -114,6 +113,10 @@ def prune_event_dict(room_version: RoomVersion, event_dict: JsonDict) -> JsonDic if not room_version.msc2176_redaction_rules: allowed_keys.extend(["prev_state", "membership"]) + # Room versions before MSC3989 kept the origin field. + if not room_version.msc3989_redaction_rules: + allowed_keys.append("origin") + event_type = event_dict["type"] new_content = {} diff --git a/tests/events/test_utils.py b/tests/events/test_utils.py index 4174a237ec84..c35f58f46213 100644 --- a/tests/events/test_utils.py +++ b/tests/events/test_utils.py @@ -143,6 +143,13 @@ def test_basic_keys(self) -> None: room_version=RoomVersions.MSC2176, ) + # As of MSC3989 we now redact the origin key. + self.run_test( + {"type": "A", "origin": "example.com"}, + {"type": "A", "content": {}, "signatures": {}, "unsigned": {}}, + room_version=RoomVersions.MSC3989, + ) + def test_unsigned(self) -> None: """Ensure that unsigned properties get stripped (except age_ts and replaces_state).""" self.run_test(