Skip to content
This repository has been archived by the owner on Apr 26, 2024. It is now read-only.

Commit

Permalink
Merge commit 'e45b83411' into anoa/dinsic_release_1_21_x
Browse files Browse the repository at this point in the history
* commit 'e45b83411':
  Add types to async_helpers (#8260)
  Fix mypy error on develop (#8282)
  Include method in thumbnail media name (#7124)
  Add types to StreamToken and RoomStreamToken (#8279)
  Add a config option for validating 'next_link' parameters against a domain whitelist (#8275)
  Clean up types for PaginationConfig (#8250)
  Use the right constructor for log records (#8278)
  Fix `MultiWriterIdGenerator.current_position`. (#8257)
  • Loading branch information
anoadragon453 committed Oct 20, 2020
2 parents 8228b9e + e45b834 commit 82c379c
Show file tree
Hide file tree
Showing 28 changed files with 553 additions and 242 deletions.
1 change: 1 addition & 0 deletions changelog.d/7124.bugfix
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Fix a bug in the media repository where remote thumbnails with the same size but different crop methods would overwrite each other. Contributed by @deepbluev7.
1 change: 1 addition & 0 deletions changelog.d/8250.misc
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Clean up type hints for `PaginationConfig`.
1 change: 1 addition & 0 deletions changelog.d/8257.misc
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Fix non-user visible bug in implementation of `MultiWriterIdGenerator.get_current_token_for_writer`.
1 change: 1 addition & 0 deletions changelog.d/8260.misc
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Add type hints to `synapse.util.async_helpers`.
1 change: 1 addition & 0 deletions changelog.d/8278.bugfix
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Fix a bug which cause the logging system to report errors, if `DEBUG` was enabled and no `context` filter was applied.
1 change: 1 addition & 0 deletions changelog.d/8279.misc
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Add type hints to `StreamToken` and `RoomStreamToken` classes.
1 change: 1 addition & 0 deletions changelog.d/8282.misc
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Clean up type hints for `PaginationConfig`.
3 changes: 2 additions & 1 deletion mypy.ini
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ files =
synapse/http/federation/well_known_resolver.py,
synapse/http/server.py,
synapse/http/site.py,
synapse/logging/,
synapse/logging,
synapse/metrics,
synapse/module_api,
synapse/notifier.py,
Expand All @@ -54,6 +54,7 @@ files =
synapse/storage/util,
synapse/streams,
synapse/types.py,
synapse/util/async_helpers.py,
synapse/util/caches/descriptors.py,
synapse/util/caches/stream_change_cache.py,
synapse/util/metrics.py,
Expand Down
11 changes: 5 additions & 6 deletions synapse/handlers/initial_sync.py
Original file line number Diff line number Diff line change
Expand Up @@ -116,14 +116,13 @@ async def _snapshot_all_rooms(
now_token = self.hs.get_event_sources().get_current_token()

presence_stream = self.hs.get_event_sources().sources["presence"]
pagination_config = PaginationConfig(from_token=now_token)
presence, _ = await presence_stream.get_pagination_rows(
user, pagination_config.get_source_config("presence"), None
presence, _ = await presence_stream.get_new_events(
user, from_key=None, include_offline=False
)

receipt_stream = self.hs.get_event_sources().sources["receipt"]
receipt, _ = await receipt_stream.get_pagination_rows(
user, pagination_config.get_source_config("receipt"), None
joined_rooms = [r.room_id for r in room_list if r.membership == Membership.JOIN]
receipt = await self.store.get_linearized_receipts_for_rooms(
joined_rooms, to_key=int(now_token.receipt_key),
)

tags_by_room = await self.store.get_tags_for_user(user_id)
Expand Down
49 changes: 27 additions & 22 deletions synapse/handlers/pagination.py
Original file line number Diff line number Diff line change
Expand Up @@ -335,20 +335,16 @@ async def get_messages(
user_id = requester.user.to_string()

if pagin_config.from_token:
room_token = pagin_config.from_token.room_key
from_token = pagin_config.from_token
else:
pagin_config.from_token = (
self.hs.get_event_sources().get_current_token_for_pagination()
)
room_token = pagin_config.from_token.room_key

room_token = RoomStreamToken.parse(room_token)
from_token = self.hs.get_event_sources().get_current_token_for_pagination()

pagin_config.from_token = pagin_config.from_token.copy_and_replace(
"room_key", str(room_token)
)
if pagin_config.limit is None:
# This shouldn't happen as we've set a default limit before this
# gets called.
raise Exception("limit not set")

source_config = pagin_config.get_source_config("room")
room_token = RoomStreamToken.parse(from_token.room_key)

with await self.pagination_lock.read(room_id):
(
Expand All @@ -358,7 +354,7 @@ async def get_messages(
room_id, user_id, allow_departed_users=True
)

if source_config.direction == "b":
if pagin_config.direction == "b":
# if we're going backwards, we might need to backfill. This
# requires that we have a topo token.
if room_token.topological:
Expand All @@ -377,26 +373,35 @@ async def get_messages(
# case "JOIN" would have been returned.
assert member_event_id

leave_token = await self.store.get_topological_token_for_event(
leave_token_str = await self.store.get_topological_token_for_event(
member_event_id
)
if RoomStreamToken.parse(leave_token).topological < max_topo:
source_config.from_key = str(leave_token)
leave_token = RoomStreamToken.parse(leave_token_str)
assert leave_token.topological is not None

if leave_token.topological < max_topo:
from_token = from_token.copy_and_replace(
"room_key", leave_token_str
)

await self.hs.get_handlers().federation_handler.maybe_backfill(
room_id, max_topo
)

to_room_key = None
if pagin_config.to_token:
to_room_key = pagin_config.to_token.room_key

events, next_key = await self.store.paginate_room_events(
room_id=room_id,
from_key=source_config.from_key,
to_key=source_config.to_key,
direction=source_config.direction,
limit=source_config.limit,
from_key=from_token.room_key,
to_key=to_room_key,
direction=pagin_config.direction,
limit=pagin_config.limit,
event_filter=event_filter,
)

next_token = pagin_config.from_token.copy_and_replace("room_key", next_key)
next_token = from_token.copy_and_replace("room_key", next_key)

if events:
if event_filter:
Expand All @@ -409,7 +414,7 @@ async def get_messages(
if not events:
return {
"chunk": [],
"start": pagin_config.from_token.to_string(),
"start": from_token.to_string(),
"end": next_token.to_string(),
}

Expand Down Expand Up @@ -438,7 +443,7 @@ async def get_messages(
events, time_now, as_client_event=as_client_event
)
),
"start": pagin_config.from_token.to_string(),
"start": from_token.to_string(),
"end": next_token.to_string(),
}

Expand Down
3 changes: 0 additions & 3 deletions synapse/handlers/presence.py
Original file line number Diff line number Diff line change
Expand Up @@ -1108,9 +1108,6 @@ async def get_new_events(
def get_current_key(self):
return self.store.get_current_presence_token()

async def get_pagination_rows(self, user, pagination_config, key):
return await self.get_new_events(user, from_key=None, include_offline=False)

@cached(num_args=2, cache_context=True)
async def _get_interested_in(self, user, explicit_room_id, cache_context):
"""Returns the set of users that the given user should see presence
Expand Down
15 changes: 0 additions & 15 deletions synapse/handlers/receipts.py
Original file line number Diff line number Diff line change
Expand Up @@ -142,18 +142,3 @@ async def get_new_events(self, from_key, room_ids, **kwargs):

def get_current_key(self, direction="f"):
return self.store.get_max_receipt_stream_id()

async def get_pagination_rows(self, user, config, key):
to_key = int(config.from_key)

if config.to_key:
from_key = int(config.to_key)
else:
from_key = None

room_ids = await self.store.get_rooms_for_user(user.to_string())
events = await self.store.get_linearized_receipts_for_rooms(
room_ids, from_key=from_key, to_key=to_key
)

return (events, to_key)
5 changes: 2 additions & 3 deletions synapse/handlers/sync.py
Original file line number Diff line number Diff line change
Expand Up @@ -1310,12 +1310,11 @@ async def _generate_sync_entry_for_presence(
presence_source = self.event_sources.sources["presence"]

since_token = sync_result_builder.since_token
presence_key = None
include_offline = False
if since_token and not sync_result_builder.full_state:
presence_key = since_token.presence_key
include_offline = True
else:
presence_key = None
include_offline = False

presence, presence_key = await presence_source.get_new_events(
user=user,
Expand Down
6 changes: 3 additions & 3 deletions synapse/logging/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,11 +29,11 @@ def _log_debug_as_f(f, msg, msg_args):
lineno = f.__code__.co_firstlineno
pathname = f.__code__.co_filename

record = logging.LogRecord(
record = logger.makeRecord(
name=name,
level=logging.DEBUG,
pathname=pathname,
lineno=lineno,
fn=pathname,
lno=lineno,
msg=msg,
args=msg_args,
exc_info=None,
Expand Down
5 changes: 3 additions & 2 deletions synapse/notifier.py
Original file line number Diff line number Diff line change
Expand Up @@ -432,8 +432,9 @@ async def get_events_for(
If explicit_room_id is set, that room will be polled for events only if
it is world readable or the user has joined the room.
"""
from_token = pagination_config.from_token
if not from_token:
if pagination_config.from_token:
from_token = pagination_config.from_token
else:
from_token = self.event_sources.get_current_token()

limit = pagination_config.limit
Expand Down
41 changes: 39 additions & 2 deletions synapse/rest/client/v2_alpha/account.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,6 @@
if TYPE_CHECKING:
from synapse.app.homeserver import HomeServer

from twisted.internet import defer

from synapse.api.constants import LoginType
from synapse.api.errors import (
Codes,
Expand Down Expand Up @@ -1103,6 +1101,45 @@ def assert_valid_next_link(hs: "HomeServer", next_link: str):
)


def assert_valid_next_link(hs: "HomeServer", next_link: str):
"""
Raises a SynapseError if a given next_link value is invalid
next_link is valid if the scheme is http(s) and the next_link.domain_whitelist config
option is either empty or contains a domain that matches the one in the given next_link
Args:
hs: The homeserver object
next_link: The next_link value given by the client
Raises:
SynapseError: If the next_link is invalid
"""
valid = True

# Parse the contents of the URL
next_link_parsed = urlparse(next_link)

# Scheme must not point to the local drive
if next_link_parsed.scheme == "file":
valid = False

# If the domain whitelist is set, the domain must be in it
if (
valid
and hs.config.next_link_domain_whitelist is not None
and next_link_parsed.hostname not in hs.config.next_link_domain_whitelist
):
valid = False

if not valid:
raise SynapseError(
400,
"'next_link' domain not included in whitelist, or not http(s)",
errcode=Codes.INVALID_PARAM,
)


class WhoamiRestServlet(RestServlet):
PATTERNS = client_patterns("/account/whoami$")

Expand Down
19 changes: 18 additions & 1 deletion synapse/rest/media/v1/filepath.py
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ def remote_media_thumbnail_rel(
self, server_name, file_id, width, height, content_type, method
):
top_level_type, sub_type = content_type.split("/")
file_name = "%i-%i-%s-%s" % (width, height, top_level_type, sub_type)
file_name = "%i-%i-%s-%s-%s" % (width, height, top_level_type, sub_type, method)
return os.path.join(
"remote_thumbnail",
server_name,
Expand All @@ -92,6 +92,23 @@ def remote_media_thumbnail_rel(

remote_media_thumbnail = _wrap_in_base_path(remote_media_thumbnail_rel)

# Legacy path that was used to store thumbnails previously.
# Should be removed after some time, when most of the thumbnails are stored
# using the new path.
def remote_media_thumbnail_rel_legacy(
self, server_name, file_id, width, height, content_type
):
top_level_type, sub_type = content_type.split("/")
file_name = "%i-%i-%s-%s" % (width, height, top_level_type, sub_type)
return os.path.join(
"remote_thumbnail",
server_name,
file_id[0:2],
file_id[2:4],
file_id[4:],
file_name,
)

def remote_media_thumbnail_dir(self, server_name, file_id):
return os.path.join(
self.base_path,
Expand Down
28 changes: 28 additions & 0 deletions synapse/rest/media/v1/media_storage.py
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,20 @@ async def fetch_media(self, file_info: FileInfo) -> Optional[Responder]:
if os.path.exists(local_path):
return FileResponder(open(local_path, "rb"))

# Fallback for paths without method names
# Should be removed in the future
if file_info.thumbnail and file_info.server_name:
legacy_path = self.filepaths.remote_media_thumbnail_rel_legacy(
server_name=file_info.server_name,
file_id=file_info.file_id,
width=file_info.thumbnail_width,
height=file_info.thumbnail_height,
content_type=file_info.thumbnail_type,
)
legacy_local_path = os.path.join(self.local_media_directory, legacy_path)
if os.path.exists(legacy_local_path):
return FileResponder(open(legacy_local_path, "rb"))

for provider in self.storage_providers:
res = await provider.fetch(path, file_info) # type: Any
if res:
Expand All @@ -170,6 +184,20 @@ async def ensure_media_is_in_local_cache(self, file_info: FileInfo) -> str:
if os.path.exists(local_path):
return local_path

# Fallback for paths without method names
# Should be removed in the future
if file_info.thumbnail and file_info.server_name:
legacy_path = self.filepaths.remote_media_thumbnail_rel_legacy(
server_name=file_info.server_name,
file_id=file_info.file_id,
width=file_info.thumbnail_width,
height=file_info.thumbnail_height,
content_type=file_info.thumbnail_type,
)
legacy_local_path = os.path.join(self.local_media_directory, legacy_path)
if os.path.exists(legacy_local_path):
return legacy_local_path

dirname = os.path.dirname(local_path)
if not os.path.exists(dirname):
os.makedirs(dirname)
Expand Down
7 changes: 3 additions & 4 deletions synapse/storage/databases/main/devices.py
Original file line number Diff line number Diff line change
Expand Up @@ -481,7 +481,7 @@ async def get_cached_devices_for_user(self, user_id: str) -> Dict[str, JsonDict]
}

async def get_users_whose_devices_changed(
self, from_key: str, user_ids: Iterable[str]
self, from_key: int, user_ids: Iterable[str]
) -> Set[str]:
"""Get set of users whose devices have changed since `from_key` that
are in the given list of user_ids.
Expand All @@ -493,7 +493,6 @@ async def get_users_whose_devices_changed(
Returns:
The set of user_ids whose devices have changed since `from_key`
"""
from_key = int(from_key)

# Get set of users who *may* have changed. Users not in the returned
# list have definitely not changed.
Expand Down Expand Up @@ -527,7 +526,7 @@ def _get_users_whose_devices_changed_txn(txn):
)

async def get_users_whose_signatures_changed(
self, user_id: str, from_key: str
self, user_id: str, from_key: int
) -> Set[str]:
"""Get the users who have new cross-signing signatures made by `user_id` since
`from_key`.
Expand All @@ -539,7 +538,7 @@ async def get_users_whose_signatures_changed(
Returns:
A set of user IDs with updated signatures.
"""
from_key = int(from_key)

if self._user_signature_stream_cache.has_entity_changed(user_id, from_key):
sql = """
SELECT DISTINCT user_ids FROM user_signature_stream
Expand Down
Loading

0 comments on commit 82c379c

Please sign in to comment.