Merge commit '03619324f' into dinsic

* commit '03619324f':
  Create a ListenerConfig object (#7681)
  Fix changelog wording
  1.15.1
  Wrap register_device coroutine in an ensureDeferred (#7684)
  Ensure the body is a string before comparing push rules. (#7701)
  Ensure etag is a string for GET room_keys/version response (#7691)
  Update m.id.phone to use 'phone' instead of 'number' (#7687)
  Fix "There was no active span when trying to log." error (#7698)
  Enable 3PID add/bind/unbind endpoints on r0 routes
  Discard RDATA from already seen positions. (#7648)
  Replace iteritems/itervalues/iterkeys with native versions. (#7692)
  Fix warnings about losing log context during UI auth. (#7688)
  Fix a typo when comparing the URI & method during UI Auth. (#7689)
  Remove "user_id" from GET /presence. (#7606)
  Increase the default SAML session expirary time to 15 minutes. (#7664)
  fix typo in sample_config.yaml (#7652)
  Take out a lock before modifying _CACHES (#7663)
  Add option to enable encryption by default for new rooms (#7639)
  Clean-up the fallback login code. (#7657)

CHANGES.md

@@ -1,3 +1,13 @@
+Synapse 1.15.1 (2020-06-16)
+===========================
+
+Bugfixes
+--------
+
+- Fix a bug introduced in v1.15.0 that would crash Synapse on start when using certain password auth providers. ([\#7684](https://github.com/matrix-org/synapse/issues/7684))
+- Fix a bug introduced in v1.15.0 which meant that some 3PID management endpoints were not accessible on the correct URL. ([\#7685](https://github.com/matrix-org/synapse/issues/7685))
+
+
 Synapse 1.15.0 (2020-06-11)
 ===========================
 

changelog.d/7606.bugfix

@@ -0,0 +1 @@
+Remove `user_id` from the response to `GET /_matrix/client/r0/presence/{userId}/status` to match the specification.

changelog.d/7639.feature

@@ -0,0 +1 @@
+Add an option to enable encryption by default for new rooms.

changelog.d/7648.bugfix

@@ -0,0 +1 @@
+In working mode, ensure that replicated data has not already been received.

changelog.d/7652.doc

@@ -0,0 +1 @@
+Spelling correction in sample_config.yaml.

changelog.d/7657.misc

@@ -0,0 +1 @@
+Clean-up the login fallback code.

changelog.d/7663.bugfix

@@ -0,0 +1 @@
+Fix intermittent exception during startup, introduced in Synapse 1.14.0.

changelog.d/7664.misc

@@ -0,0 +1 @@
+Increase the default SAML session expirary time to 15 minutes.

changelog.d/7681.misc

@@ -0,0 +1 @@
+Refactor handling of `listeners` configuration settings.

changelog.d/7687.bugfix

@@ -0,0 +1 @@
+Accept the proper field (`phone`) for the `m.id.phone` identifier type. The legacy field of `number` is still accepted as a fallback. Bug introduced in v0.20.0-rc1.

changelog.d/7688.bugfix

@@ -0,0 +1 @@
+Fix "Starting db txn 'get_completed_ui_auth_stages' from sentinel context" warning. The bug was introduced in 1.13.0rc1.

changelog.d/7689.bugfix

@@ -0,0 +1 @@
+Compare the URI and method during user interactive authentication (instead of the URI twice). Bug introduced in 1.13.0rc1.

changelog.d/7691.bugfix

@@ -0,0 +1 @@
+Fix a long standing bug where the response to the `GET room_keys/version` endpoint had the incorrect type for the `etag` field.

changelog.d/7692.misc

@@ -0,0 +1 @@
+Replace uses of `six.iterkeys`/`iteritems`/`itervalues` with `keys()`/`items()`/`values()`.

changelog.d/7698.bugfix

@@ -0,0 +1 @@
+Fix logged error during device resync in opentracing. Broke in v1.14.0.

changelog.d/7701.bugfix

@@ -0,0 +1 @@
+Do not break push rule evaluation when receiving an event with a non-string body. This is a long-standing bug.

debian/changelog

@@ -1,3 +1,9 @@
+matrix-synapse-py3 (1.15.1) stable; urgency=medium
+
+  * New synapse release 1.15.1.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 16 Jun 2020 10:27:50 +0100
+
 matrix-synapse-py3 (1.15.0) stable; urgency=medium
 
   * New synapse release 1.15.0.

docs/sample_config.yaml

@@ -283,7 +283,7 @@ listeners:
 # number of monthly active users.
 #
 # 'limit_usage_by_mau' disables/enables monthly active user blocking. When
-# anabled and a limit is reached the server returns a 'ResourceLimitError'
+# enabled and a limit is reached the server returns a 'ResourceLimitError'
 # with error type Codes.RESOURCE_LIMIT_EXCEEDED
 #
 # 'max_mau_value' is the hard limit of monthly active users above which
@@ -1629,7 +1629,7 @@ saml2_config:
 
   # The lifetime of a SAML session. This defines how long a user has to
   # complete the authentication process, if allow_unsolicited is unset.
-  # The default is 5 minutes.
+  # The default is 15 minutes.
   #
   #saml_session_lifetime: 5m
 
@@ -2148,6 +2148,26 @@ spam_checker:
    #    example_stop_events_from: ['@bad:example.com']
 
 
+## Rooms ##
+
+# Controls whether locally-created rooms should be end-to-end encrypted by
+# default.
+#
+# Possible options are "all", "invite", and "off". They are defined as:
+#
+# * "all": any locally-created room
+# * "invite": any room created with the "private_chat" or "trusted_private_chat"
+#             room creation presets
+# * "off": this option will take no effect
+#
+# The default value is "off".
+#
+# Note that this option will only affect rooms created after it is set. It
+# will also not affect rooms created by other servers.
+#
+#encryption_enabled_by_default_for_room_type: invite
+
+
 # Uncomment to allow non-server-admin users to create groups on this server
 #
 #enable_group_creation: true

synapse/__init__.py

@@ -36,7 +36,7 @@
 except ImportError:
     pass
 
-__version__ = "1.15.0"
+__version__ = "1.15.1"
 
 if bool(os.environ.get("SYNAPSE_TEST_PATCH_LOG_CONTEXTS", False)):
     # We import here so that we don't have to install a bunch of deps when

synapse/api/auth.py

@@ -16,8 +16,6 @@
 import logging
 from typing import Optional
 
-from six import itervalues
-
 import pymacaroons
 from netaddr import IPAddress
 
@@ -90,7 +88,7 @@ def check_from_context(self, room_version: str, event, context, do_sig_check=Tru
             event, prev_state_ids, for_verification=True
         )
         auth_events = yield self.store.get_events(auth_events_ids)
-        auth_events = {(e.type, e.state_key): e for e in itervalues(auth_events)}
+        auth_events = {(e.type, e.state_key): e for e in auth_events.values()}
 
         room_version_obj = KNOWN_ROOM_VERSIONS[room_version]
         event_auth.check(

synapse/api/constants.py

@@ -150,3 +150,8 @@ class EventContentFields(object):
     # Timestamp to delete the event after
     # cf https://github.com/matrix-org/matrix-doc/pull/2228
     SELF_DESTRUCT_AFTER = "org.matrix.self_destruct_after"
+
+
+class RoomEncryptionAlgorithms(object):
+    MEGOLM_V1_AES_SHA2 = "m.megolm.v1.aes-sha2"
+    DEFAULT = MEGOLM_V1_AES_SHA2

synapse/api/errors.py

@@ -20,7 +20,6 @@
 import logging
 from typing import Dict, List
 
-from six import iteritems
 from six.moves import http_client
 
 from canonicaljson import json
@@ -505,7 +504,7 @@ def cs_error(msg, code=Codes.UNKNOWN, **kwargs):
         A dict representing the error response JSON.
     """
     err = {"error": msg, "errcode": code}
-    for key, value in iteritems(kwargs):
+    for key, value in kwargs.items():
         err[key] = value
     return err
 

synapse/app/_base.py

@@ -20,6 +20,7 @@
 import socket
 import sys
 import traceback
+from typing import Iterable
 
 from daemonize import Daemonize
 from typing_extensions import NoReturn
@@ -29,6 +30,7 @@
 
 import synapse
 from synapse.app import check_bind_error
+from synapse.config.server import ListenerConfig
 from synapse.crypto import context_factory
 from synapse.logging.context import PreserveLoggingContext
 from synapse.util.async_helpers import Linearizer
@@ -234,7 +236,7 @@ def refresh_certificate(hs):
         logger.info("Context factories updated.")
 
 
-def start(hs, listeners=None):
+def start(hs: "synapse.server.HomeServer", listeners: Iterable[ListenerConfig]):
     """
     Start a Synapse server or worker.
 
@@ -245,8 +247,8 @@ def start(hs, listeners=None):
     notify systemd.
 
     Args:
-        hs (synapse.server.HomeServer)
-        listeners (list[dict]): Listener configuration ('listeners' in homeserver.yaml)
+        hs: homeserver instance
+        listeners: Listener configuration ('listeners' in homeserver.yaml)
     """
     try:
         # Set up the SIGHUP machinery.

synapse/app/generic_worker.py

@@ -37,6 +37,7 @@
 from synapse.config._base import ConfigError
 from synapse.config.homeserver import HomeServerConfig
 from synapse.config.logger import setup_logging
+from synapse.config.server import ListenerConfig
 from synapse.federation import send_queue
 from synapse.federation.transport.server import TransportLayerServer
 from synapse.handlers.presence import (
@@ -514,13 +515,18 @@ def _get_federation_out_pos(self, db_conn):
 class GenericWorkerServer(HomeServer):
     DATASTORE_CLASS = GenericWorkerSlavedStore
 
-    def _listen_http(self, listener_config):
-        port = listener_config["port"]
-        bind_addresses = listener_config["bind_addresses"]
-        site_tag = listener_config.get("tag", port)
+    def _listen_http(self, listener_config: ListenerConfig):
+        port = listener_config.port
+        bind_addresses = listener_config.bind_addresses
+
+        assert listener_config.http_options is not None
+
+        site_tag = listener_config.http_options.tag
+        if site_tag is None:
+            site_tag = port
         resources = {}
-        for res in listener_config["resources"]:
-            for name in res["names"]:
+        for res in listener_config.http_options.resources:
+            for name in res.names:
                 if name == "metrics":
                     resources[METRICS_PREFIX] = MetricsResource(RegistryProxy)
                 elif name == "client":
@@ -590,7 +596,7 @@ def _listen_http(self, listener_config):
                             " repository is disabled. Ignoring."
                         )
 
-                if name == "openid" and "federation" not in res["names"]:
+                if name == "openid" and "federation" not in res.names:
                     # Only load the openid resource separately if federation resource
                     # is not specified since federation resource includes openid
                     # resource.
@@ -625,19 +631,19 @@ def _listen_http(self, listener_config):
 
         logger.info("Synapse worker now listening on port %d", port)
 
-    def start_listening(self, listeners):
+    def start_listening(self, listeners: Iterable[ListenerConfig]):
         for listener in listeners:
-            if listener["type"] == "http":
+            if listener.type == "http":
                 self._listen_http(listener)
-            elif listener["type"] == "manhole":
+            elif listener.type == "manhole":
                 _base.listen_tcp(
-                    listener["bind_addresses"],
-                    listener["port"],
+                    listener.bind_addresses,
+                    listener.port,
                     manhole(
                         username="matrix", password="rabbithole", globals={"hs": self}
                     ),
                 )
-            elif listener["type"] == "metrics":
+            elif listener.type == "metrics":
                 if not self.get_config().enable_metrics:
                     logger.warning(
                         (
@@ -646,9 +652,9 @@ def start_listening(self, listeners):
                         )
                     )
                 else:
-                    _base.listen_metrics(listener["bind_addresses"], listener["port"])
+                    _base.listen_metrics(listener.bind_addresses, listener.port)
             else:
-                logger.warning("Unrecognized listener type: %s", listener["type"])
+                logger.warning("Unsupported listener type: %s", listener.type)
 
         self.get_tcp_replication().start_replication(self)
 
@@ -738,6 +744,11 @@ async def _process_and_notify(self, stream_name, instance_name, token, rows):
         except Exception:
             logger.exception("Error processing replication")
 
+    async def on_position(self, stream_name: str, instance_name: str, token: int):
+        await super().on_position(stream_name, instance_name, token)
+        # Also call on_rdata to ensure that stream positions are properly reset.
+        await self.on_rdata(stream_name, instance_name, token, [])
+
     def stop_pusher(self, user_id, app_id, pushkey):
         if not self.notify_pushers:
             return