Merge mainline release v1.24.0 into dinsic

DEVELOP.rst

@@ -55,6 +55,10 @@ is described in more detail in the following sections.
     deprecated in the spec. These tests will be ignored automatically if
     the ``--exclude-deprecated`` flag is provided to Sytest. 
 
+``implementation_specific``
+    A string to specify that the test should only be run for certain homeserver
+    implementations, e.g. `"Synapse"`, `"Dendrite"` etc. 
+
 A call to ``test`` is a simplified version of ``multi_test`` which produces
 only a single line of test output indicating success or failure automatically.
 A call to ``multi_test`` can make use of additional functions within the body

README.rst

@@ -146,6 +146,18 @@ synapse) and place it within the homeserver configuration directory
 
 .. _dictConfig: https://docs.python.org/2/library/logging.config.html#logging.config.dictConfig
 
+Plugins
+~~~~~~~
+
+Sytest supports plugins. Plugins follow the same project structure as sytest and can be placed
+in the ``plugins`` directory. They should contain the ``lib/SyTest/HomeserverFactory`` and
+``lib/SyTest/Homeserver``, or ``lib/SyTest/Output`` directories, similar to the root of the sytest repository.
+The path of the plugins directory can be overridden via the ``SYTEST_PLUGINS`` environment variable.
+
+Currently only ``Homeserver`` and ``Output`` implementations are supported in plugins.
+
+See https://github.com/valkum/sytest_conduit for an example of a plugin.
+
 Developing
 ----------
 

cpanfile

@@ -32,7 +32,9 @@ requires 'Email::MIME';
 requires 'File::Basename';
 requires 'File::Path';
 requires 'File::Slurper';
-requires 'Future', '>= 0.33';
+
+# Future 0.45 allows you to return immediate values from sequence functions.
+requires 'Future', '>= 0.45';
 requires 'Getopt::Long';
 requires 'IO::Async', '>= 0.69';
 requires 'IO::Async::SSL';

docker/README.md

@@ -97,3 +97,15 @@ docker run --rm -it ... matrixdotorg/sytest-synapse:py37 tests/20profile-events.
 
 The containers are built by executing `./build.sh`. You will then have to push
 them up to Docker Hub with `./push.sh`.
+
+## Loading sytest plugins at start
+
+To utilize sytest plugins and automatically load them on start set the `PLUGINS` environment variable.
+This should be one or more URLs to tar.gz files separated by whitespaces.
+
+The bootstrap script will search for `${SYTEST_TARGET}_sytest.sh` in all plugins. This can be used to
+execute custom scripts like the ones in `/scripts/`
+
+```
+docker run --rm -it -e PLUGINS="https://host/path/to/hs_plugin.tar.gz https://host2/path/to/output_plugin.tar.gz"
+```

docker/bootstrap.sh

@@ -37,6 +37,19 @@ else
 
     mkdir -p /sytest
     tar -C /sytest --strip-components=1 -xf sytest.tar.gz
+
+    if [ -n "$PLUGINS" ]; then
+        mkdir /sytest/plugins
+        echo "--- Downloading plugins for sytest"
+        IFS=' '; for plugin in $PLUGINS; do
+            plugindir=$(mktemp -d --tmpdir=/sytest/plugins)
+            wget -q $plugin -O plugin.tar.gz || {
+                echo "Failed to download plugin: $plugin" >&2
+                exit 1
+            }
+            tar -C $plugindir --strip-components=1 -xf plugin.tar.gz
+        done
+    fi
 fi
 
 echo "--- Preparing sytest for ${SYTEST_TARGET}"
@@ -51,6 +64,11 @@ elif [ -x "/sytest/docker/${SYTEST_TARGET}_sytest.sh" ]; then
     exec "/sytest/docker/${SYTEST_TARGET}_sytest.sh" "$@"
 
 else
-    echo "sytest runner script for ${SYTEST_TARGET} not found" >&2
-    exit 1
+    PLUGIN_RUNNER=$(find /sytest/plugins/ -type f -name "${SYTEST_TARGET}_sytest.sh" -print)
+    if [ -n PLUGIN_RUNNER ]; then
+        exec ${PLUGIN_RUNNER} "$@"
+    else
+        echo "sytest runner script for ${SYTEST_TARGET} not found" >&2
+        exit 1
+    fi
 fi

lib/SyTest/Homeserver/Dendrite.pm

@@ -352,6 +352,7 @@ sub _start_monolith
       '--config', $self->{paths}{config},
       '--http-bind-address', $self->{bind_host} . ':' . $self->unsecure_port,
       '--https-bind-address', $self->{bind_host} . ':' . $self->secure_port,
+      '--api-bind-address', $self->{bind_host} . ':1' . $self->unsecure_port,
       '--tls-cert', $self->{paths}{tls_cert},
       '--tls-key', $self->{paths}{tls_key},
    );

lib/SyTest/Homeserver/ProcessManager.pm

@@ -34,7 +34,7 @@ This class is a mixin for C<IO::Async::Notifier> classes which start external pr
 
 =cut
 
-struct ProcessInfo => [qw( process finished_future output_lines print_output )];
+struct ProcessInfo => [qw( process finished_future output_lines print_output name )];
 
 =head1 UTILITY METHODS
 
@@ -58,6 +58,10 @@ Passed to C<IO::Async::Process::new>, giving the command to be run.
 Optional reference to an array to pass to the underlying C<Loop>
 C<spawn_child> method.
 
+=item name => STRING
+
+Optional name for the process, used while logging.
+
 =back
 
 It returns the Process object.
@@ -75,11 +79,13 @@ sub _start_process
          if exists $params{$_};
    }
 
+   my $proc_name = $params{name} // "server";
+
    $self->{proc_info} //= {};
 
    my $fut = $self->loop->new_future;
 
-   my $proc_info = ProcessInfo( undef, $fut, [], 0 );
+   my $proc_info = ProcessInfo( undef, $fut, [], 0, $proc_name );
 
    my $on_output = sub {
       my ( $stream, $buffref, $eof ) = @_;
@@ -108,11 +114,13 @@ sub _on_output_line
    my $self = shift;
    my ( $proc_info, $line ) = @_;
 
+   my $name = $proc_info->name;
+
    push @{ $proc_info->output_lines }, $line;
    shift @{ $proc_info->output_lines } while @{ $proc_info->output_lines } > 20;
 
    if( $proc_info->print_output ) {
-      print STDERR "\e[1;35m[server]\e[m: $line\n";
+      print STDERR "\e[1;35m[$name]\e[m: $line\n";
    }
 }
 
@@ -122,19 +130,20 @@ sub _on_finish
    my $self = shift;
    my ( $process, $exitcode ) = @_;
 
+   my $proc_info = $self->{proc_info}{$process};
+   my $name = $proc_info->name;
+
    if( $exitcode > 0 ) {
       if( WIFEXITED($exitcode) ) {
-         warn "Homeserver process exited " . WEXITSTATUS($exitcode) . "\n";
+         warn "$name process exited " . WEXITSTATUS($exitcode) . "\n";
       }
       else {
-         warn "Homeserver process failed - code=$exitcode\n";
+         warn "$name process failed - code=$exitcode\n";
       }
    }
 
-   my $proc_info = $self->{proc_info}{$process};
-
    # print the last few lines of output
-   print STDERR "\e[1;35m[server]\e[m: $_\n"
+   print STDERR "\e[1;35m[$name]\e[m: $_\n"
       for @{ $proc_info->output_lines };
 
    # - and force anything that has yet to hit the buffer to be printed
@@ -225,13 +234,13 @@ sub _kill_process
 
    my $finished_future = $proc_info->finished_future;
 
-   $self->{output}->diag( "Killing process " . $process->pid );
+   $self->{output}->diag( "Killing process " . $proc_info->name );
    $process->kill( 'INT' );
    return Future->needs_any(
       $finished_future->without_cancel,
 
       $self->loop->delay_future( after => 30 )->then( sub {
-         $self->{output}->diag( "Timed out waiting for ${\ $process->pid }; sending SIGKILL" );
+         $self->{output}->diag( "Timed out waiting for ${\ $proc_info->name }; sending SIGKILL" );
          $process->kill( 'KILL' );
          Future->done;
       }),
@@ -275,13 +284,14 @@ sub _start_process_and_await_notify
    # socket before starting the process.
    my $await_fut = $self->_await_ready_notification( $env );
 
-   my $proc = $self -> _start_process( %params );
-   my $finished_future = $self->{proc_info}{$proc}->finished_future;
+   my $proc = $self->_start_process( %params );
+   my $proc_info = $self->{proc_info}{$proc};
+   my $finished_future = $proc_info->finished_future;
 
    my $fut = Future->wait_any(
       $await_fut,
       $finished_future->without_cancel()->then_fail(
-         "Process died without becoming connectable",
+         "Process died without becoming ready",
       ),
    );
    return $fut;
@@ -306,7 +316,6 @@ sub _await_ready_notification
    my ( $env ) = @_;
 
    my $loop = $self->loop;
-   my $output = $self->{output};
 
    # Create a random abstract socket name. Abstract sockets start with a null
    # byte.
@@ -329,7 +338,6 @@ sub _await_ready_notification
 
          # Payloads are newline separated list of varalbe assignments.
          foreach my $line ( split(/\n/, $dgram) ) {
-            $output->diag( "Received signal from process: $line" );
             if ( $line eq "READY=1" ) {
                $poke_fut->done;
             }