Add a Cypress Test 🌲 (#8295)

* A first, maybe working cypress test

Plus cypress plugins to manage synapses in docker containers

* Fix yaml

* This file is important

* try & find where it's put the artifact

* Download artifact to a directory

* pics or it didn't happen

* Add conditional, otherwise no artifacts on failure...

* Try increasing timeout

also actually give the test a name

* Try in chrome

* Get docker logs to see why it's failing

also document the chrome setting

* Try changing mode on homeserver.yaml

* debug

* More debugging

* more file permissions debugging

* ARGH

* more debug

* sigh

* Eugh, that's not how arguments work

* Add the option to really allow open registration

and remove debug logging / comment fixes

* failure to yaml

* Upload docker logs as artifacts

and temporarily remove contional to test

* Put the conditional back

* Upgrade types in end to end tests

to be compatible with fs-extra types

* Try reducing timeout a bit

also make password more... sensible

* Hex is not octal

* Remove file mode

Seems to be unnecessary since the signing key is perfectly fine

* Give the log files extensions

* Rename workflow file now it also does tests

* Add cypress scripts

* copyright headers

* Use ? operator

Co-authored-by: Travis Ralston <travisr@matrix.org>

* Use develop synapse image

* Tidy up any remaining synapses after each spec run

Also:
 * Move the synapseStart / synapseStop functions out to the top level
   so they can be reused
 * Add a tsconfig file
 * Give the containers names

* Don't upload video on test pass

We don't upload it anyway so tell cypress not to so it can not
bother encoding them

* Enable linting on cypress files

and fix existing lint errors

* Type check cypress files

and make it pass the type checks, specifically:
 * Upgrade sinon fake timers to a version that has the right types
 * Set module resolution
 * Type check cypress files separately

* Rename workflow file again

Probably better to just call it an element web build

* Don't plus + characters in container name

* Fix yaml

* Stream logs to file

* Add note to end to end tester to sya what's been ported

* Put docker rm in finally block

Co-authored-by: Travis Ralston <travisr@matrix.org>

Author: dbkr

.eslintrc.js

@@ -92,6 +92,7 @@ module.exports = {
             files: [
                 "src/**/*.{ts,tsx}",
                 "test/**/*.{ts,tsx}",
+                "cypress/**/*.ts",
             ],
             extends: [
                 "plugin:matrix-org/typescript",

.github/workflows/element-build-and-test.yaml

@@ -0,0 +1,49 @@
+# Produce a build of element-web with this version of react-sdk
+# and any matching branches of element-web and js-sdk, output it
+# as an artifact and run integration tests.
+name: Element Web - Build and Test
+on:
+    pull_request:
+jobs:
+    build:
+        runs-on: ubuntu-latest
+        env:
+          # This must be set for fetchdep.sh to get the right branch
+          PR_NUMBER: ${{github.event.number}}
+        steps:
+            - uses: actions/checkout@v2
+            - name: Build
+              run: scripts/ci/layered.sh && cd element-web && cp element.io/develop/config.json config.json && CI_PACKAGE=true yarn build
+            - name: Upload Artifact
+              uses: actions/upload-artifact@v2
+              with:
+                  name: previewbuild
+                  path: element-web/webapp
+                  # We'll only use this in a triggered job, then we're done with it
+                  retention-days: 1
+    cypress:
+        needs: build
+        runs-on: ubuntu-latest
+        steps:
+            - uses: actions/checkout@v2
+            - name: Download build
+              uses: actions/download-artifact@v3
+              with:
+                  name: previewbuild
+                  path: webapp
+            - name: Run Cypress tests
+              uses: cypress-io/github-action@v2
+              with:
+                # The built in Electron runner seems to grind to a halt trying
+                # to run the tests, so use chrome.
+                browser: chrome
+                start: npx serve -p 8080 webapp
+            - name: Upload Artifact
+              if: failure()
+              uses: actions/upload-artifact@v2
+              with:
+                  name: cypress-results
+                  path: |
+                      cypress/screenshots
+                      cypress/videos
+                      cypress/dockerlogs

.github/workflows/layered-build.yaml

@@ -19,3 +19,11 @@ package-lock.json
 
 .vscode
 .vscode/
+
+/cypress/videos
+/cypress/downloads
+/cypress/screenshots
+/cypress/synapselogs
+# These could have files in them but don't currently
+# Cypress will still auto-create them though...
+/cypress/fixtures

.gitignore

@@ -0,0 +1,4 @@
+{
+    "baseUrl": "http://localhost:8080",
+    "videoUploadOnPasses": false
+}

cypress.json

@@ -0,0 +1,52 @@
+/*
+Copyright 2022 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/// <reference types="cypress" />
+
+import { SynapseInstance } from "../../plugins/synapsedocker/index";
+
+describe("Registration", () => {
+    let synapseId;
+    let synapsePort;
+
+    beforeEach(() => {
+        cy.task<SynapseInstance>("synapseStart", "consent").then(result => {
+            synapseId = result.synapseId;
+            synapsePort = result.port;
+        });
+        cy.visit("/#/register");
+    });
+
+    afterEach(() => {
+        cy.task("synapseStop", synapseId);
+    });
+
+    it("registers an account and lands on the home screen", () => {
+        cy.get(".mx_ServerPicker_change", { timeout: 15000 }).click();
+        cy.get(".mx_ServerPickerDialog_otherHomeserver").type(`http://localhost:${synapsePort}`);
+        cy.get(".mx_ServerPickerDialog_continue").click();
+        // wait for the dialog to go away
+        cy.get('.mx_ServerPickerDialog').should('not.exist');
+        cy.get("#mx_RegistrationForm_username").type("alice");
+        cy.get("#mx_RegistrationForm_password").type("totally a great password");
+        cy.get("#mx_RegistrationForm_passwordConfirm").type("totally a great password");
+        cy.get(".mx_Login_submit").click();
+        cy.get(".mx_RegistrationEmailPromptDialog button.mx_Dialog_primary").click();
+        cy.get(".mx_InteractiveAuthEntryComponents_termsPolicy input").click();
+        cy.get(".mx_InteractiveAuthEntryComponents_termsSubmit").click();
+        cy.url().should('contain', '/#/home');
+    });
+});

cypress/integration/1-register/register.spec.ts

@@ -0,0 +1,23 @@
+/*
+Copyright 2022 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/// <reference types="cypress" />
+
+import { synapseDocker } from "./synapsedocker/index";
+
+export default function(on, config) {
+    synapseDocker(on, config);
+}

cypress/plugins/index.ts

@@ -0,0 +1,212 @@
+/*
+Copyright 2022 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/// <reference types="cypress" />
+
+import * as path from "path";
+import * as os from "os";
+import * as crypto from "crypto";
+import * as childProcess from "child_process";
+import * as fse from "fs-extra";
+
+// A cypress plugins to add command to start & stop synapses in
+// docker with preset templates.
+
+interface SynapseConfig {
+    configDir: string;
+    registrationSecret: string;
+}
+
+export interface SynapseInstance extends SynapseConfig {
+    synapseId: string;
+    port: number;
+}
+
+const synapses = new Map<string, SynapseInstance>();
+
+function randB64Bytes(numBytes: number): string {
+    return crypto.randomBytes(numBytes).toString("base64").replace(/=*$/, "");
+}
+
+async function cfgDirFromTemplate(template: string): Promise<SynapseConfig> {
+    const templateDir = path.join(__dirname, "templates", template);
+
+    const stats = await fse.stat(templateDir);
+    if (!stats?.isDirectory) {
+        throw new Error(`No such template: ${template}`);
+    }
+    const tempDir = await fse.mkdtemp(path.join(os.tmpdir(), 'react-sdk-synapsedocker-'));
+
+    // change permissions on the temp directory so the docker container can see its contents
+    await fse.chmod(tempDir, 0o777);
+
+    // copy the contents of the template dir, omitting homeserver.yaml as we'll template that
+    console.log(`Copy ${templateDir} -> ${tempDir}`);
+    await fse.copy(templateDir, tempDir, { filter: f => path.basename(f) !== 'homeserver.yaml' });
+
+    const registrationSecret = randB64Bytes(16);
+    const macaroonSecret = randB64Bytes(16);
+    const formSecret = randB64Bytes(16);
+
+    // now copy homeserver.yaml, applying sustitutions
+    console.log(`Gen ${path.join(templateDir, "homeserver.yaml")}`);
+    let hsYaml = await fse.readFile(path.join(templateDir, "homeserver.yaml"), "utf8");
+    hsYaml = hsYaml.replace(/{{REGISTRATION_SECRET}}/g, registrationSecret);
+    hsYaml = hsYaml.replace(/{{MACAROON_SECRET_KEY}}/g, macaroonSecret);
+    hsYaml = hsYaml.replace(/{{FORM_SECRET}}/g, formSecret);
+    await fse.writeFile(path.join(tempDir, "homeserver.yaml"), hsYaml);
+
+    // now generate a signing key (we could use synapse's config generation for
+    // this, or we could just do this...)
+    // NB. This assumes the homeserver.yaml specifies the key in this location
+    const signingKey = randB64Bytes(32);
+    console.log(`Gen ${path.join(templateDir, "localhost.signing.key")}`);
+    await fse.writeFile(path.join(tempDir, "localhost.signing.key"), `ed25519 x ${signingKey}`);
+
+    return {
+        configDir: tempDir,
+        registrationSecret,
+    };
+}
+
+// Start a synapse instance: the template must be the name of
+// one of the templates in the cypress/plugins/synapsedocker/templates
+// directory
+async function synapseStart(template: string): Promise<SynapseInstance> {
+    const synCfg = await cfgDirFromTemplate(template);
+
+    console.log(`Starting synapse with config dir ${synCfg.configDir}...`);
+
+    const containerName = `react-sdk-cypress-synapse-${crypto.randomBytes(4).toString("hex")}`;
+
+    const synapseId = await new Promise<string>((resolve, reject) => {
+        childProcess.execFile('docker', [
+            "run",
+            "--name", containerName,
+            "-d",
+            "-v", `${synCfg.configDir}:/data`,
+            "-p", "8008/tcp",
+            "matrixdotorg/synapse:develop",
+            "run",
+        ], (err, stdout) => {
+            if (err) reject(err);
+            resolve(stdout.trim());
+        });
+    });
+
+    // Get the port that docker allocated: specifying only one
+    // port above leaves docker to just grab a free one, although
+    // in hindsight we need to put the port in public_baseurl in the
+    // config really, so this will probably need changing to use a fixed
+    // / configured port.
+    const port = await new Promise<number>((resolve, reject) => {
+        childProcess.execFile('docker', [
+            "port", synapseId, "8008",
+        ], { encoding: 'utf8' }, (err, stdout) => {
+            if (err) reject(err);
+            resolve(Number(stdout.trim().split(":")[1]));
+        });
+    });
+
+    synapses.set(synapseId, Object.assign({
+        port,
+        synapseId,
+    }, synCfg));
+
+    console.log(`Started synapse with id ${synapseId} on port ${port}.`);
+    return synapses.get(synapseId);
+}
+
+async function synapseStop(id) {
+    const synCfg = synapses.get(id);
+
+    if (!synCfg) throw new Error("Unknown synapse ID");
+
+    try {
+        const synapseLogsPath = path.join("cypress", "synapselogs", id);
+        await fse.ensureDir(synapseLogsPath);
+
+        const stdoutFile = await fse.open(path.join(synapseLogsPath, "stdout.log"), "w");
+        const stderrFile = await fse.open(path.join(synapseLogsPath, "stderr.log"), "w");
+        await new Promise<void>((resolve, reject) => {
+            childProcess.spawn('docker', [
+                "logs",
+                id,
+            ], {
+                stdio: ["ignore", stdoutFile, stderrFile],
+            }).once('close', resolve);
+        });
+        await fse.close(stdoutFile);
+        await fse.close(stderrFile);
+
+        await new Promise<void>((resolve, reject) => {
+            childProcess.execFile('docker', [
+                "stop",
+                id,
+            ], err => {
+                if (err) reject(err);
+                resolve();
+            });
+        });
+    } finally {
+        await new Promise<void>((resolve, reject) => {
+            childProcess.execFile('docker', [
+                "rm",
+                id,
+            ], err => {
+                if (err) reject(err);
+                resolve();
+            });
+        });
+    }
+
+    await fse.remove(synCfg.configDir);
+
+    synapses.delete(id);
+
+    console.log(`Stopped synapse id ${id}.`);
+    // cypres deliberately fails if you return 'undefined', so
+    // return null to signal all is well and we've handled the task.
+    return null;
+}
+
+/**
+ * @type {Cypress.PluginConfig}
+ */
+// eslint-disable-next-line no-unused-vars
+export function synapseDocker(on, config) {
+    on("task", {
+        synapseStart, synapseStop,
+    });
+
+    on("after:spec", async (spec) => {
+        // Cleans up any remaining synapse instances after a spec run
+        // This is on the theory that we should avoid re-using synapse
+        // instances between spec runs: they should be cheap enough to
+        // start that we can have a separate one for each spec run or even
+        // test. If we accidentally re-use synapses, we could inadvertantly
+        // make our tests depend on each other.
+        for (const synId of synapses.keys()) {
+            console.warn(`Cleaning up synapse ID ${synId} after ${spec.name}`);
+            await synapseStop(synId);
+        }
+    });
+
+    on("before:run", async () => {
+        // tidy up old synapse log files before each run
+        await fse.emptyDir(path.join("cypress", "synapselogs"));
+    });
+}

cypress/plugins/synapsedocker/index.ts

@@ -0,0 +1,3 @@
+# Meta-template for synapse templates
+
+To make another template, you can copy this directory