feat: added the possibility to run tests in ESM mode (#7411)

<!--
Thank you for your contribution.

Before making a PR, please read our contributing guidelines at

https://github.com/DevExpress/testcafe/blob/master/CONTRIBUTING.md#code-contribution

We recommend creating a *draft* PR, so that you can mark it as 'ready
for review' when you are done.
-->

## Purpose
Add the possibility to run tests in ESM mode

## Approach
- [X] Research ways to implement running ESM files consistently
- [X] Research ways to compile test files on the run
- [X] Add loader hooks for ESM
- [X] Run tests with dynamic import
- [x] Fix tests
- [X] Add handling error `ERR_REQUIRE_ESM`
- [X] Add workflow for running tests in ESM
- [x] Add additional tests

## References
Closes #6853
PR in `bin-v8-flags-filter`
DevExpress/bin-v8-flags-filter#1
PR in `callsite-record`
inikulin/callsite-record#28
Temp `bin-v8-flags-filter` assembling
[bin-v8-flags-filter-1.2.0.zip](https://github.com/DevExpress/testcafe/files/10175237/bin-v8-flags-filter-1.2.0.zip)
Temp `callsite-record` assembling
[callsite-record-4.1.3.zip](https://github.com/DevExpress/testcafe/files/10175241/callsite-record-4.1.3.zip)

## Pre-Merge TODO
- [x] Write tests for your proposed changes
- [x] Make sure that existing tests do not fail

.github/workflows/deploy-to-artifacts.yml

@@ -165,6 +165,7 @@ jobs:
 
             tasks.push('test-functional-docker.yml');
             tasks.push('test-functional-local-chrome.yml');
+            tasks.push('test-functional-local-esm.yml');
             tasks.push('test-functional-local-debug-1.yml');
             tasks.push('test-functional-local-debug-2.yml');
             tasks.push('test-functional-local-firefox.yml');

.github/workflows/test-functional-local-esm.yml

@@ -0,0 +1,103 @@
+name: Test Functional (ESM)
+
+on:
+  workflow_dispatch:
+    inputs:
+      sha:
+        desciption: 'The test commit SHA or ref'
+        required: true
+        default: 'master'
+      merged_sha:
+        description: 'The merge commit SHA' 
+      deploy_run_id:
+        description: 'The ID of a deployment workspace run with artifacts'
+jobs:  
+  test:
+    runs-on: ubuntu-latest
+    environment: test-functional
+    env:
+      RETRY_FAILED_TESTS: true
+    steps:
+      - uses: actions/github-script@v3
+        with:
+          script: |
+            await github.repos.createCommitStatus({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              sha: context.payload.inputs.sha,
+              context: context.workflow,
+              state: 'pending',
+              target_url: `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`
+            });
+      - uses: actions/checkout@v2
+        with:
+          ref: ${{github.event.inputs.merged_sha || github.event.inputs.sha}}
+
+      - uses: actions/setup-node@v2
+        with:
+          node-version: 16
+
+      - uses: actions/github-script@v3
+        with:
+          script: |
+            const delay = ms => new Promise(resolve => setTimeout(resolve, ms));
+            
+            let artifacts = {};
+            
+            for(let i = 0;i<36&&!artifacts.total_count;i++,await delay(5000)) {
+              try {
+                ({ data: artifacts } = await github.actions.listWorkflowRunArtifacts({
+                repo: context.repo.repo,
+                owner: context.repo.owner,
+                run_id: context.payload.inputs.deploy_run_id
+              }));
+              }
+              catch (e) {
+                console.log(e);
+              }
+            }
+            
+            const { data: artifact } = await github.request(artifacts.artifacts.find(artifact=> artifact.name === 'npm').archive_download_url);
+            require('fs').writeFileSync(require('path').join(process.env.GITHUB_WORKSPACE, 'package.zip'), Buffer.from(artifact))
+        
+      - run: |
+          unzip package.zip
+          tar --strip-components=1 -xzf testcafe-*.tgz
+          
+      - name: Get npm cache directory
+        id: npm-cache-dir
+        run: |
+          echo "::set-output name=dir::$(npm config get cache)"
+      - uses: actions/cache@v2
+        with:
+          path: ${{ steps.npm-cache-dir.outputs.dir }}
+          key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-node-
+      - run: npm ci
+      - run: npx gulp prepare-functional-tests --steps-as-tasks
+      - run: npm run test-functional-local-headless-chrome-run-esm
+        timeout-minutes: 60
+      - uses: actions/github-script@v3
+        with:
+          script: |
+            await github.repos.createCommitStatus({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              sha: context.payload.inputs.sha,
+              context: context.workflow,
+              state: 'success',
+              target_url: `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`
+            });
+      - uses: actions/github-script@v3
+        if: failure() || cancelled()
+        with:
+          script: |
+            await github.repos.createCommitStatus({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              sha: context.payload.inputs.sha,
+              context: context.workflow,
+              state: 'failure',
+              target_url: `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`
+            });

.gitignore

@@ -14,3 +14,4 @@ Gemfile.lock
 .npmrc
 .DS_Store
 !gulp
+/test/functional/fixtures/**/package.json

Gulpfile.js

@@ -29,6 +29,7 @@ const promisifyStream               = require('./gulp/helpers/promisify-stream')
 const testFunctional                = require('./gulp/helpers/test-functional');
 const testClient                    = require('./gulp/helpers/test-client');
 const moduleExportsTransform        = require('./gulp/helpers/module-exports-transform');
+const createPackageFilesForTests    = require('./gulp/helpers/create-package-files-for-tests');
 
 const {
     TESTS_GLOB,
@@ -251,16 +252,30 @@ gulp.step('images', () => {
         .pipe(gulp.dest('lib'));
 });
 
+gulp.step('esm-exportable-lib-script', () => {
+    return gulp
+        .src(['src/api/exportable-lib/index.mjs'])
+        .pipe(gulp.dest('lib/api/exportable-lib/'));
+});
+
 //NOTE: Executing tasks in parallel can cause out-of-memory errors on Azure Pipelines
 const buildTasks = process.env.TF_BUILD ? gulp.series : gulp.parallel;
 
-gulp.step('package-content', buildTasks('ts-defs', 'server-scripts', 'client-scripts', 'styles', 'images', 'templates'));
+gulp.step('package-content', buildTasks('ts-defs', 'server-scripts', 'client-scripts', 'esm-exportable-lib-script', 'styles', 'images', 'templates'));
 
 gulp.task('fast-build', gulp.series('clean', 'package-content'));
 
 gulp.task('build', process.env.DEV_MODE === 'true' ? gulp.registry().get('fast-build') : buildTasks('lint', 'fast-build'));
 
 // Test
+gulp.step('prepare-functional-tests', async () => {
+    return createPackageFilesForTests();
+});
+
+gulp.step('clean-functional-tests', async () => {
+    return del('test/functional/fixtures/**/package.json');
+});
+
 gulp.step('prepare-tests', gulp.registry().get(SKIP_BUILD ? 'lint' : 'build'));
 
 gulp.step('test-server-run', () => {
@@ -380,7 +395,7 @@ gulp.step('test-functional-local-headless-chrome-run', () => {
     return testFunctional(TESTS_GLOB, functionalTestConfig.testingEnvironmentNames.localHeadlessChrome);
 });
 
-gulp.task('test-functional-local-headless-chrome', gulp.series('prepare-tests', 'test-functional-local-headless-chrome-run'));
+gulp.task('test-functional-local-headless-chrome', gulp.series('prepare-tests', 'prepare-functional-tests', 'test-functional-local-headless-chrome-run', 'clean-functional-tests'));
 
 gulp.step('test-functional-local-headless-firefox-run', () => {
     return testFunctional(TESTS_GLOB, functionalTestConfig.testingEnvironmentNames.localHeadlessFirefox);

bin/testcafe-with-v8-flag-filter.js

@@ -3,12 +3,25 @@
 'use strict';
 
 const path          = require('path');
-const v8FlagsFilter = require('bin-v8-flags-filter');
+const url           = require('url');
+const v8FlagsFilter = require('@devexpress/bin-v8-flags-filter');
 
 const EXPERIMENTAL_DEBUG_OPTION = '--experimental-debug';
+const EXPERIMENTAL_ESM_OPTION   = '--experimental-esm';
 
 if (process.argv.slice(2).includes(EXPERIMENTAL_DEBUG_OPTION))
     require('../lib/cli');
 
-else
-    v8FlagsFilter(path.join(__dirname, '../lib/cli'), { useShutdownMessage: true });
+else {
+    const forcedArgs = [];
+
+    if (process.argv.slice(2).includes(EXPERIMENTAL_ESM_OPTION)) {
+        forcedArgs.push('--no-warnings');
+        forcedArgs.push(`--experimental-loader=${url.pathToFileURL(path.join(__dirname, '../lib/compiler/esm-loader.js')).href}`);
+    }
+
+    v8FlagsFilter(path.join(__dirname, '../lib/cli'), {
+        useShutdownMessage: true,
+        forcedArgs,
+    });
+}

gulp/esm-package/package.json

@@ -0,0 +1 @@
+{ "type": "module" }

gulp/helpers/create-package-files-for-tests.js

@@ -0,0 +1,27 @@
+const gulp         = require('gulp');
+const globby       = require('globby');
+const path         = require('path');
+const { Readable } = require('stream');
+
+
+module.exports = async function createPackageFilesForTests () {
+    const testFolders = await globby([
+        'test/functional/fixtures/**/testcafe-fixtures',
+        'test/functional/fixtures/**/common',
+    ], {
+        ignore:          ['test/functional/fixtures/**/raw', 'test/functional/fixtures/**/json'],
+        onlyDirectories: true,
+    });
+
+    let stream = new Readable();
+
+    stream.push('{ "type": "module" }');
+    stream.push(null);
+
+    testFolders.forEach(testFolder => {
+        stream = gulp.src(['gulp/esm-package/package.json'])
+            .pipe(gulp.dest( path.resolve(testFolder) ));
+    });
+
+    return stream;
+};

package.json

@@ -53,6 +53,9 @@
   ],
   "scripts": {
     "test": "gulp travis",
+    "build": "gulp build",
+    "test-functional-local-headless-chrome-run-esm": "cross-env NODE_OPTIONS='--experimental-loader=./lib/compiler/esm-loader.js' gulp test-functional-local-headless-chrome-run --steps-as-tasks",
+    "test-functional-local-headless-chrome-esm": "npm run build && npm run test-functional-local-headless-chrome-run-esm",
     "publish-please-only": "publish-please",
     "publish-please": "del-cli package-lock.json node_modules && npm i && publish-please",
     "prepublishOnly": "publish-please guard"
@@ -79,7 +82,7 @@
     "async-exit-hook": "^1.1.2",
     "babel-plugin-module-resolver": "^5.0.0",
     "babel-plugin-syntax-trailing-function-commas": "^6.22.0",
-    "bin-v8-flags-filter": "^1.1.2",
+    "@devexpress/bin-v8-flags-filter": "^1.3.0",
     "bowser": "^2.8.1",
     "callsite": "^1.0.0",
     "callsite-record": "^4.0.0",
@@ -184,6 +187,7 @@
     "chai-string": "^1.5.0",
     "chrome-launcher": "^0.15.0",
     "connect": "^3.4.0",
+    "cross-env": "^7.0.3",
     "devtools-protocol": "0.0.1078443",
     "dom-walk": "^0.1.1",
     "escape-string-regexp": "^4.0.0",

src/api/exportable-lib/index.mjs

@@ -0,0 +1,27 @@
+import exportableLib from './index.js';
+
+const {
+    Role,
+    ClientFunction,
+    Selector,
+    RequestLogger,
+    RequestMock,
+    RequestHook,
+    t,
+    userVariables,
+    fixture,
+    test,
+} = exportableLib;
+
+export {
+    Role,
+    ClientFunction,
+    Selector,
+    RequestLogger,
+    RequestMock,
+    RequestHook,
+    t,
+    userVariables,
+    fixture,
+    test,
+};

src/api/structure/fixture.ts

@@ -14,6 +14,11 @@ import RequestHook from '../request-hooks/hook';
 import ClientScriptInit from '../../custom-client-scripts/client-script-init';
 import { SPECIAL_BLANK_PAGE } from 'testcafe-hammerhead';
 
+interface FixtureInitOptions {
+    baseUrl?: string;
+    testFile: TestFile;
+}
+
 export default class Fixture extends TestingUnit {
     public path: string;
     public beforeEachFn: Function | null;
@@ -39,6 +44,14 @@ export default class Fixture extends TestingUnit {
         return this.apiOrigin as unknown as Fixture;
     }
 
+    public static init (initOptions: FixtureInitOptions, name: string, ...rest: unknown[]): Fixture | null {
+        const { testFile, baseUrl } = initOptions;
+
+        const fixture = new Fixture(testFile, baseUrl);
+
+        return (fixture as unknown as Function)(name, ...rest);
+    }
+
     protected _add (name: string, ...rest: unknown[]): Function {
         name = handleTagArgs(name, rest);
 

src/api/structure/test.ts

@@ -17,6 +17,11 @@ import { TestTimeouts } from './interfaces';
 import TestTimeout from './test-timeout';
 import ESM_RUNTIME_HOLDER_NAME from '../../services/compiler/esm-runtime-holder-name';
 
+interface TestInitOptions {
+    testFile: TestFile;
+    baseUrl?: string;
+    isCompilerServiceMode?: boolean;
+}
 
 export default class Test extends TestingUnit {
     public fixture: Fixture | null;
@@ -55,12 +60,21 @@ export default class Test extends TestingUnit {
         return this.apiOrigin as unknown as Test;
     }
 
+    public static init (initOptions: TestInitOptions, name: string, fn: Function): Test {
+        const { testFile, baseUrl, isCompilerServiceMode } = initOptions;
+
+        const test = new Test(testFile, isCompilerServiceMode, baseUrl);
+
+        return (test as unknown as Function)(name, fn);
+    }
+
     private _initFixture (testFile: TestFile): void {
         this.fixture = testFile.currentFixture;
 
         if (!this.fixture)
             return;
 
+        this.pageUrl             = this.fixture.pageUrl || SPECIAL_BLANK_PAGE;
         this.requestHooks        = this.fixture.requestHooks.slice();
         this.clientScripts       = this.fixture.clientScripts.slice();
         this.skipJsErrorsOptions = this.fixture.skipJsErrorsOptions;

src/cli/argument-parser/index.ts

@@ -207,8 +207,9 @@ export default class CLIArgumentParser {
             .option('--color', 'force TestCafe to format CLI output with color')
             .option('--no-color', 'disable text color formatting in the CLI')
 
-            // NOTE: temporary hide experimental options from --help command
-            .addOption(new Option('--experimental-debug', 'enable experimental debug mode').hideHelp())
+            // NOTE: Temporarily exclude experimental options from --help output
+            .addOption(new Option('--experimental-debug', 'enable experimental the debug mode').hideHelp())
+            .addOption(new Option('--experimental-esm', 'enable experimental the esm mode').hideHelp())
             .addOption(new Option('--disable-cross-domain', 'experimental').hideHelp())
             .action((opts: CommandLineOptions) => {
                 this.opts = opts;

src/cli/cli.js

@@ -87,6 +87,7 @@ async function runTests (argParser) {
         v8Flags,
         experimentalProxyless,
         disableCrossDomain,
+        experimentalEsm,
     } = opts;
 
     const testCafe = await createTestCafe({
@@ -105,6 +106,7 @@ async function runTests (argParser) {
         v8Flags,
         experimentalProxyless,
         disableCrossDomain,
+        experimentalEsm,
     });
 
     const correctedBrowsersAndSources = await correctBrowsersAndSources(argParser, testCafe.configuration);

src/client/browser/index.js

@@ -4,6 +4,7 @@ import COMMAND from '../../browser/connection/command';
 import HeartbeatStatus from '../../browser/connection/heartbeat-status';
 import { HEARTBEAT_INTERVAL } from '../../utils/browser-connection-timeouts';
 import SERVICE_ROUTES from '../../browser/connection/service-routes';
+import isFileProtocol from '../../shared/utils/is-file-protocol';
 
 /*eslint-disable no-restricted-properties*/
 const LOCATION_HREF   = document.location.href;
@@ -15,8 +16,6 @@ const MAX_STATUS_RETRY   = 5;
 
 const SERVICE_WORKER_LOCATION = LOCATION_ORIGIN + SERVICE_ROUTES.serviceWorker;
 
-const FILE_PROTOCOL_ORIGIN = 'file://';
-
 let allowInitScriptExecution = false;
 let heartbeatIntervalId      = null;
 
@@ -58,10 +57,6 @@ function isCurrentLocation (url) {
     return LOCATION_HREF.toLowerCase() === url.toLowerCase();
 }
 
-function isFileProtocol (url = '') {
-    return url.indexOf(FILE_PROTOCOL_ORIGIN) === 0;
-}
-
 //API
 export function startHeartbeat (heartbeatUrl, createXHR) {
     function heartbeat () {