features test... command: Copy entire test folder on test execution and improve CLI command usage.

src/spec-node/devContainersSpecCLI.ts

@@ -63,7 +63,7 @@ const mountRegex = /^type=(bind|volume),source=([^,]+),target=([^,]+)(?:,externa
 	y.command('run-user-commands', 'Run user commands', runUserCommandsOptions, runUserCommandsHandler);
 	y.command('read-configuration', 'Read configuration', readConfigurationOptions, readConfigurationHandler);
 	y.command('features', 'Features commands', (y: Argv) => {
-		y.command('test <target>', 'Test features', featuresTestOptions, featuresTestHandler);
+		y.command('test [target]', 'Test features', featuresTestOptions, featuresTestHandler);
 		y.command('package <target>', 'Package features', featuresPackageOptions, featuresPackageHandler);
 		y.command('publish <target>', 'Package and publish features', featuresPublishOptions, featuresPublishHandler);
 		y.command('info <featureId>', 'Fetch info on a feature', featuresInfoOptions, featuresInfoHandler);

src/spec-node/featuresCLI/test.ts

@@ -10,7 +10,8 @@ import { doFeaturesTestCommand } from './testCommandImpl';
 export function featuresTestOptions(y: Argv) {
 	return y
 		.options({
-			'features': { type: 'array', alias: 'f', describe: 'Feature(s) to test as space-separated parameters. Omit to run all tests.  Cannot be combined with \'--global-scenarios-only\'.', },
+			'project-folder': { type: 'string', alias: 'p', default: '.', description: 'Path to folder containing \'src\' and \'test\' sub-folders. This is likely the git root of the project.' },
+			'features': { array: true, alias: 'f', describe: 'Feature(s) to test as space-separated parameters. Omit to run all tests.  Cannot be combined with \'--global-scenarios-only\'.' },
 			'global-scenarios-only': { type: 'boolean', default: false, description: 'Run only scenario tests under \'tests/_global\' .  Cannot be combined with \'-f\'.' },
 			'skip-scenarios': { type: 'boolean', default: false, description: 'Skip all \'scenario\' style tests.  Cannot be combined with \'--global--scenarios-only\'.' },
 			'skip-autogenerated': { type: 'boolean', default: false, description: 'Skip all \'autogenerated\' style tests.' },
@@ -19,7 +20,13 @@ export function featuresTestOptions(y: Argv) {
 			'log-level': { choices: ['info' as 'info', 'debug' as 'debug', 'trace' as 'trace'], default: 'info' as 'info', description: 'Log level.' },
 			'quiet': { type: 'boolean', alias: 'q', default: false, description: 'Quiets output' },
 		})
-		.positional('target', { type: 'string', default: '.', description: 'Path to folder containing \'src\' and \'test\' sub-folders.' })
+		// DEPRECATED: Positional arguments don't play nice with the variadic/array --features option.
+		// Pass target directory with '--project-folder' instead.
+		// This will still continue to work, but any value provided by --project-folder will be preferred.
+		// Omitting both will default to the current working directory.
+		.deprecateOption('target', 'Use --project-folder instead')
+		.positional('target', { type: 'string', default: '.', description: 'Path to folder containing \'src\' and \'test\' sub-folders.', })
+		// Validation
 		.check(argv => {
 			if (argv['global-scenarios-only'] && argv['features']) {
 				throw new Error('Cannot combine --global-scenarios-only and --features');
@@ -28,6 +35,7 @@ export function featuresTestOptions(y: Argv) {
 				throw new Error('Cannot combine --skip-scenarios and --global-scenarios-only');
 			}
 			return true;
+
 		});
 }
 
@@ -53,7 +61,8 @@ export function featuresTestHandler(args: FeaturesTestArgs) {
 
 async function featuresTest({
 	'base-image': baseImage,
-	'target': collectionFolder,
+	'target': collectionFolder_deprecated,
+	'project-folder': collectionFolder,
 	features,
 	'global-scenarios-only': globalScenariosOnly,
 	'skip-scenarios': skipScenarios,
@@ -73,13 +82,16 @@ async function featuresTest({
 
 	const logLevel = mapLogLevel(inputLogLevel);
 
+	// Prefer the new --project-folder option over the deprecated positional argument.
+	const targetProject = collectionFolder !== '.' ? collectionFolder : collectionFolder_deprecated;
+
 	const args: FeaturesTestCommandInput = {
 		baseImage,
 		cliHost,
 		logLevel,
 		quiet,
 		pkg,
-		collectionFolder: cliHost.path.resolve(collectionFolder),
+		collectionFolder: cliHost.path.resolve(targetProject),
 		features: features ? (Array.isArray(features) ? features as string[] : [features]) : undefined,
 		globalScenariosOnly,
 		skipScenarios,

src/spec-node/featuresCLI/testCommandImpl.ts

@@ -14,20 +14,20 @@ import { cpDirectoryLocal } from '../../spec-utils/pfs';
 const TEST_LIBRARY_SCRIPT_NAME = 'dev-container-features-test-lib';
 
 function fail(msg: string) {
-	log(msg, { prefix: '[-]', stderr: true });
+	log(msg, { prefix: '[-]', error: true });
 	process.exit(1);
 }
 
 type Scenarios = { [key: string]: DevContainerConfig };
 type TestResult = { testName: string; result: boolean };
 
-function log(msg: string, options?: { omitPrefix?: boolean; prefix?: string; info?: boolean; stderr?: boolean }) {
+function log(msg: string, options?: { omitPrefix?: boolean; prefix?: string; info?: boolean; error?: boolean }) {
 
 	const prefix = options?.prefix || '> ';
 	const output = `${options?.omitPrefix ? '' : `${prefix} `}${msg}\n`;
 
-	if (options?.stderr) {
-		process.stderr.write(chalk.red(output));
+	if (options?.error) {
+		process.stdout.write(chalk.red(output));
 	} else if (options?.info) {
 		process.stdout.write(chalk.bold.blue(output));
 	} else {
@@ -40,7 +40,7 @@ export async function doFeaturesTestCommand(args: FeaturesTestCommandInput): Pro
 
 	process.stdout.write(`
 ┌ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ┐
-|    dev container 'features' |   
+|    Dev Container Features   |   
 │           v${pkg.version}           │
 └ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ┘\n\n`);
 
@@ -163,16 +163,14 @@ async function doRunAutoTest(feature: string, workspaceFolder: string, params: D
 		fail(`Could not find test.sh script at ${testScriptPath}`);
 	}
 
-	// Move the test script into the workspaceFolder
-	const testScript = await cliHost.readFile(testScriptPath);
-	const remoteTestScriptName = `${feature}-test-${Date.now()}.sh`;
-	await cliHost.writeFile(`${workspaceFolder}/${remoteTestScriptName}`, testScript);
+	// Move the entire test directory for the given Feature into the workspaceFolder
+	await cpDirectoryLocal(featureTestFolder, workspaceFolder);
 
-	// Move the test library script into the workspaceFolder
-	await cliHost.writeFile(`${workspaceFolder}/${TEST_LIBRARY_SCRIPT_NAME}`, Buffer.from(testLibraryScript));
+	// Move the test library script into the workspaceFolder test scripts folder.
+	await cliHost.writeFile(path.join(workspaceFolder, TEST_LIBRARY_SCRIPT_NAME), Buffer.from(testLibraryScript));
 
 	// Execute Test
-	const result = await execTest(params, remoteTestScriptName, workspaceFolder);
+	const result = await execTest(params, 'test.sh', workspaceFolder);
 	testResults.push({
 		testName: feature,
 		result,
@@ -194,41 +192,41 @@ async function doScenario(pathToTestDir: string, args: FeaturesTestCommandInput,
 	const scenariosBuffer = await cliHost.readFile(scenariosPath);
 	// Parse to json
 	let scenarios: Scenarios = {};
-	try {
-		scenarios = jsonc.parse(scenariosBuffer.toString());
-	} catch (e) {
-		fail(`Failed to parse scenarios.json:  ${e.message}`);
-		return []; // We never reach here, we exit via fail().
+	let errors: jsonc.ParseError[] = [];
+	scenarios = jsonc.parse(scenariosBuffer.toString(), errors);
+	if (errors.length > 0) {
+		// Print each jsonc error
+		errors.forEach(error => {
+			log(`${jsonc.printParseErrorCode(error.error)}`, { prefix: '⚠️' });
+		});
+		fail(`Failed to parse scenarios.json at ${scenariosPath}`);
+		return []; // We never reach here, we exit via fail()
 	}
 
 	// For EACH scenario: Spin up a container and exec the scenario test script
 	for (const [scenarioName, scenarioConfig] of Object.entries(scenarios)) {
 		log(`Running scenario:  ${scenarioName}`);
 
 		// Check if we have a scenario test script, otherwise skip.
-		const scenarioTestScript = path.join(pathToTestDir, `${scenarioName}.sh`);
-		if (!(await cliHost.isFile(scenarioTestScript))) {
-			fail(`No scenario test script found at path '${scenarioTestScript}'.  Either add a script to the test folder, or remove from scenarios.json.`);
+		if (!(await cliHost.isFile(path.join(pathToTestDir, `${scenarioName}.sh`)))) {
+			fail(`No scenario test script found at path '${path.join(pathToTestDir, `${scenarioName}.sh`)}'.  Either add a script to the test folder, or remove from scenarios.json.`);
 		}
 
 		// Create Container
 		const workspaceFolder = await generateProjectFromScenario(cliHost, collectionFolder, scenarioName, scenarioConfig);
 		const params = await generateDockerParams(workspaceFolder, args);
 		await createContainerFromWorkingDirectory(params, workspaceFolder, args);
 
-		// Execute test script
-		// Move the test script into the workspaceFolder
-		const testScript = await cliHost.readFile(scenarioTestScript);
-		const remoteTestScriptName = `${scenarioName}-test-${Date.now()}.sh`;
-		await cliHost.writeFile(`${workspaceFolder}/${remoteTestScriptName}`, testScript);
+		// Move the entire test directory for the given Feature into the workspaceFolder
+		await cpDirectoryLocal(pathToTestDir, workspaceFolder);
 
 		// Move the test library script into the workspaceFolder
-		await cliHost.writeFile(`${workspaceFolder}/${TEST_LIBRARY_SCRIPT_NAME}`, Buffer.from(testLibraryScript));
+		await cliHost.writeFile(path.join(workspaceFolder, TEST_LIBRARY_SCRIPT_NAME), Buffer.from(testLibraryScript));
 
 		// Execute Test
 		testResults.push({
 			testName: scenarioName,
-			result: await execTest(params, remoteTestScriptName, workspaceFolder)
+			result: await execTest(params, `${scenarioName}.sh`, workspaceFolder)
 		});
 	}
 	return testResults;
@@ -409,13 +407,14 @@ async function launchProject(params: DockerResolverParameters, workspaceFolder:
 	}
 }
 
-async function execTest(params: DockerResolverParameters, remoteTestScriptName: string, workspaceFolder: string) {
+async function execTest(params: DockerResolverParameters, testFileName: string, workspaceFolder: string) {
+	// Ensure all the tests scripts in the workspace folder are executable
 	let cmd = 'chmod';
-	let args = ['777', `./${remoteTestScriptName}`, `./${TEST_LIBRARY_SCRIPT_NAME}`];
+	let args = ['-R', '777', '.'];
 	await exec(params, cmd, args, workspaceFolder);
 
 
-	cmd = `./${remoteTestScriptName}`;
+	cmd = `./${testFileName}`;
 	args = [];
 	return await exec(params, cmd, args, workspaceFolder);
 }

src/test/container-features/example-v2-features-sets/sharing-test-scripts/src/util/devcontainer-feature.json

@@ -0,0 +1,6 @@
+{
+    "id": "util",
+    "version": "1.0.0",
+    "name": "A utility",
+    "options": {}
+}

src/test/container-features/example-v2-features-sets/sharing-test-scripts/src/util/install.sh

@@ -0,0 +1,12 @@
+#!/bin/sh
+set -e
+
+echo "Activating feature 'util'"
+
+cat > /usr/local/bin/util \
+<< EOF
+#!/bin/sh
+echo "you did it"
+EOF
+
+chmod +x /usr/local/bin/util

src/test/container-features/example-v2-features-sets/sharing-test-scripts/test/util/a_different_script.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+echo "I AM A DIFFERENT SCRIPT"

src/test/container-features/example-v2-features-sets/sharing-test-scripts/test/util/a_helper_script_for_scenario.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+echo "I AM A HELPER SCRIPT FOR A SCENARIO"

src/test/container-features/example-v2-features-sets/sharing-test-scripts/test/util/scenarios.json

@@ -0,0 +1,8 @@
+{
+	"some_scenario": {
+		"image": "mcr.microsoft.com/devcontainers/base:ubuntu",
+		"features": {
+			"util": {}
+		}
+	}
+}

src/test/container-features/example-v2-features-sets/sharing-test-scripts/test/util/some_scenario.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+set -e
+
+# Optional: Import test library
+source dev-container-features-test-lib
+
+# Definition specific tests
+check "run a helper script for this scenario" ./a_helper_script_for_scenario.sh
+
+# Report result
+reportResults

src/test/container-features/example-v2-features-sets/sharing-test-scripts/test/util/test.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+set -e
+
+# Optional: Import test library
+source dev-container-features-test-lib
+
+# Definition specific tests
+check "run a different script" ./a_different_script.sh
+
+# Report result
+reportResults