Update test runner with console output

.gitignore

@@ -11,6 +11,7 @@ junit.xml
 results.xml
 tmp
 
+debug.log
 yarn-error.log
 
 # Docker base files

CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog
 
+## 2.1.0
+
+- Add `output` per test (user `console.log`)
+- Add `message` and `status: fail` when there are 0 tests
+
 ## 2.0.0
 
 - Add deploy to dockerhub

bin/run.sh

@@ -77,18 +77,23 @@ set -euo pipefail
 
 ROOT="$(realpath $(dirname "$0")/..)"
 REPORTER="$ROOT/dist/reporter.js"
+SETUP="$ROOT/dist/jest/setup.js"
+
 if test -f "$REPORTER"; then
-  echo "Using reporter: $REPORTER"
-  echo "Using testroot: $INPUT"
-  echo "Using baseroot: $ROOT"
+  echo "Using reporter : $REPORTER"
+  echo "Using test-root: $INPUT"
+  echo "Using base-root: $ROOT"
+  echo "Using setup-env: $SETUP"
+
   echo ""
-  echo $(ls $ROOT/dist)
 else
   >&2 echo "Expected reporter.js to exist. Did you forget to yarn build first?"
-  >&2 echo "With reporter: $REPORTER"
-  >&2 echo "With testroot: $INPUT"
-  >&2 echo "With baseroot: $ROOT"
+  >&2 echo "Using reporter : $REPORTER"
+  >&2 echo "Using test-root: $INPUT"
+  >&2 echo "Using base-root: $ROOT"
+  >&2 echo "Using setup-env: $SETUP"
   >&2 echo ""
+  >&2 echo "The following files exist in the dist folder (build output):"
   >&2 echo $(ls $ROOT/dist)
   exit 1
 fi
@@ -101,16 +106,18 @@ test_file="${INPUT}${SLUG}.spec.js"
 # Put together the path to the test results file
 result_file="${OUTPUT}results.json"
 
-# Change xtest to test so all tests are run
-if [[ "$OSTYPE" == "darwin"* ]]; then # Mac OS X
-  # BSD sed -i takes an extra parameter to specify the backup file extension
-  sed -i 'tmp' 's/xtest(/test(/g' "${test_file}"
-  sed -i 'tmp' 's/xit(/it(/g' "${test_file}"
-  sed -i 'tmp' 's/xdescribe(/describe(/g' "${test_file}"
-else
-  sed -i 's/xtest(/test(/g' "${test_file}"
-  sed -i 's/xit(/it(/g' "${test_file}"
-  sed -i 's/xdescribe(/describe(/g' "${test_file}"
+if test -f "$test_file"; then
+  # Change xtest to test so all tests are run
+  if [[ "$OSTYPE" == "darwin"* ]]; then # Mac OS X
+    # BSD sed -i takes an extra parameter to specify the backup file extension
+    sed -i 'tmp' 's/xtest(/test(/g' "${test_file}"
+    sed -i 'tmp' 's/xit(/it(/g' "${test_file}"
+    sed -i 'tmp' 's/xdescribe(/describe(/g' "${test_file}"
+  else
+    sed -i 's/xtest(/test(/g' "${test_file}"
+    sed -i 's/xit(/it(/g' "${test_file}"
+    sed -i 's/xdescribe(/describe(/g' "${test_file}"
+  fi
 fi
 
 mkdir -p "${OUTPUT}"
@@ -119,12 +126,17 @@ mkdir -p "${OUTPUT}"
 set +e
 
 # Run tests
-"$ROOT/node_modules/.bin/jest" test --no-cache "${INPUT}*" \
+"$ROOT/node_modules/.bin/jest" "${INPUT}*" \
                                --outputFile="${result_file}" \
                                --reporters "${REPORTER}" \
                                --noStackTrace \
                                --verbose=false \
-                               --roots "${INPUT}"
+                               --roots "${INPUT}" \
+                               --passWithNoTests \
+                               --ci \
+                               --runInBand \
+                               --bail 1 \
+                               --setupFilesAfterEnv ${SETUP}
 
 # Convert exit(1) (jest worked, but there are failing tests) to exit(0)
 test_exit=$?

package.json

@@ -54,14 +54,5 @@
     "eslint-plugin-jest": "^24.1.3",
     "rimraf": "^3.0.2",
     "typescript": "^4.1.3"
-  },
-  "jest": {
-    "modulePathIgnorePatterns": [
-      "package.json"
-    ],
-    "reporters": [
-      "<rootDir>/dist/reporter.js"
-    ],
-    "setupFilesAfterEnv": []
   }
 }

src/jest/setup.ts

@@ -1,4 +1,4 @@
-import spyConsole from './console'
+// import spyConsole from './console'
 
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 const originalDescribe = (jasmine as any).getEnv().describe
@@ -10,16 +10,16 @@ const originalDescribe = (jasmine as any).getEnv().describe
   ...describeArgs: T
 ) => {
   function spiedSpecDefinition(...args: T): void {
-    const restores: Array<() => void> = []
+    // const restores: Array<() => void> = []
 
     beforeEach(() => {
-      restores.push(spyConsole().restore)
+      // restores.push(spyConsole().restore)
+      console.log(`@exercism/javascript:${expect.getState().currentTestName}`)
     })
 
     afterEach(() => {
-      const restore = restores.shift()!
-
-      restore()
+      // const restore = restores.shift()!
+      // restore()
     })
 
     return specDefinitions(...args)

src/output.ts

@@ -48,9 +48,26 @@ export class Output {
         aggregatedResults.numFailedTests === 0
           ? 'pass'
           : 'fail'
+
+      // Divert status if nothing ran
+      if (
+        this.results.status === 'pass' &&
+        aggregatedResults.numPassedTests === 0
+      ) {
+        this.results.status = 'fail'
+        this.error(
+          'Expected to run at least one test, but none were found. This can ' +
+            'happen if the test file(s) (.spec.js) are missing or empty. ' +
+            'These files are normally not empty. Revert any changes or report ' +
+            'an issue if the problem persists.'
+        )
+      }
     }
 
-    const artifact = JSON.stringify(this.results, undefined, 2)
+    // Re-order the output so that tests output shows below main output
+    const { status, message, tests } = this.results
+
+    const artifact = JSON.stringify({ status, message, tests }, undefined, 2)
 
     fs.writeFileSync(this.outputFile, artifact)
   }
@@ -104,34 +121,78 @@ export class Output {
         message: ''
       })
       */
-
-      this.results.message = sanitizeErrorMessage(
-        specFilePath,
-        buildOutput(testResult.console)
-      )
     }
 
+    const consoleOutputs = testResult.console
+      ? buildOutput(specFilePath, testResult.console)
+      : ({} as Record<string, string>)
+
     const outputs = buildTestOutput(specFilePath, testResult, innerResults)
-    this.results.tests.push(...outputs.map((r) => ({ ...r, output: null })))
+    const firstFailureIndex = outputs.findIndex(
+      (output) => output.status === 'fail'
+    )
+
+    this.results.tests.push(
+      ...outputs.map((withoutOutput, i, self) => {
+        const isFirstFailure =
+          firstFailureIndex === i ||
+          (firstFailureIndex === -1 && self.length === i + 1)
+        const outputMessage =
+          consoleOutputs[withoutOutput.name.replace(/ > /g, ' ')] || null
+
+        return {
+          ...withoutOutput,
+          output: isFirstFailure
+            ? [consoleOutputs[''], outputMessage].filter(Boolean).join('\n')
+            : outputMessage,
+        }
+      })
+    )
   }
 
   public testStarted(_path: string): void {
     // noop
   }
 }
 
-function buildOutput(buffer: ConsoleBuffer): string {
-  const output = buffer
-    .map((entry) => `[${entry.type}] ${entry.message}`)
-    .join('\n')
+function buildOutput(
+  specFilePath: string,
+  buffer: ConsoleBuffer
+): Record<string, string> {
+  const [, outputs] = buffer.reduce(
+    ([lastTest, messages], entry) => {
+      // Change current test messages
+      if (entry.message.startsWith('@exercism/javascript:')) {
+        return [
+          entry.message.slice('@exercism/javascript:'.length).trim(),
+          messages,
+        ] as const
+      }
 
-  if (output.length > 500) {
-    return output
-      .slice(0, 500 - '... (500 chars max)'.length)
-      .concat('... (500 chars max)')
-  }
+      const sanitized = `[${entry.type}] ${sanitizeErrorMessage(
+        specFilePath,
+        entry.message
+      )}`
+      messages[lastTest] ||= []
+      messages[lastTest].push(sanitized)
+
+      return [lastTest, messages] as const
+    },
+    ['', {}] as readonly [string, Record<string, string[]>]
+  )
+
+  return Object.keys(outputs).reduce((results, key) => {
+    const message = (outputs[key] || []).join('\n') || ''
+    if (message.length <= 500) {
+      results[key] = message
+    } else {
+      results[key] = message
+        .slice(0, 500 - '... (500 chars max)'.length)
+        .concat('... (500 chars max)')
+    }
 
-  return output
+    return results
+  }, {} as Record<string, string>)
 }
 
 function buildTestOutput(
@@ -142,14 +203,14 @@ function buildTestOutput(
   if (testResult.testExecError) {
     return [
       {
+        name: testResult.testFilePath,
+        status: 'error',
         message: sanitizeErrorMessage(
           path,
           testResult.failureMessage
             ? removeStackTrace(testResult.failureMessage)
             : testResult.testExecError.message
         ),
-        name: testResult.testFilePath,
-        status: 'error',
       },
     ]
   }

test/fixtures/two-fer/error/empty/two-fer.js

@@ -0,0 +1 @@
+// purposefully empty

test/fixtures/two-fer/error/empty/two-fer.spec.js

@@ -0,0 +1,17 @@
+import { twoFer } from './two-fer';
+
+describe('twoFer()', () => {
+  test('no name given', () => {
+    expect(twoFer()).toEqual('One for you, one for me.');
+  });
+
+  test('a name given', () => {
+    const name = 'Alice';
+    expect(twoFer(name)).toEqual('One for Alice, one for me.');
+  });
+
+  test('another name given', () => {
+    const name = 'Bob';
+    expect(twoFer(name)).toEqual('One for Bob, one for me.');
+  });
+});

test/fixtures/two-fer/error/missing/two-fer.js

@@ -0,0 +1 @@
+export const twoFer = (name = 'you') => `One for ${name}, one for me.`

test/fixtures/two-fer/error/syntax/two-fer.js

@@ -0,0 +1 @@
+export const const () => { (

test/fixtures/two-fer/error/syntax/two-fer.spec.js

@@ -0,0 +1,17 @@
+import { twoFer } from './two-fer';
+
+describe('twoFer()', () => {
+  test('no name given', () => {
+    expect(twoFer()).toEqual('One for you, one for me.');
+  });
+
+  test('a name given', () => {
+    const name = 'Alice';
+    expect(twoFer(name)).toEqual('One for Alice, one for me.');
+  });
+
+  test('another name given', () => {
+    const name = 'Bob';
+    expect(twoFer(name)).toEqual('One for Bob, one for me.');
+  });
+});