Skip to content

chore(test): collect cli output in tests #2198

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
rubiesonthesky wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

chore(test): collect cli output in tests #2198

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 34 additions & 8 deletions src/shared/runtime.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,13 @@ export const findFirstMutations = (
}

if (result !== undefined && result.length !== 0) {
logOutput(request, mutatorName, "found mutations", result);
const stats = mutationStats(result);
logOutput(
request,
mutatorName,
`found ${result.length} mutations${stats}`,
result,
);
return result;
}
} catch (error) {
Expand All @@ -36,12 +42,32 @@ const logOutput = (
data: unknown,
) => {
request.options.output.log?.(
[
mutatorName,
` ${action} in `,
request.sourceFile.fileName,
": ",
JSON.stringify(data, null, 4),
].join(""),
`${mutatorName} ${action} in ${request.sourceFile.fileName}: ${JSON.stringify(data, null, 4)}`,
);
};

const mutationStats = (mutations: readonly Mutation[]) => {
let deletions = 0;
let insertions = 0;
let swaps = 0;

for (const mutation of mutations) {
if (mutation.type === "text-delete") {
deletions++;
} else if (mutation.type === "text-insert") {
insertions++;
} else if (mutation.type === "text-swap") {
swaps++;
}
// there is also "multiple" mutation type
}

const stats = [
insertions ? `${insertions} insertions` : undefined,
deletions ? `${deletions} deletions` : undefined,
swaps ? `${swaps} swaps` : undefined,
]
.filter((x) => !!x)
.join(", ");
return stats ? ` (${stats})` : "";
};
49 changes: 38 additions & 11 deletions src/tests/testSetup.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,14 +1,17 @@
import { runMutations } from "automutate";
import fs from "node:fs/promises";
import path from "node:path";
import { expect } from "vitest";

import { loadPendingOptions } from "../options/loadPendingOptions.js";
import { ProcessOutput } from "../output/types.js";
import { createTypeStatProvider } from "../runtime/createTypeStatProvider.js";

export interface MutationTestResult {
actualContent: string;
expectedFilePath: string;
options: string;
output: string;
}

export const runMutationTest = async (
Expand All @@ -31,12 +34,27 @@ export const runMutationTest = async (
// file needs to exists before creating compiler options
await fs.copyFile(originalFile, actualFile);

const output = {
// eslint-disable-next-line @typescript-eslint/no-empty-function
log: () => {},
const replaceFilePath = (value: string) =>
value.replaceAll(dirPath, "<rootDir>");

const cliOutput: string[] = [];

const output: ProcessOutput = {
log: (line: string) => {
if (
line.includes("text-insert") ||
line.includes("text-delete") ||
line.includes("text-swap")
) {
const index = line.indexOf("[");
line = line.slice(0, index) + "[mutations]";
}
cliOutput.push("[log] " + replaceFilePath(line));
},
stderr: console.error.bind(console),
// eslint-disable-next-line @typescript-eslint/no-empty-function
stdout: () => {},
stdout: (line: string) => {
cliOutput.push("[stdout] " + replaceFilePath(line));
},
};

const pendingOptionsList = loadPendingOptions(
Expand All @@ -61,12 +79,21 @@ export const runMutationTest = async (
const actualContent = await readFile(actualFileName);
const expectFileName = `expected.ts${fileNameSuffix}`;
const expectedFilePath = path.join(dirPath, expectFileName);
const optionsSnapshot = JSON.stringify(pendingOptionsList, null, 2);

const optionsSnapshot = JSON.stringify(
pendingOptionsList,
null,
2,
).replaceAll(dirPath, "<rootDir>");
return {
actualContent,
expectedFilePath,
options: replaceFilePath(optionsSnapshot),
output: cliOutput.join("\n"),
};
};

return { actualContent, expectedFilePath, options: optionsSnapshot };
export const checkTestResult = async (cwd: string, caseDir: string) => {
const fullPath = path.join(cwd, "cases", caseDir);
const { actualContent, expectedFilePath, options, output } =
await runMutationTest(fullPath);
await expect(actualContent).toMatchFileSnapshot(expectedFilePath);
expect(options).toMatchSnapshot("options");
expect(output).toMatchSnapshot("output");
Comment on lines +96 to +98
Copy link
Owner

@JoshuaKGoldberg JoshuaKGoldberg Feb 17, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[Refactor] Having multiple assertions in a test means we only get insights from one failure at a time. If assertion 1 of 3 fails, we don't know at a glance if 2 or 3 did.

My normal trick for this kind of thing is:

Suggested change
await expect(actualContent).toMatchFileSnapshot(expectedFilePath);
expect(options).toMatchSnapshot("options");
expect(output).toMatchSnapshot("output");
await expect({ actualContent, options, output }).toMatchFileSnapshot(expectedFilePath);

};
31 changes: 31 additions & 0 deletions test/__snapshots__/cleanups.test.ts.snap
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,21 @@ exports[`Cleanups > non-TypeErrors > options 1`] = `
]"
`;

exports[`Cleanups > non-TypeErrors > output 1`] = `
"[log] Preparing language services to visit files...
[log] Prepared language services for 1 files...
[stdout] Starting the core mutation engine. This terminal will log whenever a "wave" of mutations are written to files.
[stdout] Core mutations will complete when two waves pass with no mutations.
[stdout] The following fixes will be applied:
[stdout] Starting wave 1...
[stdout] Completed wave 1. Wrote mutations to 0 files.
[stdout] Done.

[stdout] Applying post-fix cleanups...
[stdout] Done.
"
`;

exports[`Cleanups > suppressTypeErrors > options 1`] = `
"[
{
Expand Down Expand Up @@ -207,3 +222,19 @@ exports[`Cleanups > suppressTypeErrors > options 1`] = `
}
]"
`;

exports[`Cleanups > suppressTypeErrors > output 1`] = `
"[log] Preparing language services to visit files...
[log] Prepared language services for 1 files...
[stdout] Starting the core mutation engine. This terminal will log whenever a "wave" of mutations are written to files.
[stdout] Core mutations will complete when two waves pass with no mutations.
[stdout] The following fixes will be applied:
[stdout] Starting wave 1...
[stdout] Completed wave 1. Wrote mutations to 0 files.
[stdout] Done.

[stdout] Applying post-fix cleanups...
[log] suppressTypeIssues found 4 mutations (4 insertions) in <rootDir>/actual.ts: [mutations]
[stdout] Done.
"
`;
60 changes: 60 additions & 0 deletions test/__snapshots__/customMutators.test.ts.snap
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,21 @@ exports[`Custom mutators > empty array > options 1`] = `
]"
`;

exports[`Custom mutators > empty array > output 1`] = `
"[log] Preparing language services to visit files...
[log] Prepared language services for 1 files...
[stdout] Starting the core mutation engine. This terminal will log whenever a "wave" of mutations are written to files.
[stdout] Core mutations will complete when two waves pass with no mutations.
[stdout] The following fixes will be applied:
[stdout] Starting wave 1...
[stdout] Completed wave 1. Wrote mutations to 0 files.
[stdout] Done.

[stdout] Applying post-fix cleanups...
[stdout] Done.
"
`;

exports[`Custom mutators > one mutator > options 1`] = `
"[
{
Expand Down Expand Up @@ -174,6 +189,26 @@ exports[`Custom mutators > one mutator > options 1`] = `
]"
`;

exports[`Custom mutators > one mutator > output 1`] = `
"[log] Preparing language services to visit files...
[log] Prepared language services for 1 files...
[stdout] Starting the core mutation engine. This terminal will log whenever a "wave" of mutations are written to files.
[stdout] Core mutations will complete when two waves pass with no mutations.
[stdout] The following fixes will be applied:
[stdout] Starting wave 1...
[log] ./sampleMutator.cjs found 1 mutations (1 insertions) in <rootDir>/actual.ts: [mutations]
[stdout] Completed wave 1. Wrote mutations to 1 file.
[log] Preparing language services to visit files...
[log] Prepared language services for 1 files...
[stdout] Starting wave 2...
[stdout] Completed wave 2. Wrote mutations to 0 files.
[stdout] Done.

[stdout] Applying post-fix cleanups...
[stdout] Done.
"
`;

exports[`Custom mutators > two mutators > options 1`] = `
"[
{
Expand Down Expand Up @@ -252,3 +287,28 @@ exports[`Custom mutators > two mutators > options 1`] = `
}
]"
`;

exports[`Custom mutators > two mutators > output 1`] = `
"[log] Preparing language services to visit files...
[log] Prepared language services for 1 files...
[stdout] Starting the core mutation engine. This terminal will log whenever a "wave" of mutations are written to files.
[stdout] Core mutations will complete when two waves pass with no mutations.
[stdout] The following fixes will be applied:
[stdout] Starting wave 1...
[log] ./mutator1.cjs found 1 mutations (1 insertions) in <rootDir>/actual.ts: [mutations]
[stdout] Completed wave 1. Wrote mutations to 1 file.
[log] Preparing language services to visit files...
[log] Prepared language services for 1 files...
[stdout] Starting wave 2...
[log] ./mutator2.cjs found 1 mutations (1 insertions) in <rootDir>/actual.ts: [mutations]
[stdout] Completed wave 2. Wrote mutations to 1 file.
[log] Preparing language services to visit files...
[log] Prepared language services for 1 files...
[stdout] Starting wave 3...
[stdout] Completed wave 3. Wrote mutations to 0 files.
[stdout] Done.

[stdout] Applying post-fix cleanups...
[stdout] Done.
"
`;
92 changes: 92 additions & 0 deletions test/__snapshots__/files.test.ts.snap
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,29 @@ exports[`files > addition above > options 1`] = `
]"
`;

exports[`files > addition above > output 1`] = `
"[log] Preparing language services to visit files...
[log] Prepared language services for 1 files...
[stdout] Starting the core mutation engine. This terminal will log whenever a "wave" of mutations are written to files.
[stdout] Core mutations will complete when two waves pass with no mutations.
[stdout] The following fix will be applied:
[stdout] * incompleteTypes

[stdout] Starting wave 1...
[log] fixIncompleteVariableTypes found 1 mutations (1 insertions) in <rootDir>/actual.ts: [mutations]
[log] fixIncompleteTypes found 1 mutations (1 insertions) in <rootDir>/actual.ts: [mutations]
[stdout] Completed wave 1. Wrote mutations to 1 file.
[log] Preparing language services to visit files...
[log] Prepared language services for 1 files...
[stdout] Starting wave 2...
[stdout] Completed wave 2. Wrote mutations to 0 files.
[stdout] Done.

[stdout] Marking files as modified...
[stdout] Done.
"
`;

exports[`files > addition below > options 1`] = `
"[
{
Expand Down Expand Up @@ -198,6 +221,29 @@ exports[`files > addition below > options 1`] = `
]"
`;

exports[`files > addition below > output 1`] = `
"[log] Preparing language services to visit files...
[log] Prepared language services for 1 files...
[stdout] Starting the core mutation engine. This terminal will log whenever a "wave" of mutations are written to files.
[stdout] Core mutations will complete when two waves pass with no mutations.
[stdout] The following fix will be applied:
[stdout] * incompleteTypes

[stdout] Starting wave 1...
[log] fixIncompleteVariableTypes found 1 mutations (1 insertions) in <rootDir>/actual.ts: [mutations]
[log] fixIncompleteTypes found 1 mutations (1 insertions) in <rootDir>/actual.ts: [mutations]
[stdout] Completed wave 1. Wrote mutations to 1 file.
[log] Preparing language services to visit files...
[log] Prepared language services for 1 files...
[stdout] Starting wave 2...
[stdout] Completed wave 2. Wrote mutations to 0 files.
[stdout] Done.

[stdout] Marking files as modified...
[stdout] Done.
"
`;

exports[`files > both > options 1`] = `
"[
{
Expand Down Expand Up @@ -297,6 +343,29 @@ exports[`files > both > options 1`] = `
]"
`;

exports[`files > both > output 1`] = `
"[log] Preparing language services to visit files...
[log] Prepared language services for 1 files...
[stdout] Starting the core mutation engine. This terminal will log whenever a "wave" of mutations are written to files.
[stdout] Core mutations will complete when two waves pass with no mutations.
[stdout] The following fix will be applied:
[stdout] * incompleteTypes

[stdout] Starting wave 1...
[log] fixIncompleteVariableTypes found 1 mutations (1 insertions) in <rootDir>/actual.ts: [mutations]
[log] fixIncompleteTypes found 1 mutations (1 insertions) in <rootDir>/actual.ts: [mutations]
[stdout] Completed wave 1. Wrote mutations to 1 file.
[log] Preparing language services to visit files...
[log] Prepared language services for 1 files...
[stdout] Starting wave 2...
[stdout] Completed wave 2. Wrote mutations to 0 files.
[stdout] Done.

[stdout] Marking files as modified...
[stdout] Done.
"
`;

exports[`files > empty addition > options 1`] = `
"[
{
Expand Down Expand Up @@ -395,3 +464,26 @@ exports[`files > empty addition > options 1`] = `
}
]"
`;

exports[`files > empty addition > output 1`] = `
"[log] Preparing language services to visit files...
[log] Prepared language services for 1 files...
[stdout] Starting the core mutation engine. This terminal will log whenever a "wave" of mutations are written to files.
[stdout] Core mutations will complete when two waves pass with no mutations.
[stdout] The following fix will be applied:
[stdout] * incompleteTypes

[stdout] Starting wave 1...
[log] fixIncompleteVariableTypes found 1 mutations (1 insertions) in <rootDir>/actual.ts: [mutations]
[log] fixIncompleteTypes found 1 mutations (1 insertions) in <rootDir>/actual.ts: [mutations]
[stdout] Completed wave 1. Wrote mutations to 1 file.
[log] Preparing language services to visit files...
[log] Prepared language services for 1 files...
[stdout] Starting wave 2...
[stdout] Completed wave 2. Wrote mutations to 0 files.
[stdout] Done.

[stdout] Applying post-fix cleanups...
[stdout] Done.
"
`;
Loading