Skip to content

Commit 094797d

Browse files
matejchalkmatejchalk
committed
feat(ci): implement run many command resolution for each monorepo tool
1 parent 0b9d679 commit 094797d

File tree

9 files changed

+286
-104
lines changed

9 files changed

+286
-104
lines changed

‎packages/ci/src/lib/monorepo/handlers/npm.ts

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,12 +10,14 @@ import type { MonorepoToolHandler } from '../tools.js';
1010

1111
export const npmHandler: MonorepoToolHandler = {
1212
tool: 'npm',
13+
1314
async isConfigured(options) {
1415
return (
1516
(await fileExists(join(options.cwd, 'package-lock.json'))) &&
1617
(await hasWorkspacesEnabled(options.cwd))
1718
);
1819
},
20+
1921
async listProjects(options) {
2022
const { workspaces, rootPackageJson } = await listWorkspaces(options.cwd);
2123
return workspaces
@@ -28,8 +30,13 @@ export const npmHandler: MonorepoToolHandler = {
2830
.map(({ name, packageJson }) => ({
2931
name,
3032
bin: hasScript(packageJson, options.task)
31-
? `npm -w ${name} run ${options.task} --`
32-
: `npm -w ${name} exec ${options.task} --`,
33+
? `npm --workspace=${name} run ${options.task} --`
34+
: `npm --workspace=${name} exec ${options.task} --`,
3335
}));
3436
},
37+
38+
createRunManyCommand(options) {
39+
// neither parallel execution nor projects filter are supported in NPM workspaces
40+
return `npm run ${options.task} --workspaces --if-present --`;
41+
},
3542
};

‎packages/ci/src/lib/monorepo/handlers/nx.ts

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import type { MonorepoToolHandler } from '../tools.js';
99

1010
export const nxHandler: MonorepoToolHandler = {
1111
tool: 'nx',
12+
1213
async isConfigured(options) {
1314
return (
1415
(await fileExists(join(options.cwd, 'nx.json'))) &&
@@ -18,10 +19,12 @@ export const nxHandler: MonorepoToolHandler = {
1819
args: ['nx', 'report'],
1920
cwd: options.cwd,
2021
observer: options.observer,
22+
ignoreExitCode: true,
2123
})
2224
).code === 0
2325
);
2426
},
27+
2528
async listProjects(options) {
2629
const { stdout } = await executeProcess({
2730
command: 'npx',
@@ -43,6 +46,19 @@ export const nxHandler: MonorepoToolHandler = {
4346
bin: `npx nx run ${project}:${options.task} --`,
4447
}));
4548
},
49+
50+
createRunManyCommand(options, onlyProjects) {
51+
return [
52+
'npx',
53+
'nx',
54+
'run-many', // TODO: allow affected instead of run-many?
55+
`--targets=${options.task}`,
56+
// TODO: add options.nxRunManyFilter? (e.g. --exclude=...)
57+
...(onlyProjects ? [`--projects=${onlyProjects.join(',')}`] : []),
58+
`--parallel=${options.parallel}`,
59+
'--',
60+
].join(' ');
61+
},
4662
};
4763

4864
function parseProjects(stdout: string): string[] {

‎packages/ci/src/lib/monorepo/handlers/pnpm.ts

Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,14 +11,19 @@ import type { MonorepoToolHandler } from '../tools.js';
1111

1212
const WORKSPACE_FILE = 'pnpm-workspace.yaml';
1313

14+
// https://pnpm.io/cli/recursive#--workspace-concurrency
15+
const DEFAULT_WORKSPACE_CONCURRENCY = 4;
16+
1417
export const pnpmHandler: MonorepoToolHandler = {
1518
tool: 'pnpm',
19+
1620
async isConfigured(options) {
1721
return (
1822
(await fileExists(join(options.cwd, WORKSPACE_FILE))) &&
1923
(await fileExists(join(options.cwd, 'package.json')))
2024
);
2125
},
26+
2227
async listProjects(options) {
2328
const yaml = await readTextFile(join(options.cwd, WORKSPACE_FILE));
2429
const workspace = YAML.parse(yaml) as { packages?: string[] };
@@ -34,8 +39,25 @@ export const pnpmHandler: MonorepoToolHandler = {
3439
.map(({ name, packageJson }) => ({
3540
name,
3641
bin: hasScript(packageJson, options.task)
37-
? `pnpm -F ${name} run ${options.task}`
38-
: `pnpm -F ${name} exec ${options.task}`,
42+
? `pnpm --filter=${name} run ${options.task}`
43+
: `pnpm --filter=${name} exec ${options.task}`,
3944
}));
4045
},
46+
47+
createRunManyCommand(options, onlyProjects) {
48+
const workspaceConcurrency: number =
49+
options.parallel === true
50+
? DEFAULT_WORKSPACE_CONCURRENCY
51+
: options.parallel === false
52+
? 1
53+
: options.parallel;
54+
return [
55+
'pnpm',
56+
'--recursive',
57+
`--workspace-concurrency=${workspaceConcurrency}`,
58+
...(onlyProjects?.map(project => `--filter=${project}`) ?? []),
59+
'run',
60+
options.task,
61+
].join(' ');
62+
},
4163
};

‎packages/ci/src/lib/monorepo/handlers/turbo.ts

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,19 +7,24 @@ import { yarnHandler } from './yarn.js';
77

88
const WORKSPACE_HANDLERS = [pnpmHandler, yarnHandler, npmHandler];
99

10+
// https://turbo.build/repo/docs/reference/run#--concurrency-number--percentage
11+
const DEFAULT_CONCURRENCY = 10;
12+
1013
type TurboConfig = {
1114
tasks: Record<string, object>;
1215
};
1316

1417
export const turboHandler: MonorepoToolHandler = {
1518
tool: 'turbo',
19+
1620
async isConfigured(options) {
1721
const configPath = join(options.cwd, 'turbo.json');
1822
return (
1923
(await fileExists(configPath)) &&
2024
options.task in (await readJsonFile<TurboConfig>(configPath)).tasks
2125
);
2226
},
27+
2328
async listProjects(options) {
2429
// eslint-disable-next-line functional/no-loop-statements
2530
for (const handler of WORKSPACE_HANDLERS) {
@@ -29,7 +34,7 @@ export const turboHandler: MonorepoToolHandler = {
2934
.filter(({ bin }) => bin.includes(`run ${options.task}`)) // must have package.json script
3035
.map(({ name }) => ({
3136
name,
32-
bin: `npx turbo run ${options.task} -F ${name} --`,
37+
bin: `npx turbo run ${options.task} --filter=${name} --`,
3338
}));
3439
}
3540
}
@@ -39,4 +44,22 @@ export const turboHandler: MonorepoToolHandler = {
3944
).join('/')}`,
4045
);
4146
},
47+
48+
createRunManyCommand(options, onlyProjects) {
49+
const concurrency: number =
50+
options.parallel === true
51+
? DEFAULT_CONCURRENCY
52+
: options.parallel === false
53+
? 1
54+
: options.parallel;
55+
return [
56+
'npx',
57+
'turbo',
58+
'run',
59+
options.task,
60+
...(onlyProjects?.map(project => `--filter=${project}`) ?? []),
61+
`--concurrency=${concurrency}`,
62+
'--',
63+
].join(' ');
64+
},
4265
};

‎packages/ci/src/lib/monorepo/handlers/yarn.ts

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
import { join } from 'node:path';
2-
import { fileExists } from '@code-pushup/utils';
2+
import { executeProcess, fileExists } from '@code-pushup/utils';
33
import {
44
hasCodePushUpDependency,
55
hasScript,
@@ -10,12 +10,14 @@ import type { MonorepoToolHandler } from '../tools.js';
1010

1111
export const yarnHandler: MonorepoToolHandler = {
1212
tool: 'yarn',
13+
1314
async isConfigured(options) {
1415
return (
1516
(await fileExists(join(options.cwd, 'yarn.lock'))) &&
1617
(await hasWorkspacesEnabled(options.cwd))
1718
);
1819
},
20+
1921
async listProjects(options) {
2022
const { workspaces, rootPackageJson } = await listWorkspaces(options.cwd);
2123
return workspaces
@@ -32,4 +34,26 @@ export const yarnHandler: MonorepoToolHandler = {
3234
: `yarn workspace ${name} exec ${options.task}`,
3335
}));
3436
},
37+
38+
async createRunManyCommand(options, onlyProjects) {
39+
const { stdout } = await executeProcess({ command: 'yarn', args: ['-v'] });
40+
const isV1 = stdout.startsWith('1.');
41+
42+
if (isV1) {
43+
// neither parallel execution nor projects filter are supported in Yarn v1
44+
return `yarn workspaces run ${options.task}`;
45+
}
46+
47+
return [
48+
'yarn',
49+
'workspaces',
50+
'foreach',
51+
...(options.parallel ? ['--parallel'] : []),
52+
...(typeof options.parallel === 'number'
53+
? [`--jobs=${options.parallel}`]
54+
: []),
55+
...(onlyProjects?.map(project => `--include=${project}`) ?? ['--all']),
56+
options.task,
57+
].join(' ');
58+
},
3559
};

‎packages/ci/src/lib/monorepo/list-projects.ts

Lines changed: 49 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -4,55 +4,80 @@ import type { Logger, Settings } from '../models.js';
44
import { detectMonorepoTool } from './detect-tool.js';
55
import { getToolHandler } from './handlers/index.js';
66
import { listPackages } from './packages.js';
7-
import type { MonorepoHandlerOptions, ProjectConfig } from './tools.js';
7+
import type {
8+
MonorepoHandlerOptions,
9+
MonorepoTool,
10+
ProjectConfig,
11+
} from './tools.js';
12+
13+
export type MonorepoProjects = {
14+
tool: MonorepoTool | null;
15+
projects: ProjectConfig[];
16+
runManyCommand?: (onlyProjects?: string[]) => string | Promise<string>;
17+
};
818

919
export async function listMonorepoProjects(
1020
settings: Settings,
11-
): Promise<ProjectConfig[]> {
12-
if (!settings.monorepo) {
13-
throw new Error('Monorepo mode not enabled');
14-
}
15-
21+
): Promise<MonorepoProjects> {
1622
const logger = settings.logger;
17-
1823
const options = createMonorepoHandlerOptions(settings);
1924

20-
const tool =
21-
settings.monorepo === true
22-
? await detectMonorepoTool(options)
23-
: settings.monorepo;
24-
if (settings.monorepo === true) {
25-
if (tool) {
26-
logger.info(`Auto-detected monorepo tool ${tool}`);
27-
} else {
28-
logger.info("Couldn't auto-detect any supported monorepo tool");
29-
}
30-
} else {
31-
logger.info(`Using monorepo tool "${tool}" from inputs`);
32-
}
25+
const tool = await resolveMonorepoTool(settings, options);
3326

3427
if (tool) {
3528
const handler = getToolHandler(tool);
3629
const projects = await handler.listProjects(options);
3730
logger.info(`Found ${projects.length} projects in ${tool} monorepo`);
3831
logger.debug(`Projects: ${projects.map(({ name }) => name).join(', ')}`);
39-
return projects;
32+
return {
33+
tool,
34+
projects,
35+
runManyCommand: onlyProjects =>
36+
handler.createRunManyCommand(options, onlyProjects),
37+
};
4038
}
4139

4240
if (settings.projects) {
43-
return listProjectsByGlobs({
41+
const projects = await listProjectsByGlobs({
4442
patterns: settings.projects,
4543
cwd: options.cwd,
4644
bin: settings.bin,
4745
logger,
4846
});
47+
return { tool, projects };
4948
}
5049

51-
return listProjectsByNpmPackages({
50+
const projects = await listProjectsByNpmPackages({
5251
cwd: options.cwd,
5352
bin: settings.bin,
5453
logger,
5554
});
55+
return { tool, projects };
56+
}
57+
58+
async function resolveMonorepoTool(
59+
settings: Settings,
60+
options: MonorepoHandlerOptions,
61+
): Promise<MonorepoTool | null> {
62+
if (!settings.monorepo) {
63+
// shouldn't happen, handled by caller
64+
throw new Error('Monorepo mode not enabled');
65+
}
66+
const logger = settings.logger;
67+
68+
if (typeof settings.monorepo === 'string') {
69+
logger.info(`Using monorepo tool "${settings.monorepo}" from inputs`);
70+
return settings.monorepo;
71+
}
72+
73+
const tool = await detectMonorepoTool(options);
74+
if (tool) {
75+
logger.info(`Auto-detected monorepo tool ${tool}`);
76+
} else {
77+
logger.info("Couldn't auto-detect any supported monorepo tool");
78+
}
79+
80+
return tool;
5681
}
5782

5883
function createMonorepoHandlerOptions(
@@ -61,6 +86,7 @@ function createMonorepoHandlerOptions(
6186
return {
6287
task: settings.task,
6388
cwd: settings.directory,
89+
parallel: false, // TODO: add to settings
6490
nxProjectsFilter: settings.nxProjectsFilter,
6591
...(!settings.silent && {
6692
observer: {

0 commit comments

Comments
 (0)