Skip to content
This repository was archived by the owner on Feb 25, 2025. It is now read-only.

Initializes RBE in the build config runner #50543

Merged
merged 1 commit into from
Feb 12, 2024
Merged

Initializes RBE in the build config runner #50543

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 1 addition & 31 deletions tools/gn
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -219,36 +219,6 @@ def setup_rbe(args):
# care of starting and stopping the compiler proxy.
running_on_luci = os.environ.get('LUCI_CONTEXT') is not None

# Bootstrap reproxy if not running in CI.
if not running_on_luci:
cipd_reclient_dir = os.path.join(
SRC_ROOT,
'buildtools',
buildtools_dir(),
'reclient',
)
bootstrap_path = os.path.join(cipd_reclient_dir, 'bootstrap')
bootstrap_path = bootstrap_path + '.exe' if get_host_os() == 'win' else bootstrap_path
reproxy_path = os.path.join(cipd_reclient_dir, 'reproxy')
rbe_cfg_path = os.path.join(
SRC_ROOT,
'flutter',
'build',
'rbe',
'reclient-' + get_host_os() + '.cfg',
)
bootstrap_cmd = [
bootstrap_path,
'--re_proxy=' + reproxy_path,
'--automatic_auth=true',
'--cfg=' + rbe_cfg_path,
]
try:
subprocess.call(bootstrap_cmd, cwd=SRC_ROOT)
except subprocess.CalledProcessError as exc:
print('Failed to boostrap reproxy: ', exc.returncode, exc.output)
return {}

if args.rbe_server_address:
rbe_gn_args['rbe_server_address'] = args.rbe_server_address
if args.rbe_exec_strategy:
Expand Down Expand Up @@ -1188,7 +1158,7 @@ def parse_args(args):

parser.add_argument(
'--trace-gn',
default=False,
default=True,
Copy link
Member Author

@zanderso zanderso Feb 12, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We can use this to debug the gn command being slow, and I believe there are no downsides to turning it on by default.

action='store_true',
help='Write a GN trace log (gn_trace.json) in the Chromium tracing '
'format in the build directory.'
Expand Down
9 changes: 9 additions & 0 deletions tools/pkg/engine_build_configs/bin/run.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,11 +108,20 @@ The build names are the "name" fields of the maps in the list of "builds".
return;
}

// If RBE config files aren't in the tree, then disable RBE.
final String rbeConfigPath = p.join(
engine.srcDir.path, 'flutter', 'build', 'rbe',
);
final List<String> extraGnArgs = <String>[
if (!io.Directory(rbeConfigPath).existsSync()) '--no-rbe',
];

final GlobalBuildRunner buildRunner = GlobalBuildRunner(
platform: const LocalPlatform(),
processRunner: ProcessRunner(),
engineSrcDir: engine.srcDir,
build: targetBuild,
extraGnArgs: extraGnArgs,
runGenerators: false,
runTests: false,
);
Expand Down
205 changes: 146 additions & 59 deletions tools/pkg/engine_build_configs/lib/src/build_config_runner.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@

import 'dart:async';
import 'dart:convert';
import 'dart:io' as io show Directory, Process;
import 'dart:io' as io show Directory, Process, ProcessResult;

import 'package:path/path.dart' as p;
import 'package:platform/platform.dart';
Expand Down Expand Up @@ -41,7 +41,7 @@ final class RunnerProgress extends RunnerEvent {
RunnerProgress(
super.name, super.command, super.timestamp,
this.what, this.completed, this.total, this.done,
) : percent = (completed * 1000) / total;
) : percent = (completed * 100) / total;
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oops.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That would have been exciting


/// What a command is currently working on, for example a build target or
/// the name of a test.
Expand Down Expand Up @@ -263,7 +263,6 @@ final class GlobalBuildRunner extends Runner {
gnArgs.remove(arg.$1);
}
}

return gnArgs;
}();

Expand All @@ -289,67 +288,157 @@ final class GlobalBuildRunner extends Runner {
return result.ok;
}

// TODO(zanderso): This should start and stop RBE when it is an --rbe build.
Future<bool> _runNinja(RunnerEventHandler eventHandler) async {
final String ninjaPath = p.join(
engineSrcDir.path, 'flutter', 'third_party', 'ninja', 'ninja',
late final String _hostCpu = (){
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You might (here and below) be able to get away with a regular final

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Tried this. Since here and below the initializers access instance members like platform, they can't be regular final.

if (platform.isWindows) {
return platform.environment['PROCESSOR_ARCHITECTURE'] ?? 'x64';
}
final List<String> unameCommand = <String>['uname', '-m'];
final io.ProcessResult unameResult = processRunner.processManager.runSync(
unameCommand,
);
final String outDir = p.join(engineSrcDir.path, 'out', build.ninja.config);
final List<String> command = <String>[
ninjaPath,
'-C', outDir,
if (_isGomaOrRbe) ...<String>['-j', '200'],
...extraNinjaArgs,
...build.ninja.targets,
];
eventHandler(RunnerStart('${build.name}: ninja', command, DateTime.now()));
return unameResult.exitCode == 0 ? (unameResult.stdout as String).trim() : 'x64';
}();
Comment on lines +291 to +300
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'd consider using Abi.current for this: https://api.dart.dev/stable/3.2.6/dart-ffi/Abi-class.html


final ProcessRunnerResult processResult;
late final String _buildtoolsPath = (){
final String platformDir = switch (platform.operatingSystem) {
Platform.linux => 'linux-$_hostCpu',
Platform.macOS => 'mac-$_hostCpu',
Platform.windows => 'windows-$_hostCpu',
_ => '<unknown>',
};
return p.join(engineSrcDir.path, 'buildtools', platformDir);
}();

Future<bool> _bootstrapRbe(
RunnerEventHandler eventHandler, {
bool shutdown = false,
}) async {
final String reclientPath = p.join(_buildtoolsPath, 'reclient');
final String exe = platform.isWindows ? '.exe' : '';
final String bootstrapPath = p.join(reclientPath, 'bootstrap$exe');
final String reproxyPath = p.join(reclientPath, 'reproxy$exe');
final String reclientConfigFile = switch (platform.operatingSystem) {
Platform.linux => 'reclient-linux.cfg',
Platform.macOS => 'reclient-mac.cfg',
Platform.windows => 'reclient-win.cfg',
_ => '<unknown>',
};
final String reclientConfigPath = p.join(
engineSrcDir.path, 'flutter', 'build', 'rbe', reclientConfigFile,
);
final List<String> bootstrapCommand = <String>[
bootstrapPath,
'--re_proxy=$reproxyPath',
'--automatic_auth=true',
if (shutdown)
'--shutdown'
else ...<String>[
'--cfg=$reclientConfigPath'
],
];
if (!processRunner.processManager.canRun(bootstrapPath)) {
eventHandler(RunnerError(
build.name, <String>[], DateTime.now(), '"$bootstrapPath" not found.',
));
return false;
}
eventHandler(RunnerStart(
'${build.name}: RBE ${shutdown ? 'shutdown' : 'startup'}',
bootstrapCommand,
DateTime.now(),
));
final ProcessRunnerResult bootstrapResult;
if (dryRun) {
processResult = _dryRunResult;
bootstrapResult = _dryRunResult;
} else {
final io.Process process = await processRunner.processManager.start(
command,
workingDirectory: engineSrcDir.path,
bootstrapResult = await processRunner.runProcess(
bootstrapCommand, failOk: true,
);
final List<int> stderrOutput = <int>[];
final Completer<void> stdoutComplete = Completer<void>();
final Completer<void> stderrComplete = Completer<void>();

process.stdout
.transform<String>(const Utf8Decoder())
.transform(const LineSplitter())
.listen(
(String line) {
_ninjaProgress(eventHandler, command, line);
},
onDone: () async => stdoutComplete.complete(),
);
}
eventHandler(RunnerResult(
'${build.name}: RBE ${shutdown ? 'shutdown' : 'startup'}',
bootstrapCommand,
DateTime.now(),
bootstrapResult,
));
return bootstrapResult.exitCode == 0;
}

process.stderr.listen(
stderrOutput.addAll,
onDone: () async => stderrComplete.complete(),
Future<bool> _runNinja(RunnerEventHandler eventHandler) async {
if (_isRbe) {
if (!await _bootstrapRbe(eventHandler)) {
return false;
}
}
bool success = false;
try {
final String ninjaPath = p.join(
engineSrcDir.path, 'flutter', 'third_party', 'ninja', 'ninja',
);

await Future.wait<void>(<Future<void>>[
stdoutComplete.future, stderrComplete.future,
]);
final int exitCode = await process.exitCode;

processResult = ProcessRunnerResult(
exitCode,
<int>[], // stdout.
stderrOutput, // stderr.
stderrOutput, // combined,
pid: process.pid, // pid,
final String outDir = p.join(
engineSrcDir.path, 'out', build.ninja.config,
);
}
final List<String> command = <String>[
ninjaPath,
'-C', outDir,
if (_isGomaOrRbe) ...<String>['-j', '200'],
...extraNinjaArgs,
...build.ninja.targets,
];
eventHandler(RunnerStart(
'${build.name}: ninja', command, DateTime.now()),
);
final ProcessRunnerResult processResult;
if (dryRun) {
processResult = _dryRunResult;
} else {
final io.Process process = await processRunner.processManager.start(
command,
workingDirectory: engineSrcDir.path,
);
final List<int> stderrOutput = <int>[];
final Completer<void> stdoutComplete = Completer<void>();
final Completer<void> stderrComplete = Completer<void>();

process.stdout
.transform<String>(const Utf8Decoder())
.transform(const LineSplitter())
.listen(
(String line) {
_ninjaProgress(eventHandler, command, line);
},
onDone: () async => stdoutComplete.complete(),
);

process.stderr.listen(
stderrOutput.addAll,
onDone: () async => stderrComplete.complete(),
);

final RunnerResult result = RunnerResult(
'${build.name}: ninja', command, DateTime.now(), processResult,
);
eventHandler(result);
return result.ok;
await Future.wait<void>(<Future<void>>[
stdoutComplete.future, stderrComplete.future,
]);
final int exitCode = await process.exitCode;

processResult = ProcessRunnerResult(
exitCode,
<int>[], // stdout.
stderrOutput, // stderr.
stderrOutput, // combined,
pid: process.pid, // pid,
);
}
eventHandler(RunnerResult(
'${build.name}: ninja', command, DateTime.now(), processResult,
));
success = processResult.exitCode == 0;
} finally {
if (_isRbe) {
// Ignore failures to shutdown.
await _bootstrapRbe(eventHandler, shutdown: true);
}
}
return success;
}

// Parse lines of the form '[6232/6269] LINK ./accessibility_unittests'.
Expand Down Expand Up @@ -389,10 +478,8 @@ final class GlobalBuildRunner extends Runner {
));
}

late final bool _isGoma = build.gn.contains('--goma') ||
extraGnArgs.contains('--goma');
late final bool _isRbe = build.gn.contains('--rbe') ||
extraGnArgs.contains('--rbe');
late final bool _isGoma = _mergedGnArgs.contains('--goma');
late final bool _isRbe = _mergedGnArgs.contains('--rbe');
late final bool _isGomaOrRbe = _isGoma || _isRbe;

Future<bool> _runGenerators(RunnerEventHandler eventHandler) async {
Expand Down
Loading