Skip to content

Commit 8d0db80

Browse files
kenzieschmollkenzieschmoll
authored
Add initial framework for integration testing. (#4960)
* Add initial framework for integration testing. * remove this * fix async * review comments
1 parent be4031e commit 8d0db80

File tree

5 files changed

+368
-0
lines changed

5 files changed

+368
-0
lines changed

‎packages/devtools_app/integration_test/test_infra/io_utils.dart‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
// Copyright 2022 The Chromium Authors. All rights reserved.
2+
// Use of this source code is governed by a BSD-style license that can be
3+
// found in the LICENSE file.
4+
5+
import 'dart:async';
6+
import 'dart:convert';
7+
import 'dart:io';
8+
9+
Stream<String> transformToLines(Stream<List<int>> byteStream) {
10+
return byteStream
11+
.transform<String>(utf8.decoder)
12+
.transform<String>(const LineSplitter());
13+
}
14+
15+
mixin IOMixin {
16+
final stdoutController = StreamController<String>.broadcast();
17+
18+
final stderrController = StreamController<String>.broadcast();
19+
20+
final streamSubscriptions = <StreamSubscription<String>>[];
21+
22+
void listenToProcessOutput(
23+
Process process, {
24+
void Function(String) printCallback = _defaultPrintCallback,
25+
}) {
26+
streamSubscriptions.addAll([
27+
transformToLines(process.stdout)
28+
.listen((String line) => stdoutController.add(line)),
29+
transformToLines(process.stderr)
30+
.listen((String line) => stderrController.add(line)),
31+
32+
// This is just debug printing to aid running/debugging tests locally.
33+
stdoutController.stream.listen(printCallback),
34+
stderrController.stream.listen(printCallback),
35+
]);
36+
}
37+
38+
Future<void> cancelAllStreamSubscriptions() async {
39+
await Future.wait(streamSubscriptions.map((s) => s.cancel()));
40+
streamSubscriptions.clear();
41+
}
42+
43+
static void _defaultPrintCallback(String line) {
44+
print(line);
45+
}
46+
}

‎packages/devtools_app/integration_test/test_infra/test_app_driver.dart‎

Lines changed: 297 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,297 @@
1+
// Copyright 2022 The Chromium Authors. All rights reserved.
2+
// Use of this source code is governed by a BSD-style license that can be
3+
// found in the LICENSE file.
4+
5+
import 'dart:async';
6+
import 'dart:convert';
7+
import 'dart:io';
8+
9+
import 'io_utils.dart';
10+
11+
// Set this to true for debugging to get JSON written to stdout.
12+
const bool _printDebugOutputToStdOut = false;
13+
14+
class TestFlutterApp extends _TestApp {
15+
TestFlutterApp({String appPath = 'test/test_infra/fixtures/flutter_app'})
16+
: super(appPath);
17+
18+
Directory get workingDirectory => Directory(testAppPath);
19+
20+
@override
21+
Future<void> startProcess() async {
22+
runProcess = await Process.start(
23+
'flutter',
24+
[
25+
'run',
26+
'--machine',
27+
'-d',
28+
'flutter-tester',
29+
],
30+
);
31+
}
32+
}
33+
34+
// TODO(kenz): implement for running integration tests against a Dart CLI app.
35+
class TestDartCliApp {}
36+
37+
abstract class _TestApp with IOMixin {
38+
_TestApp(this.testAppPath);
39+
40+
static const _appStartTimeout = Duration(seconds: 120);
41+
42+
static const _defaultTimeout = Duration(seconds: 40);
43+
44+
static const _quitTimeout = Duration(seconds: 10);
45+
46+
/// The path relative to the 'devtools_app' directory where the test app
47+
/// lives.
48+
///
49+
/// This will either be a file path or a directory path depending on the type
50+
/// of app.
51+
final String testAppPath;
52+
53+
late Process? runProcess;
54+
55+
late int runProcessId;
56+
57+
final _allMessages = StreamController<String>.broadcast();
58+
59+
Uri get vmServiceUri => _vmServiceWsUri;
60+
late Uri _vmServiceWsUri;
61+
62+
Future<void> startProcess();
63+
64+
Future<void> start() async {
65+
await startProcess();
66+
assert(
67+
runProcess != null,
68+
'\'runProcess\' cannot be null. Assign \'runProcess\' inside the '
69+
'\'startProcess\' method.',
70+
);
71+
72+
// This class doesn't use the result of the future. It's made available
73+
// via a getter for external uses.
74+
unawaited(
75+
runProcess!.exitCode.then((int code) {
76+
_debugPrint('Process exited ($code)');
77+
}),
78+
);
79+
80+
listenToProcessOutput(runProcess!, printCallback: _debugPrint);
81+
82+
// Stash the PID so that we can terminate the VM more reliably than using
83+
// proc.kill() (because proc is a shell, because `flutter` is a shell
84+
// script).
85+
final connected =
86+
await waitFor(event: FlutterDaemonConstants.daemonConnected.key);
87+
runProcessId = (connected[FlutterDaemonConstants.params.key]!
88+
as Map<String, Object?>)[FlutterDaemonConstants.pid.key] as int;
89+
90+
// Set this up now, but we don't await it yet. We want to make sure we don't
91+
// miss it while waiting for debugPort below.
92+
final started = waitFor(
93+
event: FlutterDaemonConstants.appStarted.key,
94+
timeout: _appStartTimeout,
95+
);
96+
97+
final debugPort = await waitFor(
98+
event: FlutterDaemonConstants.appDebugPort.key,
99+
timeout: _appStartTimeout,
100+
);
101+
final wsUriString = (debugPort[FlutterDaemonConstants.params.key]!
102+
as Map<String, Object?>)[FlutterDaemonConstants.wsUri.key] as String;
103+
_vmServiceWsUri = Uri.parse(wsUriString);
104+
105+
// Map to WS URI.
106+
_vmServiceWsUri =
107+
convertToWebSocketUrl(serviceProtocolUrl: _vmServiceWsUri);
108+
109+
await started;
110+
}
111+
112+
Future<int> killGracefully() async {
113+
_debugPrint('Sending SIGTERM to $runProcessId..');
114+
Process.killPid(runProcessId);
115+
116+
final killFuture =
117+
runProcess!.exitCode.timeout(_quitTimeout, onTimeout: _killForcefully);
118+
unawaited(_killAndShutdown(killFuture));
119+
return killFuture;
120+
}
121+
122+
Future<int> _killForcefully() async {
123+
_debugPrint('Sending SIGKILL to $runProcessId..');
124+
Process.killPid(runProcessId, ProcessSignal.sigkill);
125+
126+
final killFuture = runProcess!.exitCode;
127+
unawaited(_killAndShutdown(killFuture));
128+
return killFuture;
129+
}
130+
131+
Future<void> _killAndShutdown(Future<int> killFuture) async {
132+
unawaited(
133+
killFuture.then((_) async {
134+
await cancelAllStreamSubscriptions();
135+
}),
136+
);
137+
}
138+
139+
Future<Map<String, Object?>> waitFor({
140+
String? event,
141+
int? id,
142+
Duration? timeout,
143+
bool ignoreAppStopEvent = false,
144+
}) async {
145+
final response = Completer<Map<String, Object?>>();
146+
late StreamSubscription<String> sub;
147+
sub = stdoutController.stream.listen(
148+
(String line) => _handleStdout(
149+
line,
150+
subscription: sub,
151+
response: response,
152+
event: event,
153+
id: id,
154+
ignoreAppStopEvent: ignoreAppStopEvent,
155+
),
156+
);
157+
158+
return _timeoutWithMessages<Map<String, Object?>>(
159+
() => response.future,
160+
timeout: timeout,
161+
message: event != null
162+
? 'Did not receive expected $event event.'
163+
: 'Did not receive response to request "$id".',
164+
).whenComplete(() => sub.cancel());
165+
}
166+
167+
void _handleStdout(
168+
String line, {
169+
required StreamSubscription<String> subscription,
170+
required Completer<Map<String, Object?>> response,
171+
required String? event,
172+
required int? id,
173+
bool ignoreAppStopEvent = false,
174+
}) async {
175+
final json = _parseFlutterResponse(line);
176+
if (json == null) {
177+
return;
178+
} else if ((event != null &&
179+
json[FlutterDaemonConstants.event.key] == event) ||
180+
(id != null && json[FlutterDaemonConstants.id.key] == id)) {
181+
await subscription.cancel();
182+
response.complete(json);
183+
} else if (!ignoreAppStopEvent &&
184+
json[FlutterDaemonConstants.event.key] ==
185+
FlutterDaemonConstants.appStop.key) {
186+
await subscription.cancel();
187+
final error = StringBuffer();
188+
error.write('Received app.stop event while waiting for ');
189+
error.write(
190+
'${event != null ? '$event event' : 'response to request $id.'}.\n\n',
191+
);
192+
final errorFromJson = (json[FlutterDaemonConstants.params.key]
193+
as Map<String, Object?>?)?[FlutterDaemonConstants.error.key];
194+
if (errorFromJson != null) {
195+
error.write('$errorFromJson\n\n');
196+
}
197+
final traceFromJson = (json[FlutterDaemonConstants.params.key]
198+
as Map<String, Object?>?)?[FlutterDaemonConstants.trace.key];
199+
if (traceFromJson != null) {
200+
error.write('$traceFromJson\n\n');
201+
}
202+
response.completeError(error.toString());
203+
}
204+
}
205+
206+
Future<T> _timeoutWithMessages<T>(
207+
Future<T> Function() f, {
208+
Duration? timeout,
209+
String? message,
210+
}) {
211+
// Capture output to a buffer so if we don't get the response we want we can show
212+
// the output that did arrive in the timeout error.
213+
final messages = StringBuffer();
214+
final start = DateTime.now();
215+
void logMessage(String m) {
216+
final int ms = DateTime.now().difference(start).inMilliseconds;
217+
messages.writeln('[+ ${ms.toString().padLeft(5)}] $m');
218+
}
219+
220+
final sub = _allMessages.stream.listen(logMessage);
221+
222+
return f().timeout(
223+
timeout ?? _defaultTimeout,
224+
onTimeout: () {
225+
logMessage('<timed out>');
226+
throw '$message';
227+
},
228+
).catchError((error) {
229+
throw '$error\nReceived:\n${messages.toString()}';
230+
}).whenComplete(() => sub.cancel());
231+
}
232+
233+
Map<String, Object?>? _parseFlutterResponse(String line) {
234+
if (line.startsWith('[') && line.endsWith(']')) {
235+
try {
236+
final Map<String, Object?>? resp = json.decode(line)[0];
237+
return resp;
238+
} catch (e) {
239+
// Not valid JSON, so likely some other output that was surrounded by [brackets]
240+
return null;
241+
}
242+
}
243+
return null;
244+
}
245+
246+
String _debugPrint(String msg) {
247+
const maxLength = 500;
248+
final truncatedMsg =
249+
msg.length > maxLength ? msg.substring(0, maxLength) + '...' : msg;
250+
_allMessages.add(truncatedMsg);
251+
if (_printDebugOutputToStdOut) {
252+
print('$truncatedMsg');
253+
}
254+
return msg;
255+
}
256+
}
257+
258+
/// Map the URI to a WebSocket URI for the VM service protocol.
259+
///
260+
/// If the URI is already a VM Service WebSocket URI it will not be modified.
261+
Uri convertToWebSocketUrl({required Uri serviceProtocolUrl}) {
262+
final isSecure = serviceProtocolUrl.isScheme('wss') ||
263+
serviceProtocolUrl.isScheme('https');
264+
final scheme = isSecure ? 'wss' : 'ws';
265+
266+
final path = serviceProtocolUrl.path.endsWith('/ws')
267+
? serviceProtocolUrl.path
268+
: (serviceProtocolUrl.path.endsWith('/')
269+
? '${serviceProtocolUrl.path}ws'
270+
: '${serviceProtocolUrl.path}/ws');
271+
272+
return serviceProtocolUrl.replace(scheme: scheme, path: path);
273+
}
274+
275+
// TODO(kenz): consider moving these constants to devtools_shared if they are
276+
// used outside of these integration tests. Optionally, we could consider making
277+
// these constants where the flutter daemon is defined in flutter tools.
278+
enum FlutterDaemonConstants {
279+
event,
280+
error,
281+
id,
282+
params,
283+
trace,
284+
wsUri,
285+
pid,
286+
appStop(nameOverride: 'app.stop'),
287+
appStarted(nameOverride: 'app.started'),
288+
appDebugPort(nameOverride: 'app.debugPort'),
289+
daemonConnected(nameOverride: 'daemon.connected');
290+
291+
const FlutterDaemonConstants({String? nameOverride})
292+
: _nameOverride = nameOverride;
293+
294+
final String? _nameOverride;
295+
296+
String get key => _nameOverride ?? name;
297+
}

‎packages/devtools_app/pubspec.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,10 @@ dev_dependencies:
6767
devtools_test: 2.21.0-dev.0
6868
flutter_test:
6969
sdk: flutter
70+
flutter_driver:
71+
sdk: flutter
72+
integration_test:
73+
sdk: flutter
7074
mockito: ^5.1.0
7175
webkit_inspection_protocol: '>=0.5.0 <2.0.0'
7276
stager: ^0.1.0

‎packages/devtools_app/test/test_infra/flutter_test_driver.dart‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,10 @@ import 'package:vm_service/utils.dart';
1111
import 'package:vm_service/vm_service.dart';
1212
import 'package:vm_service/vm_service_io.dart';
1313

14+
// TODO(kenz): eventually delete this class in favor of
15+
// integration_test/test_infra/test_app_driver.dart once the tests that
16+
// depend on this class are moved over to be true integration tests.
17+
1418
/// This class was copied from
1519
/// flutter/packages/flutter_tools/test/integration/test_driver.dart. Its
1620
/// supporting classes were also copied from flutter/packages/flutter_tools.

‎packages/devtools_app/test_driver/integration_test.dart‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
// Copyright 2022 The Chromium Authors. All rights reserved.
2+
// Use of this source code is governed by a BSD-style license that can be
3+
// found in the LICENSE file.
4+
5+
import 'package:flutter_driver/flutter_driver.dart';
6+
import 'package:integration_test/integration_test_driver_extended.dart';
7+
8+
Future<void> main() async {
9+
final FlutterDriver driver = await FlutterDriver.connect();
10+
await integrationDriver(
11+
driver: driver,
12+
onScreenshot: (String screenshotName, List<int> screenshotBytes) async {
13+
// TODO(kenz): implement golden image testing.
14+
return true;
15+
},
16+
);
17+
}

0 commit comments

Comments
 (0)