Skip to content

Commit 5a1301b

Browse files
CareFEgor
CareF
authored and
Egor
committed
[e2e] Add new e2e_driver for handling response data and performance watcher (flutter#2906)
1 parent 7a78626 commit 5a1301b

File tree

6 files changed

+318
-6
lines changed

6 files changed

+318
-6
lines changed

‎packages/e2e/CHANGELOG.md

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,8 @@
1+
## 0.6.3
2+
3+
* Add customizable `flutter_driver` adaptor.
4+
* Add utilities for tracking frame performance in an e2e test.
5+
16
## 0.6.2+1
27

38
* Fix incorrect test results when one test passes then another fails

‎packages/e2e/example/test_driver/example_e2e_test.dart

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,4 +2,4 @@ import 'dart:async';
22

33
import 'package:e2e/e2e_driver.dart' as e2e;
44

5-
Future<void> main() async => e2e.main();
5+
Future<void> main() async => e2e.e2eDriver();

‎packages/e2e/lib/e2e_driver.dart

Lines changed: 76 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,93 @@
1+
// Copyright 2014 The Flutter Authors. All rights reserved.
2+
// Use of this source code is governed by a BSD-style license that can be
3+
// found in the LICENSE file.
4+
15
import 'dart:async';
6+
import 'dart:convert';
27
import 'dart:io';
38

4-
import 'package:e2e/common.dart' as e2e;
59
import 'package:flutter_driver/flutter_driver.dart';
610

7-
Future<void> main() async {
11+
import 'package:e2e/common.dart' as e2e;
12+
import 'package:path/path.dart' as path;
13+
14+
/// This method remains for backword compatibility.
15+
Future<void> main() => e2eDriver();
16+
17+
/// Flutter Driver test output directory.
18+
///
19+
/// Tests should write any output files to this directory. Defaults to the path
20+
/// set in the FLUTTER_TEST_OUTPUTS_DIR environment variable, or `build` if
21+
/// unset.
22+
String testOutputsDirectory =
23+
Platform.environment['FLUTTER_TEST_OUTPUTS_DIR'] ?? 'build';
24+
25+
/// The callback type to handle [e2e.Response.data] after the test succcess.
26+
typedef ResponseDataCallback = FutureOr<void> Function(Map<String, dynamic>);
27+
28+
/// Writes a json-serializable json data to to
29+
/// [testOutputsDirectory]/`testOutputFilename.json`.
30+
///
31+
/// This is the default `responseDataCallback` in [e2eDriver].
32+
Future<void> writeResponseData(
33+
Map<String, dynamic> data, {
34+
String testOutputFilename = 'e2e_response_data',
35+
String destinationDirectory,
36+
}) async {
37+
assert(testOutputFilename != null);
38+
destinationDirectory ??= testOutputsDirectory;
39+
await fs.directory(destinationDirectory).create(recursive: true);
40+
final File file = fs.file(path.join(
41+
destinationDirectory,
42+
'$testOutputFilename.json',
43+
));
44+
final String resultString = _encodeJson(data, true);
45+
await file.writeAsString(resultString);
46+
}
47+
48+
/// Adaptor to run E2E test using `flutter drive`.
49+
///
50+
/// `timeout` controls the longest time waited before the test ends.
51+
/// It is not necessarily the execution time for the test app: the test may
52+
/// finish sooner than the `timeout`.
53+
///
54+
/// `responseDataCallback` is the handler for processing [e2e.Response.data].
55+
/// The default value is `writeResponseData`.
56+
///
57+
/// To an E2E test `<test_name>.dart` using `flutter drive`, put a file named
58+
/// `<test_name>_test.dart` in the app's `test_driver` directory:
59+
///
60+
/// ```dart
61+
/// import 'dart:async';
62+
///
63+
/// import 'package:e2e/e2e_driver.dart' as e2e;
64+
///
65+
/// Future<void> main() async => e2e.e2eDriver();
66+
///
67+
/// ```
68+
Future<void> e2eDriver({
69+
Duration timeout = const Duration(minutes: 1),
70+
ResponseDataCallback responseDataCallback = writeResponseData,
71+
}) async {
872
final FlutterDriver driver = await FlutterDriver.connect();
9-
final String jsonResult =
10-
await driver.requestData(null, timeout: const Duration(minutes: 1));
73+
final String jsonResult = await driver.requestData(null, timeout: timeout);
1174
final e2e.Response response = e2e.Response.fromJson(jsonResult);
1275
await driver.close();
1376

1477
if (response.allTestsPassed) {
1578
print('All tests passed.');
79+
if (responseDataCallback != null) {
80+
await responseDataCallback(response.data);
81+
}
1682
exit(0);
1783
} else {
1884
print('Failure Details:\n${response.formattedFailureDetails}');
1985
exit(1);
2086
}
2187
}
88+
89+
const JsonEncoder _prettyEncoder = JsonEncoder.withIndent(' ');
90+
91+
String _encodeJson(Map<String, dynamic> jsonObject, bool pretty) {
92+
return pretty ? _prettyEncoder.convert(jsonObject) : json.encode(jsonObject);
93+
}

‎packages/e2e/lib/e2e_perf.dart

Lines changed: 199 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,199 @@
1+
// Copyright 2014 The Flutter Authors. All rights reserved.
2+
// Use of this source code is governed by a BSD-style license that can be
3+
// found in the LICENSE file.
4+
5+
import 'dart:async';
6+
import 'dart:ui';
7+
8+
import 'package:flutter/scheduler.dart';
9+
import 'package:flutter_test/flutter_test.dart';
10+
import 'package:flutter/widgets.dart';
11+
12+
import 'package:e2e/e2e.dart';
13+
14+
/// The maximum amount of time considered safe to spend for a frame's build
15+
/// phase. Anything past that is in the danger of missing the frame as 60FPS.
16+
///
17+
/// Changing this doesn't re-evaluate existing summary.
18+
Duration kBuildBudget = const Duration(milliseconds: 16);
19+
// TODO(CareF): Automatically calculate the refresh budget (#61958)
20+
21+
bool _firstRun = true;
22+
23+
/// The warning message to show when a benchmark is performed with assert on.
24+
/// TODO(CareF) remove this and update pubspect when flutter/flutter#61509 is
25+
/// in released version.
26+
const String kDebugWarning = '''
27+
┏╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍┓
28+
┇ ⚠ THIS BENCHMARK IS BEING RUN IN DEBUG MODE ⚠ ┇
29+
┡╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍┦
30+
│ │
31+
│ Numbers obtained from a benchmark while asserts are │
32+
│ enabled will not accurately reflect the performance │
33+
│ that will be experienced by end users using release ╎
34+
│ builds. Benchmarks should be run using this command ╎
35+
│ line: "flutter run --profile test.dart" or ┊
36+
│ or "flutter drive --profile -t test.dart". ┊
37+
│ ┊
38+
└─────────────────────────────────────────────────╌┄┈ 🐢
39+
''';
40+
41+
/// watches the [FrameTiming] of `action` and report it to the e2e binding.
42+
Future<void> watchPerformance(
43+
E2EWidgetsFlutterBinding binding,
44+
Future<void> action(), {
45+
String reportKey = 'performance',
46+
}) async {
47+
assert(() {
48+
if (_firstRun) {
49+
debugPrint(kDebugWarning);
50+
_firstRun = false;
51+
}
52+
return true;
53+
}());
54+
final List<FrameTiming> frameTimings = <FrameTiming>[];
55+
final TimingsCallback watcher = frameTimings.addAll;
56+
binding.addTimingsCallback(watcher);
57+
await action();
58+
binding.removeTimingsCallback(watcher);
59+
final FrameTimingSummarizer frameTimes = FrameTimingSummarizer(frameTimings);
60+
binding.reportData = <String, dynamic>{reportKey: frameTimes.summary};
61+
}
62+
63+
/// This class and summarizes a list of [FrameTiming] for the performance
64+
/// metrics.
65+
class FrameTimingSummarizer {
66+
/// Summarize `data` to frame build time and frame rasterizer time statistics.
67+
///
68+
/// See [TimelineSummary.summaryJson] for detail.
69+
factory FrameTimingSummarizer(List<FrameTiming> data) {
70+
assert(data != null);
71+
assert(data.isNotEmpty);
72+
final List<Duration> frameBuildTime = List<Duration>.unmodifiable(
73+
data.map<Duration>((FrameTiming datum) => datum.buildDuration),
74+
);
75+
final List<Duration> frameBuildTimeSorted =
76+
List<Duration>.from(frameBuildTime)..sort();
77+
final List<Duration> frameRasterizerTime = List<Duration>.unmodifiable(
78+
data.map<Duration>((FrameTiming datum) => datum.rasterDuration),
79+
);
80+
final List<Duration> frameRasterizerTimeSorted =
81+
List<Duration>.from(frameRasterizerTime)..sort();
82+
final Duration Function(Duration, Duration) add =
83+
(Duration a, Duration b) => a + b;
84+
return FrameTimingSummarizer._(
85+
frameBuildTime: frameBuildTime,
86+
frameRasterizerTime: frameRasterizerTime,
87+
// This avarage calculation is microsecond precision, which is fine
88+
// because typical values of these times are milliseconds.
89+
averageFrameBuildTime: frameBuildTime.reduce(add) ~/ data.length,
90+
p90FrameBuildTime: _findPercentile(frameBuildTimeSorted, 0.90),
91+
p99FrameBuildTime: _findPercentile(frameBuildTimeSorted, 0.99),
92+
worstFrameBuildTime: frameBuildTimeSorted.last,
93+
missedFrameBuildBudget: _countExceed(frameBuildTimeSorted, kBuildBudget),
94+
averageFrameRasterizerTime:
95+
frameRasterizerTime.reduce(add) ~/ data.length,
96+
p90FrameRasterizerTime: _findPercentile(frameRasterizerTimeSorted, 0.90),
97+
p99FrameRasterizerTime: _findPercentile(frameRasterizerTimeSorted, 0.99),
98+
worstFrameRasterizerTime: frameRasterizerTimeSorted.last,
99+
missedFrameRasterizerBudget:
100+
_countExceed(frameRasterizerTimeSorted, kBuildBudget),
101+
);
102+
}
103+
104+
const FrameTimingSummarizer._({
105+
@required this.frameBuildTime,
106+
@required this.frameRasterizerTime,
107+
@required this.averageFrameBuildTime,
108+
@required this.p90FrameBuildTime,
109+
@required this.p99FrameBuildTime,
110+
@required this.worstFrameBuildTime,
111+
@required this.missedFrameBuildBudget,
112+
@required this.averageFrameRasterizerTime,
113+
@required this.p90FrameRasterizerTime,
114+
@required this.p99FrameRasterizerTime,
115+
@required this.worstFrameRasterizerTime,
116+
@required this.missedFrameRasterizerBudget,
117+
});
118+
119+
/// List of frame build time in microseconds
120+
final List<Duration> frameBuildTime;
121+
122+
/// List of frame rasterizer time in microseconds
123+
final List<Duration> frameRasterizerTime;
124+
125+
/// The average value of [frameBuildTime] in milliseconds.
126+
final Duration averageFrameBuildTime;
127+
128+
/// The 90-th percentile value of [frameBuildTime] in milliseconds
129+
final Duration p90FrameBuildTime;
130+
131+
/// The 99-th percentile value of [frameBuildTime] in milliseconds
132+
final Duration p99FrameBuildTime;
133+
134+
/// The largest value of [frameBuildTime] in milliseconds
135+
final Duration worstFrameBuildTime;
136+
137+
/// Number of items in [frameBuildTime] that's greater than [kBuildBudget]
138+
final int missedFrameBuildBudget;
139+
140+
/// The average value of [frameRasterizerTime] in milliseconds.
141+
final Duration averageFrameRasterizerTime;
142+
143+
/// The 90-th percentile value of [frameRasterizerTime] in milliseconds.
144+
final Duration p90FrameRasterizerTime;
145+
146+
/// The 99-th percentile value of [frameRasterizerTime] in milliseconds.
147+
final Duration p99FrameRasterizerTime;
148+
149+
/// The largest value of [frameRasterizerTime] in milliseconds.
150+
final Duration worstFrameRasterizerTime;
151+
152+
/// Number of items in [frameRasterizerTime] that's greater than [kBuildBudget]
153+
final int missedFrameRasterizerBudget;
154+
155+
/// Convert the summary result to a json object.
156+
///
157+
/// See [TimelineSummary.summaryJson] for detail.
158+
Map<String, dynamic> get summary => <String, dynamic>{
159+
'average_frame_build_time_millis':
160+
averageFrameBuildTime.inMicroseconds / 1E3,
161+
'90th_percentile_frame_build_time_millis':
162+
p90FrameBuildTime.inMicroseconds / 1E3,
163+
'99th_percentile_frame_build_time_millis':
164+
p99FrameBuildTime.inMicroseconds / 1E3,
165+
'worst_frame_build_time_millis':
166+
worstFrameBuildTime.inMicroseconds / 1E3,
167+
'missed_frame_build_budget_count': missedFrameBuildBudget,
168+
'average_frame_rasterizer_time_millis':
169+
averageFrameRasterizerTime.inMicroseconds / 1E3,
170+
'90th_percentile_frame_rasterizer_time_millis':
171+
p90FrameRasterizerTime.inMicroseconds / 1E3,
172+
'99th_percentile_frame_rasterizer_time_millis':
173+
p99FrameRasterizerTime.inMicroseconds / 1E3,
174+
'worst_frame_rasterizer_time_millis':
175+
worstFrameRasterizerTime.inMicroseconds / 1E3,
176+
'missed_frame_rasterizer_budget_count': missedFrameRasterizerBudget,
177+
'frame_count': frameBuildTime.length,
178+
'frame_build_times': frameBuildTime
179+
.map<int>((Duration datum) => datum.inMicroseconds)
180+
.toList(),
181+
'frame_rasterizer_times': frameRasterizerTime
182+
.map<int>((Duration datum) => datum.inMicroseconds)
183+
.toList(),
184+
};
185+
}
186+
187+
// The following helper functions require data sorted
188+
189+
// return the 100*p-th percentile of the data
190+
T _findPercentile<T>(List<T> data, double p) {
191+
assert(p >= 0 && p <= 1);
192+
return data[((data.length - 1) * p).round()];
193+
}
194+
195+
// return the number of items in data that > threshold
196+
int _countExceed<T extends Comparable<T>>(List<T> data, T threshold) {
197+
return data.length -
198+
data.indexWhere((T datum) => datum.compareTo(threshold) > 0);
199+
}

‎packages/e2e/pubspec.yaml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
name: e2e
22
description: Runs tests that use the flutter_test API as integration tests.
3-
version: 0.6.2+1
3+
version: 0.6.3
44
homepage: https://github.com/flutter/plugins/tree/master/packages/e2e
55

66
environment:
@@ -14,6 +14,7 @@ dependencies:
1414
sdk: flutter
1515
flutter_test:
1616
sdk: flutter
17+
path: ^1.6.4
1718

1819
dev_dependencies:
1920
pedantic: ^1.8.0

‎packages/e2e/test/frame_timing_summarizer_test.dart

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
import 'dart:ui';
2+
3+
import 'package:flutter_test/flutter_test.dart';
4+
5+
import 'package:e2e/e2e_perf.dart';
6+
7+
void main() {
8+
test('Test FrameTimingSummarizer', () {
9+
List<int> buildTimes = <int>[
10+
for (int i = 1; i <= 100; i += 1) 1000 * i,
11+
];
12+
buildTimes = buildTimes.reversed.toList();
13+
List<int> rasterTimes = <int>[
14+
for (int i = 1; i <= 100; i += 1) 1000 * i + 1000,
15+
];
16+
rasterTimes = rasterTimes.reversed.toList();
17+
List<FrameTiming> inputData = <FrameTiming>[
18+
for (int i = 0; i < 100; i += 1)
19+
FrameTiming(<int>[0, buildTimes[i], 500, rasterTimes[i]]),
20+
];
21+
FrameTimingSummarizer summary = FrameTimingSummarizer(inputData);
22+
expect(summary.averageFrameBuildTime.inMicroseconds, 50500);
23+
expect(summary.p90FrameBuildTime.inMicroseconds, 90000);
24+
expect(summary.p99FrameBuildTime.inMicroseconds, 99000);
25+
expect(summary.worstFrameBuildTime.inMicroseconds, 100000);
26+
expect(summary.missedFrameBuildBudget, 84);
27+
28+
expect(summary.averageFrameRasterizerTime.inMicroseconds, 51000);
29+
expect(summary.p90FrameRasterizerTime.inMicroseconds, 90500);
30+
expect(summary.p99FrameRasterizerTime.inMicroseconds, 99500);
31+
expect(summary.worstFrameRasterizerTime.inMicroseconds, 100500);
32+
expect(summary.missedFrameRasterizerBudget, 85);
33+
expect(summary.frameBuildTime.length, 100);
34+
});
35+
}

0 commit comments

Comments
 (0)