Reduce noise on the memory benchmarks (flutter#19630)

- Check memory usage in release builds, not profile. - Use multiple runs and average the results.
shenguangyao · Aug 2, 2018 · 8eb5cb7 · 8eb5cb7
1 parent 1f7082d
commit 8eb5cb7
Show file tree

Hide file tree

Showing 21 changed files with 562 additions and 218 deletions.
diff --git a/dev/benchmarks/complex_layout/test_memory/scroll_perf.dart b/dev/benchmarks/complex_layout/test_memory/scroll_perf.dart
@@ -0,0 +1,72 @@
+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+
+import 'package:complex_layout/main.dart';
+import 'package:flutter/material.dart';
+import 'package:flutter/scheduler.dart';
+import 'package:flutter_test/flutter_test.dart';
+
+/// The speed, in pixels per second, that the drag gestures should end with.
+const double speed = 1500.0;
+
+/// The number of down drags and the number of up drags. The total number of
+/// gestures is twice this number.
+const int maxIterations = 4;
+
+/// The time that is allowed between gestures for the fling effect to settle.
+const Duration pauses = Duration(milliseconds: 500);
+
+Future<void> main() async {
+  final Completer<void> ready = new Completer<void>();
+  runApp(new GestureDetector(
+    onTap: () {
+      debugPrint('Received tap.');
+      ready.complete();
+    },
+    behavior: HitTestBehavior.opaque,
+    child: new IgnorePointer(
+      ignoring: true,
+      child: new ComplexLayoutApp(),
+    ),
+  ));
+  await SchedulerBinding.instance.endOfFrame;
+
+  /// Wait 50ms to allow the GPU thread to actually put up the frame. (The
+  /// endOfFrame future ends when we send the data to the engine, before the GPU
+  /// thread has had a chance to rasterize, etc.)
+  await new Future<Null>.delayed(const Duration(milliseconds: 50));
+  debugPrint('==== MEMORY BENCHMARK ==== READY ====');
+
+  await ready.future; // waits for tap sent by devicelab task
+  debugPrint('Continuing...');
+
+  // remove onTap handler, enable pointer events for app
+  runApp(new GestureDetector(
+    child: new IgnorePointer(
+      ignoring: false,
+      child: new ComplexLayoutApp(),
+    ),
+  ));
+  await SchedulerBinding.instance.endOfFrame;
+
+  final WidgetController controller = new LiveWidgetController(WidgetsBinding.instance);
+
+  // Scroll down
+  for (int iteration = 0; iteration < maxIterations; iteration += 1) {
+    debugPrint('Scroll down... $iteration/$maxIterations');
+    await controller.fling(find.byType(ListView), const Offset(0.0, -700.0), speed);
+    await new Future<Null>.delayed(pauses);
+  }
+
+  // Scroll up
+  for (int iteration = 0; iteration < maxIterations; iteration += 1) {
+    debugPrint('Scroll up... $iteration/$maxIterations');
+    await controller.fling(find.byType(ListView), const Offset(0.0, 300.0), speed);
+    await new Future<Null>.delayed(pauses);
+  }
+
+  debugPrint('==== MEMORY BENCHMARK ==== DONE ====');
+}
diff --git a/dev/devicelab/bin/tasks/complex_layout_scroll_perf__memory.dart b/dev/devicelab/bin/tasks/complex_layout_scroll_perf__memory.dart
@@ -4,11 +4,16 @@
 
 import 'dart:async';
 
-import 'package:flutter_devicelab/tasks/perf_tests.dart';
 import 'package:flutter_devicelab/framework/adb.dart';
 import 'package:flutter_devicelab/framework/framework.dart';
+import 'package:flutter_devicelab/framework/utils.dart';
+import 'package:flutter_devicelab/tasks/perf_tests.dart';
 
 Future<Null> main() async {
   deviceOperatingSystem = DeviceOperatingSystem.android;
-  await task(createComplexLayoutScrollMemoryTest());
+  await task(new MemoryTest(
+    '${flutterDirectory.path}/dev/benchmarks/complex_layout',
+    'test_memory/scroll_perf.dart',
+    'com.yourcompany.complexLayout',
+  ).run);
 }
diff --git a/dev/devicelab/bin/tasks/flutter_gallery__back_button_memory.dart b/dev/devicelab/bin/tasks/flutter_gallery__back_button_memory.dart
@@ -2,11 +2,57 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+/// Measure application memory usage after pausing and resuming the app
+/// with the Android back button.
+
 import 'dart:async';
 
-import 'package:flutter_devicelab/tasks/perf_tests.dart';
+import 'package:flutter_devicelab/framework/adb.dart';
 import 'package:flutter_devicelab/framework/framework.dart';
+import 'package:flutter_devicelab/framework/utils.dart';
+import 'package:flutter_devicelab/tasks/perf_tests.dart';
+
+const String packageName = 'io.flutter.demo.gallery';
+const String activityName = 'io.flutter.demo.gallery.MainActivity';
+
+class BackButtonMemoryTest extends MemoryTest {
+  BackButtonMemoryTest() : super('${flutterDirectory.path}/examples/flutter_gallery', 'test_memory/back_button.dart', packageName);
+
+  @override
+  AndroidDevice get device => super.device;
+
+  /// Perform a series of back button suspend and resume cycles.
+  @override
+  Future<void> useMemory() async {
+    await launchApp();
+    await recordStart();
+    for (int iteration = 0; iteration < 10; iteration += 1) {
+      print('back/forward iteration $iteration');
+
+      // Push back button, wait for it to be seen by the Flutter app.
+      prepareForNextMessage('AppLifecycleState.paused');
+      await device.shellExec('input', <String>['keyevent', 'KEYCODE_BACK']);
+      await receivedNextMessage;
+
+      // Give Android time to settle (e.g. run GCs) after closing the app.
+      await new Future<Null>.delayed(const Duration(milliseconds: 100));
+
+      // Relaunch the app, wait for it to launch.
+      prepareForNextMessage('READY');
+      final String output = await device.shellEval('am', <String>['start', '-n', '$packageName/$activityName']);
+      print('adb shell am start: $output');
+      if (output.contains('Error'))
+        fail('unable to launch activity');
+      await receivedNextMessage;
+
+      // Wait for the Flutter app to settle (e.g. run GCs).
+      await new Future<Null>.delayed(const Duration(milliseconds: 100));
+    }
+    await recordEnd();
+  }
+}
 
 Future<Null> main() async {
-  await task(createGalleryBackButtonMemoryTest());
+  deviceOperatingSystem = DeviceOperatingSystem.android;
+  await task(new BackButtonMemoryTest().run);
 }
diff --git a/dev/devicelab/bin/tasks/flutter_gallery__memory_nav.dart b/dev/devicelab/bin/tasks/flutter_gallery__memory_nav.dart
@@ -4,9 +4,14 @@
 
 import 'dart:async';
 
-import 'package:flutter_devicelab/tasks/perf_tests.dart';
 import 'package:flutter_devicelab/framework/framework.dart';
+import 'package:flutter_devicelab/framework/utils.dart';
+import 'package:flutter_devicelab/tasks/perf_tests.dart';
 
 Future<Null> main() async {
-  await task(createGalleryNavigationMemoryTest());
+  await task(new MemoryTest(
+    '${flutterDirectory.path}/examples/flutter_gallery',
+    'test_memory/memory_nav.dart',
+    'io.flutter.demo.gallery',
+  ).run);
 }
diff --git a/dev/devicelab/bin/tasks/hello_world__memory.dart b/dev/devicelab/bin/tasks/hello_world__memory.dart
@@ -4,9 +4,36 @@
 
 import 'dart:async';
 
-import 'package:flutter_devicelab/tasks/perf_tests.dart';
 import 'package:flutter_devicelab/framework/framework.dart';
+import 'package:flutter_devicelab/framework/utils.dart';
+import 'package:flutter_devicelab/tasks/perf_tests.dart';
+
+class HelloWorldMemoryTest extends MemoryTest {
+  HelloWorldMemoryTest() : super(
+    '${flutterDirectory.path}/examples/hello_world',
+    'lib/main.dart',
+    'io.flutter.examples.hello_world',
+  );
+
+  /// Launch an app with no instrumentation and measure its memory usage after
+  /// 1.5s and 3.0s.
+  @override
+  Future<void> useMemory() async {
+    print('launching $project$test on device...');
+    await flutter('run', options: <String>[
+      '--verbose',
+      '--release',
+      '--no-resident',
+      '-d', device.deviceId,
+      test,
+    ]);
+    await new Future<Null>.delayed(const Duration(milliseconds: 1500));
+    await recordStart();
+    await new Future<Null>.delayed(const Duration(milliseconds: 3000));
+    await recordEnd();
+  }
+}
 
 Future<Null> main() async {
-  await task(createHelloWorldMemoryTest());
+  await task(new HelloWorldMemoryTest().run);
 }
diff --git a/dev/devicelab/lib/framework/adb.dart b/dev/devicelab/lib/framework/adb.dart
@@ -83,9 +83,18 @@ abstract class Device {
   /// Assumes the device doesn't have a secure unlock pattern.
   Future<Null> unlock();
 
+  /// Emulate a tap on the touch screen.
+  Future<Null> tap(int x, int y);
+
   /// Read memory statistics for a process.
   Future<Map<String, dynamic>> getMemoryStats(String packageName);
 
+  /// Stream the system log from the device.
+  ///
+  /// Flutter applications' `print` statements end up in this log
+  /// with some prefix.
+  Stream<String> get logcat;
+
   /// Stop a process.
   Future<Null> stop(String packageName);
 }
@@ -237,6 +246,11 @@ class AndroidDevice implements Device {
     await shellExec('input', const <String>['keyevent', '82']);
   }
 
+  @override
+  Future<Null> tap(int x, int y) async {
+    await shellExec('input', <String>['tap', '$x', '$y']);
+  }
+
   /// Retrieves device's wakefulness state.
   ///
   /// See: https://android.googlesource.com/platform/frameworks/base/+/master/core/java/android/os/PowerManagerInternal.java
@@ -271,6 +285,63 @@ class AndroidDevice implements Device {
     };
   }
 
+  @override
+  Stream<String> get logcat {
+    final Completer<void> stdoutDone = new Completer<void>();
+    final Completer<void> stderrDone = new Completer<void>();
+    final Completer<void> processDone = new Completer<void>();
+    final Completer<void> abort = new Completer<void>();
+    bool aborted = false;
+    StreamController<String> stream;
+    stream = new StreamController<String>(
+      onListen: () async {
+        await adb(<String>['logcat', '--clear']);
+        final Process process = await startProcess(adbPath, <String>['-s', deviceId, 'logcat']);
+        process.stdout
+          .transform(utf8.decoder)
+          .transform(const LineSplitter())
+          .listen((String line) {
+            print('adb logcat: $line');
+            stream.sink.add(line);
+          }, onDone: () { stdoutDone.complete(); });
+        process.stderr
+          .transform(utf8.decoder)
+          .transform(const LineSplitter())
+          .listen((String line) {
+            print('adb logcat stderr: $line');
+          }, onDone: () { stderrDone.complete(); });
+        process.exitCode.then((int exitCode) {
+          print('adb logcat process terminated with exit code $exitCode');
+          if (!aborted) {
+            stream.addError(BuildFailedError('adb logcat failed with exit code $exitCode.'));
+            processDone.complete();
+          }
+        });
+        await Future.any<dynamic>(<Future<dynamic>>[
+          Future.wait<void>(<Future<void>>[
+            stdoutDone.future,
+            stderrDone.future,
+            processDone.future,
+          ]),
+          abort.future,
+        ]);
+        aborted = true;
+        print('terminating adb logcat');
+        process.kill();
+        print('closing logcat stream');
+        await stream.close();
+      },
+      onCancel: () {
+        if (!aborted) {
+          print('adb logcat aborted');
+          aborted = true;
+          abort.complete();
+        }
+      },
+    );
+    return stream.stream;
+  }
+
   @override
   Future<Null> stop(String packageName) async {
     return shellExec('am', <String>['force-stop', packageName]);
@@ -371,11 +442,21 @@ class IosDevice implements Device {
   @override
   Future<Null> unlock() async {}
 
+  @override
+  Future<Null> tap(int x, int y) async {
+    throw 'Not implemented';
+  }
+
   @override
   Future<Map<String, dynamic>> getMemoryStats(String packageName) async {
     throw 'Not implemented';
   }
 
+  @override
+  Stream<String> get logcat {
+    throw 'Not implemented';
+  }
+
   @override
   Future<Null> stop(String packageName) async {}
 }

diff --git a/dev/devicelab/lib/framework/framework.dart b/dev/devicelab/lib/framework/framework.dart
@@ -81,12 +81,15 @@ class _TaskRunner {
   Future<TaskResult> run(Duration taskTimeout) async {
     try {
       _taskStarted = true;
+      print('Running task.');
       final TaskResult result = await _performTask().timeout(taskTimeout);
       _completer.complete(result);
       return result;
     } on TimeoutException catch (_) {
+      print('Task timed out after $taskTimeout.');
       return new TaskResult.failure('Task timed out after $taskTimeout');
     } finally {
+      print('Cleaning up after task...');
       await forceQuitRunningProcesses();
       _closeKeepAlivePort();
     }

diff --git a/dev/devicelab/lib/framework/utils.dart b/dev/devicelab/lib/framework/utils.dart
@@ -244,7 +244,7 @@ Future<Null> forceQuitRunningProcesses() async {
 
   // Whatever's left, kill it.
   for (ProcessInfo p in _runningProcesses) {
-    print('Force quitting process:\n$p');
+    print('Force-quitting process:\n$p');
     if (!p.process.kill()) {
       print('Failed to force quit process');
     }
@@ -528,8 +528,6 @@ int parseServicePort(String line, {
   // e.g. "An Observatory debugger and profiler on ... is available at: http://127.0.0.1:8100/"
   final RegExp pattern = new RegExp('$prefix(\\S+:(\\d+)/\\S*)\$', multiLine: multiLine);
   final Match match = pattern.firstMatch(line);
-  print(pattern);
-  print(match);
   return match == null ? null : int.parse(match.group(2));
 }