Bug 1356681 - Expand headless mode support for linux. r=automatedtest…

…er,jrmuizel,kanru

Full Firefox on Linux can now be run with a --headless flag.
This includes seven parts:
1) Running all marionette tests in headless mode.
2) Prevents crashes where Firefox calls into GTK.
3) Adds a headless screen helper which supports changing the headless
screen size with the environment variables MOZ_HEADLESS_WIDTH and
MOZ_HEADLESS_HEIGHT.
4) Supports simulating moving a headless window.
5) Adds a stubbed out nsSound implementation.
6) Supports simulating size mode changes of headless windows.
7) Adds the --headless flag for Firefox.

dom/ipc/ContentChild.cpp

@@ -551,28 +551,32 @@ ContentChild::Init(MessageLoop* aIOLoop,
   // to use, and when starting under XWayland, it may choose to start with
   // the wayland backend instead of the x11 backend.
   // The DISPLAY environment variable is normally set by the parent process.
-  const char* display_name = DetectDisplay();
-  if (display_name) {
-    int argc = 3;
-    char option_name[] = "--display";
-    char* argv[] = {
-      // argv0 is unused because g_set_prgname() was called in
-      // XRE_InitChildProcess().
-      nullptr,
-      option_name,
-      const_cast<char*>(display_name),
-      nullptr
-    };
-    char** argvp = argv;
-    gtk_init(&argc, &argvp);
-  } else {
-    gtk_init(nullptr, nullptr);
+  if (!gfxPlatform::IsHeadless()) {
+    const char* display_name = DetectDisplay();
+    if (display_name) {
+      int argc = 3;
+      char option_name[] = "--display";
+      char* argv[] = {
+        // argv0 is unused because g_set_prgname() was called in
+        // XRE_InitChildProcess().
+        nullptr,
+        option_name,
+        const_cast<char*>(display_name),
+        nullptr
+      };
+      char** argvp = argv;
+      gtk_init(&argc, &argvp);
+    } else {
+      gtk_init(nullptr, nullptr);
+    }
   }
 #endif
 
 #ifdef MOZ_X11
-  // Do this after initializing GDK, or GDK will install its own handler.
-  XRE_InstallX11ErrorHandler();
+  if (!gfxPlatform::IsHeadless()) {
+    // Do this after initializing GDK, or GDK will install its own handler.
+    XRE_InstallX11ErrorHandler();
+  }
 #endif
 
   NS_ASSERTION(!sSingleton, "only one ContentChild per child");
@@ -603,10 +607,12 @@ ContentChild::Init(MessageLoop* aIOLoop,
   GetIPCChannel()->SendBuildID();
 
 #ifdef MOZ_X11
-  // Send the parent our X socket to act as a proxy reference for our X
-  // resources.
-  int xSocketFd = ConnectionNumber(DefaultXDisplay());
-  SendBackUpXResources(FileDescriptor(xSocketFd));
+  if (!gfxPlatform::IsHeadless()) {
+    // Send the parent our X socket to act as a proxy reference for our X
+    // resources.
+    int xSocketFd = ConnectionNumber(DefaultXDisplay());
+    SendBackUpXResources(FileDescriptor(xSocketFd));
+  }
 #endif
 
 #ifdef MOZ_CRASHREPORTER

dom/ipc/TabChild.cpp

@@ -2504,7 +2504,8 @@ TabChild::RecvSetDocShellIsActive(const bool& aIsActive,
     MOZ_ASSERT(mPuppetWidget);
     MOZ_ASSERT(mPuppetWidget->GetLayerManager());
     MOZ_ASSERT(mPuppetWidget->GetLayerManager()->GetBackendType() == LayersBackend::LAYERS_CLIENT
-            || mPuppetWidget->GetLayerManager()->GetBackendType() == LayersBackend::LAYERS_WR);
+            || mPuppetWidget->GetLayerManager()->GetBackendType() == LayersBackend::LAYERS_WR
+            || (gfxPlatform::IsHeadless() && mPuppetWidget->GetLayerManager()->GetBackendType() == LayersBackend::LAYERS_BASIC));
 
     // We send the current layer observer epoch to the compositor so that
     // TabParent knows whether a layer update notification corresponds to the
@@ -3021,7 +3022,8 @@ TabChild::DidComposite(uint64_t aTransactionId,
   MOZ_ASSERT(mPuppetWidget);
   MOZ_ASSERT(mPuppetWidget->GetLayerManager());
   MOZ_ASSERT(mPuppetWidget->GetLayerManager()->GetBackendType() == LayersBackend::LAYERS_CLIENT
-             || mPuppetWidget->GetLayerManager()->GetBackendType() == LayersBackend::LAYERS_WR);
+             || mPuppetWidget->GetLayerManager()->GetBackendType() == LayersBackend::LAYERS_WR
+             || (gfxPlatform::IsHeadless() && mPuppetWidget->GetLayerManager()->GetBackendType() == LayersBackend::LAYERS_BASIC));
 
   mPuppetWidget->GetLayerManager()->DidComposite(aTransactionId, aCompositeStart, aCompositeEnd);
 }
@@ -3058,7 +3060,8 @@ TabChild::ClearCachedResources()
   MOZ_ASSERT(mPuppetWidget);
   MOZ_ASSERT(mPuppetWidget->GetLayerManager());
   MOZ_ASSERT(mPuppetWidget->GetLayerManager()->GetBackendType() == LayersBackend::LAYERS_CLIENT
-             || mPuppetWidget->GetLayerManager()->GetBackendType() == LayersBackend::LAYERS_WR);
+             || mPuppetWidget->GetLayerManager()->GetBackendType() == LayersBackend::LAYERS_WR
+             || (gfxPlatform::IsHeadless() && mPuppetWidget->GetLayerManager()->GetBackendType() == LayersBackend::LAYERS_BASIC));
 
   mPuppetWidget->GetLayerManager()->ClearCachedResources();
 }
@@ -3069,7 +3072,8 @@ TabChild::InvalidateLayers()
   MOZ_ASSERT(mPuppetWidget);
   MOZ_ASSERT(mPuppetWidget->GetLayerManager());
   MOZ_ASSERT(mPuppetWidget->GetLayerManager()->GetBackendType() == LayersBackend::LAYERS_CLIENT
-             || mPuppetWidget->GetLayerManager()->GetBackendType() == LayersBackend::LAYERS_WR);
+             || mPuppetWidget->GetLayerManager()->GetBackendType() == LayersBackend::LAYERS_WR
+             || (gfxPlatform::IsHeadless() && mPuppetWidget->GetLayerManager()->GetBackendType() == LayersBackend::LAYERS_BASIC));
 
   RefPtr<LayerManager> lm = mPuppetWidget->GetLayerManager();
   FrameLayerBuilder::InvalidateAllLayers(lm);
@@ -3158,7 +3162,8 @@ TabChild::CompositorUpdated(const TextureFactoryIdentifier& aNewIdentifier,
                             uint64_t aDeviceResetSeqNo)
 {
   MOZ_ASSERT(mPuppetWidget->GetLayerManager()->GetBackendType() == LayersBackend::LAYERS_CLIENT
-             || mPuppetWidget->GetLayerManager()->GetBackendType() == LayersBackend::LAYERS_WR);
+             || mPuppetWidget->GetLayerManager()->GetBackendType() == LayersBackend::LAYERS_WR
+             || (gfxPlatform::IsHeadless() && mPuppetWidget->GetLayerManager()->GetBackendType() == LayersBackend::LAYERS_BASIC));
 
   RefPtr<LayerManager> lm = mPuppetWidget->GetLayerManager();
 

gfx/thebes/gfxPlatform.cpp

@@ -836,7 +836,13 @@ gfxPlatform::InitMoz2DLogging()
 /* static */ bool
 gfxPlatform::IsHeadless()
 {
-    return PR_GetEnv("MOZ_HEADLESS");
+    static bool initialized = false;
+    static bool headless = false;
+    if (!initialized) {
+      initialized = true;
+      headless = PR_GetEnv("MOZ_HEADLESS");
+    }
+    return headless;
 }
 
 static bool sLayersIPCIsUp = false;

taskcluster/ci/test/test-platforms.yml

@@ -35,6 +35,7 @@ linux64/debug:
     test-sets:
         - common-tests
         - web-platform-tests
+        - headless
 linux64/opt:
     build-platform: linux64/opt
     test-sets:
@@ -44,6 +45,7 @@ linux64/opt:
         - desktop-screenshot-capture
         - talos
         - awsy
+        - headless
 linux64-nightly/opt:
     build-platform: linux64-nightly/opt
     test-sets:
@@ -53,6 +55,7 @@ linux64-nightly/opt:
         - desktop-screenshot-capture
         - talos
         - awsy
+        - headless
 
 # TODO: use 'pgo' and 'asan' labels here, instead of -pgo/opt
 linux64-pgo/opt:
@@ -61,11 +64,13 @@ linux64-pgo/opt:
         - common-tests
         - web-platform-tests
         - talos
+        - headless
 
 linux64-asan/opt:
     build-platform: linux64-asan/opt
     test-sets:
         - common-tests
+        - headless
 
 # Stylo builds only run a subset of tests for the moment. So give them
 # their own test set.

taskcluster/ci/test/test-sets.yml

@@ -69,6 +69,9 @@ talos:
 awsy:
     - awsy
 
+headless:
+    - marionette-headless
+
 ##
 # Limited test sets for specific platforms
 

taskcluster/ci/test/tests.yml

@@ -418,6 +418,44 @@ marionette:
                             - marionette/prod_config.py
                             - remove_executables.py
 
+marionette-headless:
+    description: "Marionette headless unittest run"
+    suite: marionette
+    treeherder-symbol: tc(MnH)
+    max-run-time:
+        by-test-platform:
+            default: 5400
+    instance-size:
+        by-test-platform:
+            default: default
+    docker-image: {"in-tree": "desktop1604-test"}
+    tier:
+        by-test-platform:
+            default: default
+    chunks:
+        by-test-platform:
+            default: 1
+    e10s:
+        by-test-platform:
+            default: both
+    run-on-projects:
+        by-test-platform:
+            default: ['all']
+    mozharness:
+        by-test-platform:
+            default:
+                script: marionette.py
+                no-read-buildbot-config: true
+                config:
+                    by-test-platform:
+                        default:
+                            - marionette/prod_config.py
+                            - remove_executables.py
+                extra-options:
+                    by-test-platform:
+                        default:
+                            - --headless
+
 mochitest:
     description: "Mochitest plain run"
     suite:

testing/marionette/client/marionette_driver/geckoinstance.py

@@ -114,7 +114,7 @@ class GeckoInstance(object):
 
     def __init__(self, host=None, port=None, bin=None, profile=None, addons=None,
                  app_args=None, symbols_path=None, gecko_log=None, prefs=None,
-                 workspace=None, verbose=0):
+                 workspace=None, verbose=0, headless=False):
         self.runner_class = Runner
         self.app_args = app_args or []
         self.runner = None
@@ -140,6 +140,7 @@ def __init__(self, host=None, port=None, bin=None, profile=None, addons=None,
         self._gecko_log_option = gecko_log
         self._gecko_log = None
         self.verbose = verbose
+        self.headless = headless
 
     @property
     def gecko_log(self):
@@ -230,6 +231,10 @@ def _get_runner_args(self):
 
         env = os.environ.copy()
 
+        if self.headless:
+            env["MOZ_HEADLESS"] = "1"
+            env["DISPLAY"] = "77"  # Set a fake display.
+
         # environment variables needed for crashreporting
         # https://developer.mozilla.org/docs/Environment_variables_affecting_crash_reporting
         env.update({"MOZ_CRASHREPORTER": "1",

testing/marionette/harness/marionette_harness/runner/base.py

@@ -359,6 +359,11 @@ def dir_path(path):
                           dest='e10s',
                           default=True,
                           help='Disable e10s when running marionette tests.')
+        self.add_argument('--headless',
+                          action='store_true',
+                          dest='headless',
+                          default=False,
+                          help='Enable headless mode when running marionette tests.')
         self.add_argument('--tag',
                           action='append', dest='test_tags',
                           default=None,
@@ -507,7 +512,7 @@ def __init__(self, address=None,
                  prefs=None, test_tags=None,
                  socket_timeout=BaseMarionetteArguments.socket_timeout_default,
                  startup_timeout=None, addons=None, workspace=None,
-                 verbose=0, e10s=True, emulator=False, **kwargs):
+                 verbose=0, e10s=True, emulator=False, headless=False, **kwargs):
         self._appinfo = None
         self._appName = None
         self._capabilities = None
@@ -548,6 +553,7 @@ def __init__(self, address=None,
         # and default location for profile is TMP
         self.workspace_path = workspace or os.getcwd()
         self.verbose = verbose
+        self.headless = headless
         self.e10s = e10s
         if self.e10s:
             self.prefs.update({
@@ -770,6 +776,9 @@ def _build_kwargs(self):
                     raise exc, msg.format(host, port, e), tb
         if self.workspace:
             kwargs['workspace'] = self.workspace_path
+        if self.headless:
+            kwargs['headless'] = True
+
         return kwargs
 
     def record_crash(self):
@@ -959,6 +968,7 @@ def add_test(self, test, expected='pass', group='default'):
                 "appname": self.appName,
                 "e10s": self.e10s,
                 "manage_instance": self.marionette.instance is not None,
+                "headless": self.headless
             }
             values.update(mozinfo.info)
 

testing/marionette/harness/marionette_harness/tests/unit/unit-tests.ini

@@ -80,6 +80,7 @@ skip-if = appname == 'fennec'
 [test_window_status_chrome.py]
 
 [test_screenshot.py]
+skip-if = headless # Relies on native styling which headless doesn't support.
 [test_cookies.py]
 [test_title.py]
 [test_title_chrome.py]

testing/mozharness/scripts/marionette.py

@@ -90,6 +90,13 @@ class MarionetteTest(TestingMixin, MercurialScript, BlobUploadMixin, TransferMix
          "default": False,
          "help": "Run tests with multiple processes. (Desktop builds only)",
          }
+    ], [
+        ["--headless"],
+        {"action": "store_true",
+         "dest": "headless",
+         "default": False,
+         "help": "Run tests in headless mode.",
+        }
     ], [
        ["--allow-software-gl-layers"],
        {"action": "store_true",
@@ -285,6 +292,9 @@ def run_tests(self):
         if not self.config['e10s']:
             cmd.append('--disable-e10s')
 
+        if self.config['headless']:
+            cmd.append('--headless')
+
         cmd.append('--gecko-log=%s' % os.path.join(dirs["abs_blob_upload_dir"],
                                                    'gecko.log'))
 

toolkit/xre/nsAppRunner.cpp

@@ -1660,6 +1660,10 @@ DumpHelp()
   printf("  --console          Start %s with a debugging console.\n", (const char*) gAppData->name);
 #endif
 
+#ifdef MOZ_WIDGET_GTK
+  printf("  --headless         Run without a GUI.\n");
+#endif
+
   // this works, but only after the components have registered.  so if you drop in a new command line handler, --help
   // won't not until the second run.
   // out of the bug, because we ship a component.reg file, it works correctly.
@@ -3139,6 +3143,10 @@ XREMain::XRE_mainInit(bool* aExitFlag)
     printf_stderr("*** You are running in chaos test mode. See ChaosMode.h. ***\n");
   }
 
+  if (CheckArg("headless")) {
+    PR_SetEnv("MOZ_HEADLESS=1");
+  }
+
   if (gfxPlatform::IsHeadless()) {
 #ifdef MOZ_WIDGET_GTK
     Output(false, "*** You are running in headless mode.\n");
@@ -4778,7 +4786,9 @@ XREMain::XRE_main(int argc, char* argv[], const BootstrapConfig& aConfig)
     }
 
 #ifdef MOZ_WIDGET_GTK
-    MOZ_gdk_display_close(mGdkDisplay);
+    if (!gfxPlatform::IsHeadless()) {
+      MOZ_gdk_display_close(mGdkDisplay);
+    }
 #endif
 
     {

widget/gtk/nsAppShell.cpp

@@ -21,11 +21,14 @@
 #ifdef MOZ_ENABLE_DBUS
 #include "WakeLockListener.h"
 #endif
+#include "gfxPlatform.h"
 #include "ScreenHelperGTK.h"
+#include "HeadlessScreenHelper.h"
 #include "mozilla/widget/ScreenManager.h"
 
 using mozilla::Unused;
 using mozilla::widget::ScreenHelperGTK;
+using mozilla::widget::HeadlessScreenHelper;
 using mozilla::widget::ScreenManager;
 using mozilla::LazyLogModule;
 
@@ -162,7 +165,11 @@ nsAppShell::Init()
 
     if (XRE_IsParentProcess()) {
         ScreenManager& screenManager = ScreenManager::GetSingleton();
-        screenManager.SetHelper(mozilla::MakeUnique<ScreenHelperGTK>());
+        if (gfxPlatform::IsHeadless()) {
+            screenManager.SetHelper(mozilla::MakeUnique<HeadlessScreenHelper>());
+        } else {
+            screenManager.SetHelper(mozilla::MakeUnique<ScreenHelperGTK>());
+        }
     }
 
 #if MOZ_WIDGET_GTK == 3