Backed out changeset 3c0a7527608a (bug 1335873)

python/mach_commands.py

@@ -57,6 +57,11 @@ def python(self, args):
         default=False,
         action='store_true',
         help='Stop running tests after the first error or failure.')
+    @CommandArgument('--path-only',
+        default=False,
+        action='store_true',
+        help=('Collect all tests under given path instead of default '
+              'test resolution. Supports pytest-style tests.'))
     @CommandArgument('-j', '--jobs',
         default=1,
         type=int,
@@ -74,6 +79,7 @@ def python_test(self,
                     test_objects=None,
                     subsuite=None,
                     verbose=False,
+                    path_only=False,
                     stop=False,
                     jobs=1):
         self._activate_virtualenv()
@@ -103,19 +109,30 @@ def find_tests_by_path():
         # which produces output in the format Mozilla infrastructure expects.
         # Some tests are run via pytest.
         if test_objects is None:
-            from mozbuild.testing import TestResolver
-            resolver = self._spawn(TestResolver)
-            if tests:
-                # If we were given test paths, try to find tests matching them.
-                test_objects = resolver.resolve_tests(paths=tests,
-                                                      flavor='python')
+            # If we're not being called from `mach test`, do our own
+            # test resolution.
+            if path_only:
+                if tests:
+                    test_objects = [{'path': p} for p in find_tests_by_path()]
+                else:
+                    self.log(logging.WARN, 'python-test', {},
+                             'TEST-UNEXPECTED-FAIL | No tests specified')
+                    test_objects = []
             else:
-                # Otherwise just run everything in PYTHON_UNITTEST_MANIFESTS
-                test_objects = resolver.resolve_tests(flavor='python')
+                from mozbuild.testing import TestResolver
+                resolver = self._spawn(TestResolver)
+                if tests:
+                    # If we were given test paths, try to find tests matching them.
+                    test_objects = resolver.resolve_tests(paths=tests,
+                                                          flavor='python')
+                else:
+                    # Otherwise just run everything in PYTHON_UNITTEST_MANIFESTS
+                    test_objects = resolver.resolve_tests(flavor='python')
 
         if not test_objects:
-            message = 'TEST-UNEXPECTED-FAIL | No tests collected ' + \
-                      '(Not in PYTHON_UNITTEST_MANIFESTS?)'
+            message = 'TEST-UNEXPECTED-FAIL | No tests collected'
+            if not path_only:
+                message += ' (Not in PYTHON_UNITTEST_MANIFESTS? Try --path-only?)'
             self.log(logging.WARN, 'python-test', {}, message)
             return 1
 

taskcluster/ci/marionette-harness/kind.yml

@@ -0,0 +1,50 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# NOTE: please write a description of this kind in taskcluster/docs/kinds.rst
+
+implementation: taskgraph.task.transform:TransformTask
+
+transforms:
+   - taskgraph.transforms.marionette_harness:transforms
+   - taskgraph.transforms.task:transforms
+
+# NOTE: this task should be refactored so that it is invoked as a job either
+# with a run.using of "mozharness", and combined with the source-check kind.
+
+jobs:
+    marionette-harness/opt:
+        description: "Marionette harness unit test"
+        attributes:
+            build_platform: marionette-harness
+            build_type: opt
+        treeherder:
+            platform: linux64/opt
+            kind: test
+            tier: 2
+            symbol: tc(Mn-h)
+        worker-type: aws-provisioner-v1/gecko-t-linux-xlarge
+        worker:
+            implementation: docker-worker
+            docker-image: {in-tree: desktop-build}  # NOTE: better to use the lint image
+            env:
+                JOB_SCRIPT: "taskcluster/scripts/tester/harness-test-linux.sh"
+                MOZHARNESS_SCRIPT: "testing/mozharness/scripts/marionette_harness_tests.py"
+                TOOLS_DISABLE: "true"
+            artifacts:
+              - name: public/logs/
+                path: /home/worker/workspace/mozharness_workspace/upload/logs/
+                type: directory
+            command:
+              - "bash"
+              - "/home/worker/bin/build.sh"
+              - "--tests=testing/marionette/harness/marionette_harness/tests/harness_unit"
+              - "--work-dir=mozharness_workspace"
+            max-run-time: 1800
+        when:
+            files-changed:
+              - "testing/marionette/harness/**"
+              - "testing/mozbase/mozlog/mozlog/pytest_mozlog/**"
+              - "testing/mozharness/scripts/marionette_harness_tests.py"
+              - "testing/config/marionette_harness_test_requirements.txt"

taskcluster/ci/source-test/python-tests.yml

@@ -22,35 +22,6 @@ taskgraph-tests/opt:
             - 'config/mozunit.py'
             - 'python/mach/**/*.py'
 
-marionette-harness/opt:
-    description: testing/marionette/harness unit tests
-    platforms:
-        - linux64/opt
-    treeherder:
-        symbol: py(mnh)
-        kind: test
-        tier: 2
-    worker-type:
-        by-platform:
-            linux64.*: aws-provisioner-v1/b2gtest
-    worker:
-        by-platform:
-            linux64.*:
-                implementation: docker-worker
-                docker-image: {in-tree: "lint"}
-                max-run-time: 3600
-    run:
-        using: mach
-        mach: python-test --subsuite marionette-harness
-    run-on-projects:
-        - integration
-        - release
-    when:
-        files-changed:
-          - 'testing/marionette/harness/**'
-          - 'testing/mozbase/mozlog/mozlog/pytest_mozlog/**'
-          - 'python/mach_commands.py'
-
 mozbase/opt:
     description: testing/mozbase unit tests
     platforms:

taskcluster/docs/kinds.rst

@@ -90,6 +90,11 @@ spidermonkey
 Spidermonkey tasks check out the full gecko source tree, then compile only the
 spidermonkey portion.  Each task runs specific tests after the build.
 
+marionette-harness
+------------------
+
+TBD (Maja)
+
 Tests
 -----
 

taskcluster/scripts/tester/harness-test-linux.sh

@@ -0,0 +1,40 @@
+#! /bin/bash -vex
+
+set -x -e
+
+echo "running as" $(id)
+
+####
+# Taskcluster friendly wrapper for running a script in
+# testing/mozharness/scripts in a source checkout (no build).
+# Example use: Python-only harness unit tests
+####
+
+: WORKSPACE                     ${WORKSPACE:=/home/worker/workspace}
+: SRC_ROOT                      ${SRC_ROOT:=$WORKSPACE/build/src}
+# These paths should be relative to $SRC_ROOT
+: MOZHARNESS_SCRIPT             ${MOZHARNESS_SCRIPT}
+: MOZHARNESS_CONFIG             ${MOZHARNESS_CONFIG}
+: mozharness args               "${@}"
+
+set -v
+cd $WORKSPACE
+
+fail() {
+    echo # make sure error message is on a new line
+    echo "[harness-test-linux.sh:error]" "${@}"
+    exit 1
+}
+
+if [[ -z ${MOZHARNESS_SCRIPT} ]]; then fail "MOZHARNESS_SCRIPT is not set"; fi
+
+# support multiple, space delimited, config files
+config_cmds=""
+for cfg in $MOZHARNESS_CONFIG; do
+  config_cmds="${config_cmds} --config-file ${SRC_ROOT}/${cfg}"
+done
+
+python2.7 $SRC_ROOT/${MOZHARNESS_SCRIPT} ${config_cmds} "${@}"
+
+
+

taskcluster/taskgraph/try_option_syntax.py

@@ -37,6 +37,7 @@
 JOB_KINDS = set([
     'source-test',
     'toolchain',
+    'marionette-harness',
     'android-stuff',
 ])
 

testing/config/marionette_harness_test_requirements.txt

@@ -0,0 +1,13 @@
+-r mozbase_requirements.txt
+
+# TODO - if we structure common.tests.zip to match the in-tree structure of the
+# testing directory, we could use ./marionette_requirements.txt instead
+../web-platform/tests/tools/wptserve
+../marionette/client
+../marionette/harness/marionette_harness/runner/mixins/browsermob-proxy-py
+../marionette/harness
+
+# pytest
+../../python/py
+../../python/pytest
+../../python/mock-1.0.0

testing/marionette/harness/marionette_harness/tests/harness_unit/test_httpd.py

@@ -87,5 +87,4 @@ def handler(request, response):
 
 if __name__ == "__main__":
     import sys
-    sys.exit(pytest.main(
-        ['-p', 'no:terminalreporter', '--log-tbpl=-', __file__]))
+    sys.exit(pytest.main(["--verbose", __file__]))

testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_arguments.py

@@ -29,5 +29,4 @@ def _is_float_convertible(value):
 
 if __name__ == '__main__':
     import sys
-    sys.exit(pytest.main(
-        ['-p', 'no:terminalreporter', '--log-tbpl=-', __file__]))
+    sys.exit(pytest.main(['--verbose', __file__]))

testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_harness.py

@@ -105,5 +105,4 @@ def test_harness_sets_up_default_test_handlers(mach_parsed_kwargs):
 
 if __name__ == '__main__':
     import sys
-    sys.exit(pytest.main(
-        ['-p', 'no:terminalreporter', '--log-tbpl=-', __file__]))
+    sys.exit(pytest.main(['--verbose', __file__]))

testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_runner.py

@@ -439,5 +439,4 @@ def test_e10s_option_clash_raises(mock_runner):
 
 if __name__ == '__main__':
     import sys
-    sys.exit(pytest.main(
-        ['-p', 'no:terminalreporter', '--log-tbpl=-', __file__]))
+    sys.exit(pytest.main(['--verbose', __file__]))

testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_test_result.py

@@ -51,5 +51,4 @@ def test_crash_is_recorded_as_error(empty_marionette_test,
 
 if __name__ == '__main__':
     import sys
-    sys.exit(pytest.main(
-        ['-p', 'no:terminalreporter', '--log-tbpl=-', __file__]))
+    sys.exit(pytest.main(['--verbose', __file__]))

testing/marionette/harness/marionette_harness/tests/harness_unit/test_serve.py

@@ -64,5 +64,4 @@ def test_where_is():
 
 if __name__ == "__main__":
     import sys
-    sys.exit(pytest.main(
-        ['-s', '-p', 'no:terminalreporter', '--log-tbpl=-', __file__]))
+    sys.exit(pytest.main(["-s", "--verbose", __file__]))

testing/mozbase/packages.txt

@@ -8,7 +8,7 @@ mozhttpd.pth:testing/mozbase/mozhttpd
 mozinfo.pth:testing/mozbase/mozinfo
 mozinstall.pth:testing/mozbase/mozinstall
 mozleak.pth:testing/mozbase/mozleak
-setup.py:testing/mozbase/mozlog:develop
+mozlog.pth:testing/mozbase/mozlog
 moznetwork.pth:testing/mozbase/moznetwork
 mozprocess.pth:testing/mozbase/mozprocess
 mozprofile.pth:testing/mozbase/mozprofile