Backed out 5 changesets (bug 1498640, bug 1436037, bug 1497660, bug 1…

…505579) for mass failures on a CLOSED TREE

Backed out changeset 914a7a899dd1 (bug 1436037)
Backed out changeset e072757bf691 (bug 1436037)
Backed out changeset b0805a8cf6eb (bug 1505579)
Backed out changeset aa32ba9e6df3 (bug 1498640)
Backed out changeset ce441b8a784c (bug 1497660)

python/mozlint/test/python.ini

@@ -1,5 +1,5 @@
 [DEFAULT]
-subsuite = mozlint
+subsuite = mozlint, os == "linux"
 skip-if = python == 3
 
 [test_cli.py]

taskcluster/ci/source-test/python.yml

@@ -4,14 +4,11 @@ job-defaults:
     worker-type:
         by-platform:
             linux64.*: aws-provisioner-v1/gecko-t-linux-xlarge
-            windows10-64.*: aws-provisioner-v1/gecko-t-win10-64
     worker:
         by-platform:
             linux64.*:
                 docker-image: {in-tree: "lint"}
                 max-run-time: 3600
-            default:
-                max-run-time: 3600
     treeherder:
         kind: test
         tier: 2
@@ -37,9 +34,6 @@ taskgraph-tests:
 
 marionette-harness:
     description: testing/marionette/harness unit tests
-    platform:
-        - linux64/opt
-        - windows10-64/opt
     python-version: [2]
     treeherder:
         symbol: mnh
@@ -95,9 +89,6 @@ mochitest-harness:
 
 mozbase:
     description: testing/mozbase unit tests
-    platform:
-        - linux64/opt
-        - windows10-64/opt
     python-version: [2, 3]
     treeherder:
         symbol: mb
@@ -124,9 +115,6 @@ mozharness:
 
 mozlint:
     description: python/mozlint unit tests
-    platform:
-        - linux64/opt
-        - windows10-64/opt
     python-version: [2]
     treeherder:
         symbol: ml
@@ -152,9 +140,6 @@ mozrelease:
 
 mozterm:
     description: python/mozterm unit tests
-    platform:
-        - linux64/opt
-        - windows10-64/opt
     python-version: [2, 3]
     treeherder:
         symbol: term
@@ -179,9 +164,6 @@ mozversioncontrol:
 
 raptor:
     description: testing/raptor unit tests
-    platform:
-        - linux64/opt
-        - windows10-64/opt
     python-version: [2]
     treeherder:
         symbol: rap
@@ -233,9 +215,6 @@ reftest-harness:
 
 tryselect:
     description: tools/tryselect unit tests
-    platform:
-        - linux64/opt
-        - windows10-64/opt
     python-version: [2]
     treeherder:
         symbol: try

taskcluster/docs/docker-images.rst

@@ -94,8 +94,8 @@ for docker images that are pushed to a registry.
 Docker Registry Images (prebuilt)
 :::::::::::::::::::::::::::::::::
 
-***Warning: Registry images are only used for ``decision`` and
-``image_builder`` images.***
+***Warning: Use of prebuilt images should only be used for base images (those that other images
+will inherit from), or private images that must be stored in a private docker registry account.***
 
 These are images that are intended to be pushed to a docker registry and used
 by specifying the docker image name in task definitions.  They are generally
@@ -107,16 +107,17 @@ Example:
 
     image: taskcluster/decision:0.1.10@sha256:c5451ee6c655b3d97d4baa3b0e29a5115f23e0991d4f7f36d2a8f793076d6854
 
-Such images must always be referred to with both a version and a repo digest.
-For the decision image, the repo digest is stored in the ``HASH`` file in the
-image directory and used to refer to the image as above.  The version for both
-images is in ``VERSION``.
+Each image has a repo digest and a version. The repo digest is stored in the
+``HASH`` file in the image directory and used to refer to the image as above.
+The version is in ``VERSION``.
 
-The version file serves to help users identify which image is being used, and makes old
-versions easy to discover in the registry.
+The version file only serves to provide convenient names, such that old
+versions are easy to discover in the registry (and ensuring old versions aren't
+deleted by garbage-collection).
 
-The file ``taskcluster/docker/REGISTRY`` specifies the image registry to which
-the completed image should be uploaded.
+Each image directory also has a ``REGISTRY``, defaulting to the ``REGISTRY`` in
+the ``taskcluster/docker`` directory, and specifying the image registry to
+which the completed image should be uploaded.
 
 Docker Hashes and Digests
 .........................
@@ -163,19 +164,13 @@ Docker Registry Images
 
 Landing docker registry images takes a little more care.
 
-Begin by bumping the ``VERSION``.  Once the new version of the image has been
-built and tested locally, push it to the docker registry and make note of the
-resulting repo digest.  Put this value in the ``HASH`` file for the
-``decision`` image and in ``taskcluster/taskgraph/transforms/docker_image.py``
-for the ``image_builder`` image.
+Once a new version of the image has been built and tested locally, push it to
+the docker registry and make note of the resulting repo digest.  Put this value
+in the ``HASH`` file, and update any references to the image in the code or
+task definitions.
 
 The change is now safe to use in Try pushes.
 
-Note that ``image_builder`` change can be tested directly in try pushes without
-using a registry, as the in-registry ``image_builder`` image is used to build a
-task image which is then used to build other images.  It is referenced by hash
-in ``taskcluster/taskgraph/transforms/docker_image.py``.
-
 Special Dockerfile Syntax
 -------------------------
 

taskcluster/scripts/run-task

@@ -40,8 +40,8 @@ import urllib.request
 FINGERPRINT_URL = 'http://taskcluster/secrets/v1/secret/project/taskcluster/gecko/hgfingerprint'
 FALLBACK_FINGERPRINT = {
     'fingerprints':
-        "sha256:17:38:aa:92:0b:84:3e:aa:8e:52:52:e9:4c:2f:98:a9:0e:bf:6c:3e:e9"
-        ":15:ff:0a:29:80:f7:06:02:5b:e8:48"}
+        "sha256:8e:ad:f7:6a:eb:44:06:15:ed:f3:e4:69:a6:64:60:37:2d:ff:98:88:37"
+        ":bf:d7:b8:40:84:01:48:9c:26:ce:d9"}
 
 
 CACHE_UID_GID_MISMATCH = '''
@@ -448,14 +448,13 @@ def vcs_checkout(source_repo, dest, store_path,
     if sparse_profile:
         args.extend(['--sparseprofile', sparse_profile])
 
-    dest = os.path.abspath(dest)
     args.extend([
         revision_flag, revision_value,
         source_repo, dest,
     ])
 
     res = run_and_prefix_output(b'vcs', args,
-                                extra_env={'PYTHONUNBUFFERED': '1'})
+                                extra_env={b'PYTHONUNBUFFERED': b'1'})
     if res:
         sys.exit(res)
 
@@ -739,9 +738,6 @@ def main(args):
         return 1
 
     try:
-        if 'GECKO_PATH' in os.environ:
-            os.environ['GECKO_PATH'] = os.path.abspath(os.environ['GECKO_PATH'])
-
         if 'MOZ_FETCHES' in os.environ:
             fetch_artifacts()
 

taskcluster/taskgraph/transforms/docker_image.py

@@ -189,12 +189,11 @@ def fill_template(config, tasks):
 
         # We use the in-tree image_builder image to build docker images, but
         # that can't be used to build the image_builder image itself,
-        # obviously. So we fall back to an image on docker hub, identified
-        # by hash.  After the image-builder image is updated, it's best to push
-        # and update this hash as well, to keep image-builder builds up to date.
+        # obviously. So we fall back to the last snapshot of the image that
+        # was uploaded to docker hub.
         if image_name == 'image_builder':
-            hash = 'sha256:c6622fd3e5794842ad83d129850330b26e6ba671e39c58ee288a616a3a1c4c73'
-            worker['docker-image'] = 'taskcluster/image_builder@' + hash
+            worker['docker-image'] = 'taskcluster/image_builder@sha256:' + \
+                '24ce54a1602453bc93515aecd9d4ad25a22115fbc4b209ddb5541377e9a37315'
             # Keep in sync with the Dockerfile used to generate the
             # docker image whose digest is referenced above.
             worker['volumes'] = [

taskcluster/taskgraph/transforms/job/common.py

@@ -64,20 +64,10 @@ def support_vcs_checkout(config, job, taskdesc, sparse=False):
     This can only be used with ``run-task`` tasks, as the cache name is
     reserved for ``run-task`` tasks.
     """
-    is_win = job['worker']['os'] == 'windows'
-
-    if is_win:
-        checkoutdir = './build'
-        geckodir = '{}/src'.format(checkoutdir)
-        hgstore = 'y:/hg-shared'
-    else:
-        checkoutdir = '{workdir}/checkouts'.format(**job['run'])
-        geckodir = '{}/gecko'.format(checkoutdir)
-        hgstore = '{}/hg-store'.format(checkoutdir)
-
     level = config.params['level']
-    # native-engine and generic-worker do not support caches (yet), so we just
-    # do a full clone every time :(
+
+    # native-engine does not support caches (yet), so we just do a full clone
+    # every time :(
     if job['worker']['implementation'] in ('docker-worker', 'docker-engine'):
         name = 'level-%s-checkouts' % level
 
@@ -94,15 +84,15 @@ def support_vcs_checkout(config, job, taskdesc, sparse=False):
         taskdesc['worker'].setdefault('caches', []).append({
             'type': 'persistent',
             'name': name,
-            'mount-point': checkoutdir,
+            'mount-point': '{workdir}/checkouts'.format(**job['run']),
         })
 
     taskdesc['worker'].setdefault('env', {}).update({
         'GECKO_BASE_REPOSITORY': config.params['base_repository'],
         'GECKO_HEAD_REPOSITORY': config.params['head_repository'],
         'GECKO_HEAD_REV': config.params['head_rev'],
-        'GECKO_PATH': geckodir,
-        'HG_STORE_PATH': hgstore,
+        'GECKO_PATH': '{workdir}/checkouts/gecko'.format(**job['run']),
+        'HG_STORE_PATH': '{workdir}/checkouts/hg-store'.format(**job['run']),
     })
 
     if 'comm_base_repository' in config.params:

taskcluster/taskgraph/transforms/job/mach.py

@@ -30,19 +30,13 @@
 })
 
 
-defaults = {
-    'comm-checkout': False,
-}
-
-
-@run_job_using("docker-worker", "mach", schema=mach_schema, defaults=defaults)
-@run_job_using("native-engine", "mach", schema=mach_schema, defaults=defaults)
-@run_job_using("generic-worker", "mach", schema=mach_schema, defaults=defaults)
-def configure_mach(config, job, taskdesc):
+@run_job_using("docker-worker", "mach", schema=mach_schema, defaults={'comm-checkout': False})
+@run_job_using("native-engine", "mach", schema=mach_schema, defaults={'comm-checkout': False})
+def docker_worker_mach(config, job, taskdesc):
     run = job['run']
 
     # defer to the run_task implementation
-    run['command'] = 'cd $GECKO_PATH && ./mach {mach}'.format(**run)
+    run['command'] = 'cd {workdir}/checkouts/gecko && ./mach {mach}'.format(**run)
     run['using'] = 'run-task'
     del run['mach']
     configure_taskdesc_for_run(config, job, taskdesc, job['worker']['implementation'])

taskcluster/taskgraph/transforms/job/python_test.py

@@ -25,19 +25,16 @@
 })
 
 
-defaults = {
-    'python-version': 2,
-    'subsuite': 'default',
-}
-
-
-@run_job_using('docker-worker', 'python-test', schema=python_test_schema, defaults=defaults)
-@run_job_using('generic-worker', 'python-test', schema=python_test_schema, defaults=defaults)
-def configure_python_test(config, job, taskdesc):
+@run_job_using(
+    'docker-worker',
+    'python-test',
+    schema=python_test_schema,
+    defaults={'python-version': 2, 'subsuite': 'default'})
+def docker_worker_python_test(config, job, taskdesc):
     run = job['run']
 
     # defer to the run_task implementation
-    run['command'] = 'cd $GECKO_PATH && ' \
+    run['command'] = 'cd {workdir}/checkouts/gecko && ' \
         './mach python-test --python {python-version} --subsuite {subsuite}'.format(**run)
     run['using'] = 'run-task'
     del run['python-version']

taskcluster/taskgraph/transforms/job/run_task.py

@@ -40,12 +40,12 @@
 })
 
 
-def common_setup(config, job, taskdesc, command, geckodir):
+def common_setup(config, job, taskdesc, command, checkoutdir):
     run = job['run']
     if run['checkout']:
         support_vcs_checkout(config, job, taskdesc,
                              sparse=bool(run['sparse-profile']))
-        command.append('--vcs-checkout={}'.format(geckodir))
+        command.append('--vcs-checkout={}/gecko'.format(checkoutdir))
 
     if run['sparse-profile']:
         command.append('--sparse-profile=build/sparse-profiles/%s' %
@@ -72,8 +72,7 @@ def docker_worker_run_task(config, job, taskdesc):
     run = job['run']
     worker = taskdesc['worker'] = job['worker']
     command = ['/builds/worker/bin/run-task']
-    common_setup(config, job, taskdesc, command,
-                 geckodir='{workdir}/checkouts/gecko'.format(**run))
+    common_setup(config, job, taskdesc, command, checkoutdir='{workdir}/checkouts'.format(**run))
 
     if run.get('cache-dotcache'):
         worker['caches'].append({
@@ -99,8 +98,7 @@ def native_engine_run_task(config, job, taskdesc):
     run = job['run']
     worker = taskdesc['worker'] = job['worker']
     command = ['./run-task']
-    common_setup(config, job, taskdesc, command,
-                 geckodir='{workdir}/checkouts/gecko'.format(**run))
+    common_setup(config, job, taskdesc, command, checkoutdir='{workdir}/checkouts'.format(**run))
 
     worker['context'] = run_task_url(config)
 
@@ -119,16 +117,8 @@ def native_engine_run_task(config, job, taskdesc):
 def generic_worker_run_task(config, job, taskdesc):
     run = job['run']
     worker = taskdesc['worker'] = job['worker']
-    is_win = worker['os'] == 'windows'
-
-    if is_win:
-        command = ['C:/mozilla-build/python3/python3.exe', 'run-task']
-        geckodir = './build/src'
-    else:
-        command = ['./run-task']
-        geckodir = '{workdir}/checkouts/gecko'.format(**run)
-
-    common_setup(config, job, taskdesc, command, geckodir=geckodir)
+    command = ['./run-task']
+    common_setup(config, job, taskdesc, command, checkoutdir='{workdir}/checkouts'.format(**run))
 
     worker.setdefault('mounts', [])
     if run.get('cache-dotcache'):
@@ -145,15 +135,7 @@ def generic_worker_run_task(config, job, taskdesc):
 
     run_command = run['command']
     if isinstance(run_command, basestring):
-        run_command = ['bash', '-cx', '"{}"'.format(run_command)]
-
+        run_command = ['bash', '-cx', run_command]
     command.append('--')
     command.extend(run_command)
-
-    if is_win:
-        worker['command'] = [' '.join(command)]
-    else:
-        worker['command'] = [
-            ['chmod', '+x', 'run-task'],
-            command,
-        ]
+    worker['command'] = [['chmod', '+x', 'run-task'], command]

taskcluster/taskgraph/transforms/task.py

@@ -1869,8 +1869,8 @@ def check_caches_are_volumes(task):
         return
 
     raise Exception('task %s (image %s) has caches that are not declared as '
-                    'Docker volumes: %s '
-                    '(have you added them as VOLUMEs in the Dockerfile?)'
+                    'Docker volumes: %s'
+                    'Have you added them as VOLUMEs in the Dockerfile?'
                     % (task['label'], task['worker']['docker-image'],
                        ', '.join(sorted(missing))))
 

taskcluster/taskgraph/util/docker.py

@@ -103,8 +103,6 @@ def post_to_docker(tar, api_path, **kwargs):
                 sys.stderr.write('{}\n'.format(data['status']))
         elif 'stream' in data:
             sys.stderr.write(data['stream'])
-        elif 'aux' in data:
-            sys.stderr.write(repr(data['aux']))
         elif 'error' in data:
             sys.stderr.write('{}\n'.format(data['error']))
             # Sadly, docker doesn't give more than a plain string for errors,