bug 1397552 - add a release promotion action. r=bstack

Add a release promotion custom action for releng's TC release promotion migration work.

This action generates a graph dependent on previously built tasks. To track these, we add the `do_not_optimize` and `existing_tasks` parameters. The `do_not_optimize` parameter specifies tasks that we want to explicitly exclude from taskgraph optimization. The `existing_tasks` parameter specifies a label-to-taskid map for tasks from previous graphs.

MozReview-Commit-ID: 1vKrNUavM4V

taskcluster/docs/parameters.rst

@@ -113,9 +113,6 @@ syntax or reading a project-specific configuration file).
     The method to use to determine the target task set.  This is the suffix of
     one of the functions in ``taskcluster/taskgraph/target_tasks.py``.
 
-``optimize_target_tasks``
-    If true, then target tasks are eligible for optimization.
-
 ``include_nightly``
     If true, then nightly tasks are eligible for optimization.
 
@@ -125,6 +122,22 @@ syntax or reading a project-specific configuration file).
    Suitable contents can be generated with ``mach release-history``,
    which will print to the console by default.
 
+Optimization
+------------
+
+``optimize_target_tasks``
+    If true, then target tasks are eligible for optimization.
+
+``do_not_optimize``
+   Specify tasks to not optimize out of the graph. This is a list of labels.
+   Any tasks in the graph matching one of the labels will not be optimized out
+   of the graph.
+
+``existing_tasks``
+   Specify tasks to optimize out of the graph. This is a dictionary of label to taskId.
+   Any tasks in the graph matching one of the labels will use the previously-run
+   taskId rather than submitting a new task.
+
 Comm Push Information
 ---------------------
 

taskcluster/mach_commands.py

@@ -30,7 +30,7 @@ class ShowTaskGraphSubCommand(SubCommand):
     def __call__(self, func):
         after = SubCommand.__call__(self, func)
         args = [
-            CommandArgument('--root', '-r', default='taskcluster/ci',
+            CommandArgument('--root', '-r',
                             help="root of the taskgraph definition relative to topsrcdir"),
             CommandArgument('--quiet', '-q', action="store_true",
                             help="suppress all logging output"),
@@ -104,7 +104,6 @@ def taskgraph_morphed(self, **options):
     @SubCommand('taskgraph', 'decision',
                 description="Run the decision task")
     @CommandArgument('--root', '-r',
-                     default='taskcluster/ci',
                      help="root of the taskgraph definition relative to topsrcdir")
     @CommandArgument('--base-repository',
                      required=True,
@@ -322,7 +321,7 @@ def show_taskgraph(self, graph_attr, options):
             parameters.check()
 
             tgg = taskgraph.generator.TaskGraphGenerator(
-                root_dir=options['root'],
+                root_dir=options.get('root'),
                 parameters=parameters)
 
             tg = getattr(tgg, graph_attr)

taskcluster/taskgraph/actions/release_promotion.py

@@ -0,0 +1,112 @@
+# -*- coding: utf-8 -*-
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import os
+
+from .registry import register_callback_action
+
+from .util import (find_decision_task, find_existing_tasks_from_previous_kinds,
+                   find_hg_revision_pushlog_id)
+from taskgraph.util.taskcluster import get_artifact
+from taskgraph.taskgraph import TaskGraph
+from taskgraph.decision import taskgraph_decision
+from taskgraph.parameters import Parameters
+
+
+@register_callback_action(
+    name='release-promotion',
+    title='Release Promotion',
+    symbol='Relpro',
+    description="Promote a release.",
+    order=10000,
+    context=[],
+    schema={
+        'type': 'object',
+        'properties': {
+            'build_number': {
+                'type': 'integer',
+                'default': 1,
+                'minimum': 1,
+                'title': 'The release build number',
+                'description': ('The release build number. Starts at 1 per '
+                                'release version, and increments on rebuild.'),
+            },
+            'do_not_optimize': {
+                'type': 'array',
+                'description': ('An list of labels to avoid optimizing out '
+                                'of the graph (to force a rerun of, say, '
+                                'funsize docker-image tasks).'),
+                'items': {
+                    'type': 'string',
+                },
+            },
+            'revision': {
+                'type': 'string',
+                'title': 'Optional: revision to promote',
+                'description': ('Optional: the revision to promote. If specified, '
+                                'and if neither `pushlog_id` nor `previous_graph_kinds` '
+                                'is specified, find the `pushlog_id using the '
+                                'revision.'),
+            },
+            'target_tasks_method': {
+                'type': 'string',
+                'title': 'target task method',
+                'description': ('The target task method to use to generate the new '
+                                'graph.'),
+            },
+            'previous_graph_kinds': {
+                'type': 'array',
+                'description': 'An array of kinds to use from the previous graph(s).',
+                'items': {
+                    'type': 'string',
+                },
+            },
+            'previous_graph_ids': {
+                'type': 'array',
+                'description': ('An array of taskIds of decision or action tasks '
+                                'from the previous graph(s) to use to populate '
+                                'our `previous_graph_kinds`.'),
+                'items': {
+                    'type': 'string',
+                },
+            },
+        }
+        "required": ['build_number', 'target_tasks_method', 'previous_graph_kinds'],
+    }
+)
+def release_promotion_action(parameters, input, task_group_id, task_id, task):
+    # build_number, previous_graph_kinds, target_tasks_method are required
+    os.environ['BUILD_NUMBER'] = str(input['build_number'])
+    target_tasks_method = input['target_tasks_method']
+    previous_graph_kinds = input['previous_graph_kinds']
+    do_not_optimize = input.get('do_not_optimize', [])
+    # make parameters read-write
+    parameters = dict(parameters)
+    # Build previous_graph_ids from ``previous_graph_ids``, ``pushlog_id``,
+    # or ``revision``.
+    previous_graph_ids = input.get('previous_graph_ids')
+    if not previous_graph_ids:
+        revision = input.get('revision')
+        parameters['pushlog_id'] = parameters['pushlog_id'] or \
+            find_hg_revision_pushlog_id(parameters, revision)
+        previous_graph_ids = [find_decision_task(parameters)]
+
+    # Download parameters and full task graph from the first decision task.
+    parameters = get_artifact(previous_graph_ids[0], "public/parameters.yml")
+    full_task_graph = get_artifact(previous_graph_ids[0], "public/full-task-graph.json")
+    _, full_task_graph = TaskGraph.from_json(full_task_graph)
+    parameters['existing_tasks'] = find_existing_tasks_from_previous_kinds(
+        full_task_graph, previous_graph_ids, previous_graph_kinds
+    )
+    parameters['do_not_optimize'] = do_not_optimize
+    parameters['target_tasks_method'] = target_tasks_method
+
+    # make parameters read-only
+    parameters = Parameters(**parameters)
+
+    taskgraph_decision({}, parameters=parameters)

taskcluster/taskgraph/actions/util.py

@@ -7,6 +7,7 @@
 from __future__ import absolute_import, print_function, unicode_literals
 
 import logging
+import requests
 import os
 
 from requests.exceptions import HTTPError
@@ -19,6 +20,8 @@
 
 logger = logging.getLogger(__name__)
 
+PUSHLOG_TMPL = '{}/json-pushes?version=2&changeset={}&tipsonly=1&full=1'
+
 
 def find_decision_task(parameters):
     """Given the parameters for this action, find the taskId of the decision
@@ -28,6 +31,37 @@ def find_decision_task(parameters):
         parameters['pushlog_id']))
 
 
+def find_hg_revision_pushlog_id(parameters, revision):
+    """Given the parameters for this action and a revision, find the
+    pushlog_id of the revision."""
+    pushlog_url = PUSHLOG_TMPL.format(parameters['head_repository'], revision)
+    r = requests.get(pushlog_url)
+    r.raise_for_status()
+    pushes = r.json()['pushes'].keys()
+    if len(pushes) != 1:
+        raise RuntimeError(
+            "Unable to find a single pushlog_id for {} revision {}: {}".format(
+                parameters['head_repository'], revision, pushes
+            )
+        )
+    return pushes[0]
+
+
+def find_existing_tasks_from_previous_kinds(full_task_graph, previous_graph_ids,
+                                            previous_graph_kinds):
+    """Given a list of previous decision/action taskIds and kinds to replace
+    from the previous graphs, return a dictionary of labels-to-taskids to use
+    as ``existing_tasks`` in the optimization step."""
+    existing_tasks = {}
+    for previous_graph_id in previous_graph_ids:
+        label_to_taskid = get_artifact(previous_graph_id, "public/label-to-taskid.json")
+        kind_labels = set(t.label for t in full_task_graph.tasks.itervalues()
+                          if t.attributes['kind'] in previous_graph_kinds)
+        for label in set(label_to_taskid.keys()).intersection(kind_labels):
+            existing_tasks[label] = label_to_taskid[label]
+    return existing_tasks
+
+
 def fetch_graph_and_labels(parameters):
     decision_task_id = find_decision_task(parameters)
 

taskcluster/taskgraph/decision.py

@@ -84,7 +84,7 @@
 }
 
 
-def taskgraph_decision(options):
+def taskgraph_decision(options, parameters=None):
     """
     Run the decision task.  This function implements `mach taskgraph decision`,
     and is responsible for
@@ -96,11 +96,11 @@ def taskgraph_decision(options):
      * calling TaskCluster APIs to create the graph
     """
 
-    parameters = get_decision_parameters(options)
+    parameters = parameters or get_decision_parameters(options)
 
     # create a TaskGraphGenerator instance
     tgg = TaskGraphGenerator(
-        root_dir=options['root'],
+        root_dir=options.get('root'),
         parameters=parameters)
 
     # write out the parameters used to generate this graph
@@ -163,6 +163,8 @@ def get_decision_parameters(options):
         'check_servo',
         'target_tasks_method',
     ]
+    parameters['existing_tasks'] = {}
+    parameters['do_not_optimize'] = []
 
     # owner must be an email, but sometimes (e.g., for ffxbld) it is not, in which
     # case, fake it

taskcluster/taskgraph/generator.py

@@ -87,6 +87,8 @@ def __init__(self, root_dir, parameters):
         @param parameters: parameters for this task-graph generation
         @type parameters: dict
         """
+        if root_dir is None:
+            root_dir = 'taskcluster/ci'
         self.root_dir = root_dir
         self.parameters = parameters
 
@@ -262,12 +264,14 @@ def _run(self):
         yield verifications('target_task_graph', target_task_graph)
 
         logger.info("Generating optimized task graph")
-        do_not_optimize = set()
+        existing_tasks = self.parameters.get('existing_tasks')
+        do_not_optimize = set(self.parameters.get('do_not_optimize', []))
         if not self.parameters.get('optimize_target_tasks', True):
-            do_not_optimize = target_task_set.graph.nodes
+            do_not_optimize = set(target_task_set.graph.nodes).union(do_not_optimize)
         optimized_task_graph, label_to_taskid = optimize_task_graph(target_task_graph,
                                                                     self.parameters,
-                                                                    do_not_optimize)
+                                                                    do_not_optimize,
+                                                                    existing_tasks=existing_tasks)
 
         yield verifications('optimized_task_graph', optimized_task_graph)
 

taskcluster/taskgraph/parameters.py

@@ -31,6 +31,8 @@ def get_head_ref():
 PARAMETERS = {
     'base_repository': 'https://hg.mozilla.org/mozilla-unified',
     'build_date': lambda: int(time.time()),
+    'do_not_optimize': [],
+    'existing_tasks': {},
     'filters': ['check_servo', 'target_tasks_method'],
     'head_ref': get_head_ref,
     'head_repository': 'https://hg.mozilla.org/mozilla-central',