pipeline_controller

#!/usr/bin/env python
# ----------------------------------------------------------------------
# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2015, Numenta, Inc.  Unless you have purchased from
# Numenta, Inc. a separate commercial license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU Affero Public License for more details.
#
# You should have received a copy of the GNU Affero Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------

"""
  This script represents the controller for a master Jenkins pipeline. There
  will be a single Jenkins job that executes automatically whenever the repo is
  modified. It is the resonsibility of this script to ensure that the correct
  Jenkins jobs are subsequently called, based on which folders contain
  modifications.  This script assumes that the `products` repo is in the correct
  state for building and testing.

  Each Jenkins job will call the relevant `Single Point of Entry` for the
  folder. This allows us to take advantage of Jenkins for scheduling,
  multi-threading, delegation to appropriate slaves, notification, etc. But, it
  also allows us to ensure that only the relevant jobs are triggered with any
  given commit.
"""

import argparse
import os

from jenkinsapi.jenkins import Jenkins
from jenkinsapi.custom_exceptions import UnknownJob

from infrastructure.utilities.exceptions import (
  CommandFailedError, InvalidParametersError)
from infrastructure.utilities.diagnostics import initPipelineLogger
from infrastructure.utilities.git import (
  getActiveBranch,
  getCurrentSha,
  getModifiedFilesBetweenRevisions)


g_logger = initPipelineLogger("product-master-pipeline")



def getConnectionToJenkins():
  """
    Connects to Jenkins and returns the connection object.

    :returns: Jenkins connection
  """
  return Jenkins(
    os.environ["JENKINS_HOST"],
    os.environ["JENKINS_USER"],
    os.environ["JENKINS_PASSWORD"])


def invokeBuild(jobName, buildParams):
  """
    This function executes a specified Jenkins job with the parameters provided.

    :param jobName: Name of the job to be executed. Must be a literal match to a
      Jenkins job

    :param buildParams: a dict containing {key:value} pairs for use by Jenkins.
      The Keys are defined by the Jenkins jobs and valid values will be
      different per Job.

    :raises jenkinsapi.custom_exceptions.UnknownJob: if Jenkins isn't able to
      find the specified job.
  """
  buildNum = os.environ.get("BUILD_NUMBER", "local")
  getConnectionToJenkins()[jobName].invoke(build_params=buildParams,
                                           cause="product-master-build-%s" %
                                                 buildNum)


def invokeHtmItBuild(gitRemote, branch, buildSha):
  """
    This function invokes the Jenkins job related to HTM-IT builds.

    :param gitRemote: The remote to be used when accessing git (e.g.:
      git@github.com:Numenta/numenta-apps.git)

    :param branch: The git branch to use.

    :param buildSha: The commit SHA to set the git repo to before building.
  """
  params = {"GIT_REMOTE": gitRemote,
            "BRANCH": branch,
            "COMMIT_SHA": buildSha}
  invokeBuild("htm-it-product-pipeline", params)
  g_logger.info("Invoking HTM-IT Build")


def invokeHtmItMobileBuild(gitRemote, branch, buildSha):
  """
    This function invokes the Jenkins job related to HTM-IT-Mobile builds.

    :param gitRemote: The remote to be used when accessing git (e.g.:
      git@github.com:Numenta/numenta-apps.git)

    :param branch: The git branch to use.

    :param buildSha: The commit SHA to set the git repo to before building.
  """
  g_logger.info("Invoking HTM-IT Mobile Build")
  params = {"GIT_REMOTE": gitRemote,
            "BRANCH": branch,
            "COMMIT_SHA": buildSha,
            "PIPELINE": "product-master-build"}
  invokeBuild("htm-it-mobile-product-pipeline", params)


def invokeTaurusBuild(gitRemote, branch, buildSha):
  """
    Invoke `refresh-taurus-servers` pipeline, overriding USER_GIT_URL,
    USER_GIT_BRANCH_NAME, and USER_GIT_COMMIT environment variable defaults.

    :param gitRemote: The remote to be used when accessing git (e.g.:
      git@github.com:Numenta/numenta-apps.git)

    :param branch: The git branch to use.

    :param buildSha: The commit SHA to set the git repo to before building.
  """
  params = {"USER_GIT_URL": gitRemote,
            "USER_GIT_BRANCH_NAME": branch,
            "USER_GIT_COMMIT": buildSha}
  invokeBuild("refresh-taurus-servers", params)
  g_logger.info("Invoking Taurus Build")


def invokeTaurusMobileBuild(gitRemote, branch, buildSha):
  """
    This function invokes the Jenkins job related to Taurus-Mobile builds.

    :param gitRemote: The remote to be used when accessing git (e.g.:
      git@github.com:Numenta/numenta-apps.git)

    :param branch: The git branch to use.

    :param buildSha: The commit SHA to set the git repo to before building.
  """
  params = {"GIT_REMOTE": gitRemote,
            "BRANCH": branch,
            "COMMIT_SHA": buildSha}
  invokeBuild("taurus-mobile-product-pipeline", params)
  g_logger.info("Invoking Taurus Mobile Build")


def invokeInfrastructureBuild(gitRemote, commitish):
  """
    This function will be used to invoke the Jenkins job related to
    Infrastructure builds as appropriate.

    :param gitRemote: The remote to be used when accessing git (e.g.:
      git@github.com:Numenta/numenta-apps.git)

    :param commitish: The commit-ish to set the git repo to before building.
  """
  params = {"GIT_REMOTE": gitRemote,
            "COMMITISH": commitish}
  invokeBuild("infrastructure-python-pipeline", params)
  g_logger.info("Invoking Infrastructure Build")


def invokeRpmBuild(gitRemote, branch, buildSha):
  """
    This function will be used to invoke the Jenkins job related to
    infrastruture-rpm-pipeline builds as appropriate.

    :param gitRemote: The remote to be used when accessing git (e.g.:
      git@github.com:Numenta/numenta-apps.git)

    :param branch: The git branch to use.

    :param buildSha: The commit SHA to set the git repo to before building.
  """
  params = {"GIT_REMOTE": gitRemote,
            "BRANCH": branch,
            "COMMIT_SHA": buildSha}
  invokeBuild("infrastruture-rpm-pipeline", params)
  g_logger.info("Invoking infrastruture-rpm-pipeline Build")



def getLatestGreenRevision(jobName):
  """
    Returns the SHA of the latest green build of the specified pipeline

    :param jobName: Name of the job to be executed. Must be a literal match to a
      Jenkins job

    :returns: Commit SHA of the latest passing Green build of the given Job.
  """
  return getConnectionToJenkins()[jobName].get_last_good_build().get_revision()


def getChangedFoldersBetweenRevisions(startSha, endSha):
  """
    This function returns a list of the root folders that contain changes from
    one commit SHA to another. This allows us to take, for example, the latest
    green build of this pipeline and compare it to the current commit SHA being
    executed and determine which downstream Jenkins jobs to execute.

    :param startSha: The first commit SHA to use in a revision history to see
      what has changed

    :param endSha: The last commit SHA to use in a revision history to see what
      has changed

    :returns: A `set` of folders that have any modified files in them between
      the commit SHAs specified.
    :rtype: set

  """
  files = getModifiedFilesBetweenRevisions(startSha=startSha,
                                           endSha=endSha,
                                           logger=g_logger)
  rootFolders = []
  for name in files:
    if not name.split("/")[0] in rootFolders:
      rootFolders.append(name.split("/")[0])
  return set(rootFolders)


def parseArgs():
  """
    Parse the command line arguments
  """
  parser = argparse.ArgumentParser(description="Jenkins pipeline controller")
  parser.add_argument("--gitRemote", dest="gitRemote", type=str,
                      default="git@github.com:Numenta/numenta-apps.git",
                      help="Specifies which git remote to use")
  parser.add_argument("--startSha", dest="startSha", type=str,
                      help=("**OPTIONAL** If specified, this determines which "
                            "SHA to start with for comparing which folders "
                            "have changed. If unspecified, the script will use "
                            "the latest green build commit SHA from the "
                            "product-master-build Jenkins job."))
  parser.add_argument("--branch", dest="branch", type=str,
                      help=("**OPTIONAL** If specified, this determines which "
                            "branch to work off of. Otherwise, this will be "
                            "set to the active branch."))
  return parser.parse_args()


def main(args):
  """
    Main function for the pipeline. Executes all sub-tasks.

    :param args: Parsed command line arguments
  """

  try:
    if (not os.environ["JENKINS_HOST"] or not os.environ["JENKINS_USER"] or
        not os.environ["JENKINS_PASSWORD"]):
      raise InvalidParametersError("You must set the JENKINS_HOST, JENKINS_USER"
                                   " and JENKINS_PASSWORD env vars to proceed")
  except KeyError:
    raise InvalidParametersError("You must set the JENKINS_HOST, JENKINS_USER"
                                 " and JENKINS_PASSWORD env vars to proceed")

  try:
    #STEP 1: Determine which folders have been modified
    startSha = args.startSha or getLatestGreenRevision("product-master-build")
    currentSha = getCurrentSha(logger=g_logger)
    branch = args.branch or getActiveBranch(logger=g_logger)

    rootFolders = getChangedFoldersBetweenRevisions(startSha, currentSha)
    g_logger.info("Folders with modifications: %s", rootFolders)

    #STEP 2: Execute the appropriate sub-pipelines as needed
    if set(["htm.it", "htmengine", "nta.utils"]) & rootFolders:
      invokeHtmItBuild(args.gitRemote, branch, currentSha)
    if (set(["taurus", "taurus.metric_collectors", "htmengine", "nta.utils"]) &
        rootFolders):
      invokeTaurusBuild(args.gitRemote, branch, currentSha)
    # Only run the htm-it-mobile pipeline if it has been updated itself. Otherwise
    # rely on the HTM-IT pipeline to kick off this build on success.
    if (set(["htm-it-mobile", "mobile-core", "nta.utils"]) & rootFolders):
      invokeHtmItMobileBuild(args.gitRemote, branch, currentSha)
    if set(["taurus", "taurus.metric_collectors", "nta.utils", "htmengine",
            "taurus-mobile", "mobile-core"]) & rootFolders:
      invokeTaurusMobileBuild(args.gitRemote, branch, currentSha)
    if set(["infrastructure"]) & rootFolders:
      invokeInfrastructureBuild(args.gitRemote, currentSha)

    if (set(["infrastructure/utilities", "infrastructure/saltcellar"]) &
        rootFolders):
      invokeRpmBuild(args.gitRemote, branch, currentSha)

  except CommandFailedError:
    g_logger.exception("Not running from a git repo or other git related error")
    raise
  except UnknownJob:
    g_logger.exception("Could not find the Jenkins job specified; aborting")
    raise
  except:
    g_logger.exception("Script execution failed")
    raise


if __name__ == "__main__":
  main(parseArgs())