Merge branch 'master' into alerting/client-types

* master:
  [kbn/pm] add caching to bootstrap (elastic#53622)
  adds createdAt and updatedAt fields to alerting (elastic#53793)

Author: gmmorris

packages/kbn-dev-utils/package.json

@@ -15,6 +15,7 @@
     "execa": "^3.2.0",
     "exit-hook": "^2.2.0",
     "getopts": "^2.2.5",
+    "load-json-file": "^6.2.0",
     "moment": "^2.24.0",
     "rxjs": "^6.5.3",
     "tree-kill": "^1.2.1",

packages/kbn-dev-utils/src/constants.ts

@@ -27,6 +27,6 @@ export {
 export { createAbsolutePathSerializer } from './serializers';
 export { CA_CERT_PATH, ES_KEY_PATH, ES_CERT_PATH } from './certs';
 export { run, createFailError, createFlagError, combineErrors, isFailError, Flags } from './run';
-export { REPO_ROOT } from './constants';
+export { REPO_ROOT } from './repo_root';
 export { KbnClient } from './kbn_client';
 export * from './axios';

packages/kbn-dev-utils/src/index.ts

@@ -0,0 +1,59 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import Path from 'path';
+import Fs from 'fs';
+
+import loadJsonFile from 'load-json-file';
+
+const isKibanaDir = (dir: string) => {
+  try {
+    const path = Path.resolve(dir, 'package.json');
+    const json = loadJsonFile.sync(path);
+    if (json && typeof json === 'object' && 'name' in json && json.name === 'kibana') {
+      return true;
+    }
+  } catch (error) {
+    if (error && error.code === 'ENOENT') {
+      return false;
+    }
+
+    throw error;
+  }
+};
+
+// search for the kibana directory, since this file is moved around it might
+// not be where we think but should always be a relatively close parent
+// of this directory
+const startDir = Fs.realpathSync(__dirname);
+const { root: rootDir } = Path.parse(startDir);
+let cursor = startDir;
+while (true) {
+  if (isKibanaDir(cursor)) {
+    break;
+  }
+
+  const parent = Path.dirname(cursor);
+  if (parent === rootDir) {
+    throw new Error(`unable to find kibana directory from ${startDir}`);
+  }
+  cursor = parent;
+}
+
+export const REPO_ROOT = cursor;

packages/kbn-dev-utils/src/repo_root.ts

@@ -34,6 +34,8 @@
     "@types/tempy": "^0.2.0",
     "@types/wrap-ansi": "^2.0.15",
     "@types/write-pkg": "^3.1.0",
+    "@kbn/dev-utils": "1.0.0",
+    "@yarnpkg/lockfile": "^1.1.0",
     "babel-loader": "^8.0.6",
     "chalk": "^2.4.2",
     "cmd-shim": "^2.1.0",
@@ -48,6 +50,7 @@
     "indent-string": "^3.2.0",
     "lodash.clonedeepwith": "^4.5.0",
     "log-symbols": "^2.2.0",
+    "multimatch": "^4.0.0",
     "ncp": "^2.0.0",
     "ora": "^1.4.0",
     "prettier": "^1.19.1",

packages/kbn-pm/dist/index.js

@@ -47,6 +47,7 @@ function help() {
        -i, --include          Include only specified projects. If left unspecified, it defaults to including all projects.
        --oss                  Do not include the x-pack when running command.
        --skip-kibana-plugins  Filter all plugins in ./plugins and ../kibana-extra when running command.
+       --no-cache             Disable the bootstrap cache
   `);
 }
 
@@ -65,7 +66,10 @@ export async function run(argv: string[]) {
       h: 'help',
       i: 'include',
     },
-    boolean: ['prefer-offline', 'frozen-lockfile'],
+    default: {
+      cache: true,
+    },
+    boolean: ['prefer-offline', 'frozen-lockfile', 'cache'],
   });
 
   const args = options._;

packages/kbn-pm/package.json

@@ -29,6 +29,7 @@ import { Project } from '../utils/project';
 import { buildProjectGraph } from '../utils/projects';
 import { installInDir, runScriptInPackageStreaming, yarnWorkspacesInfo } from '../utils/scripts';
 import { BootstrapCommand } from './bootstrap';
+import { Kibana } from '../utils/kibana';
 
 const mockInstallInDir = installInDir as jest.Mock;
 const mockRunScriptInPackageStreaming = runScriptInPackageStreaming as jest.Mock;
@@ -107,6 +108,7 @@ test('handles dependencies of dependencies', async () => {
     ['bar', bar],
     ['baz', baz],
   ]);
+  const kbn = new Kibana(projects);
   const projectGraph = buildProjectGraph(projects);
 
   const logMock = jest.spyOn(console, 'log').mockImplementation(noop);
@@ -115,6 +117,7 @@ test('handles dependencies of dependencies', async () => {
     extraArgs: [],
     options: {},
     rootPath: '',
+    kbn,
   });
 
   expect(mockInstallInDir.mock.calls).toMatchSnapshot('install in dir');
@@ -142,6 +145,7 @@ test('does not run installer if no deps in package', async () => {
     ['kibana', kibana],
     ['bar', bar],
   ]);
+  const kbn = new Kibana(projects);
   const projectGraph = buildProjectGraph(projects);
 
   const logMock = jest.spyOn(console, 'log').mockImplementation(noop);
@@ -150,6 +154,7 @@ test('does not run installer if no deps in package', async () => {
     extraArgs: [],
     options: {},
     rootPath: '',
+    kbn,
   });
 
   expect(mockInstallInDir.mock.calls).toMatchSnapshot('install in dir');
@@ -167,6 +172,7 @@ test('handles "frozen-lockfile"', async () => {
   });
 
   const projects = new Map([['kibana', kibana]]);
+  const kbn = new Kibana(projects);
   const projectGraph = buildProjectGraph(projects);
 
   jest.spyOn(console, 'log').mockImplementation(noop);
@@ -177,6 +183,7 @@ test('handles "frozen-lockfile"', async () => {
       'frozen-lockfile': true,
     },
     rootPath: '',
+    kbn,
   });
 
   expect(mockInstallInDir.mock.calls).toMatchSnapshot('install in dir');
@@ -205,6 +212,7 @@ test('calls "kbn:bootstrap" scripts and links executables after installing deps'
     ['kibana', kibana],
     ['bar', bar],
   ]);
+  const kbn = new Kibana(projects);
   const projectGraph = buildProjectGraph(projects);
 
   jest.spyOn(console, 'log').mockImplementation(noop);
@@ -213,6 +221,7 @@ test('calls "kbn:bootstrap" scripts and links executables after installing deps'
     extraArgs: [],
     options: {},
     rootPath: '',
+    kbn,
   });
 
   expect(mockLinkProjectExecutables.mock.calls).toMatchSnapshot('link bins');

packages/kbn-pm/src/cli.ts

@@ -24,12 +24,14 @@ import { log } from '../utils/log';
 import { parallelizeBatches } from '../utils/parallelize';
 import { topologicallyBatchProjects } from '../utils/projects';
 import { ICommand } from './';
+import { getAllChecksums } from '../utils/project_checksums';
+import { BootstrapCacheFile } from '../utils/bootstrap_cache_file';
 
 export const BootstrapCommand: ICommand = {
   description: 'Install dependencies and crosslink projects',
   name: 'bootstrap',
 
-  async run(projects, projectGraph, { options }) {
+  async run(projects, projectGraph, { options, kbn }) {
     const batchedProjectsByWorkspace = topologicallyBatchProjects(projects, projectGraph, {
       batchByWorkspace: true,
     });
@@ -65,9 +67,18 @@ export const BootstrapCommand: ICommand = {
      * have to, as it will slow down the bootstrapping process.
      */
     log.write(chalk.bold('\nLinking executables completed, running `kbn:bootstrap` scripts\n'));
-    await parallelizeBatches(batchedProjects, async pkg => {
-      if (pkg.hasScript('kbn:bootstrap')) {
-        await pkg.runScriptStreaming('kbn:bootstrap');
+
+    const checksums = options.cache ? await getAllChecksums(kbn, log) : false;
+    await parallelizeBatches(batchedProjects, async project => {
+      if (project.hasScript('kbn:bootstrap')) {
+        const cacheFile = new BootstrapCacheFile(kbn, project, checksums);
+        if (cacheFile.isValid()) {
+          log.success(`[${project.name}] cache up to date`);
+        } else {
+          cacheFile.delete();
+          await project.runScriptStreaming('kbn:bootstrap');
+          cacheFile.write();
+        }
       }
     });