[kbn/optimizer] report limits with ci metrics (#78205)

Co-authored-by: spalger <spalger@users.noreply.github.com>
Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

Author: 3 people

packages/kbn-dev-utils/src/ci_stats_reporter/README.md

@@ -8,10 +8,23 @@ This class integrates with the `ciStats.trackBuild {}` Jenkins Pipeline function
 
 To create an instance of the reporter, import the class and call `CiStatsReporter.fromEnv(log)` (passing it a tooling log).
 
-#### `CiStatsReporter#metrics(metrics: Array<{ group: string, id: string, value: number }>)`
+#### `CiStatsReporter#metrics(metrics: Metric[])`
 
 Use this method to record metrics in the Kibana CI Stats service.
 
+```ts
+interface Metric {
+  group: string,
+  id: string,
+  value: number,
+  // optional limit, values which exceed the limit will fail PRs
+  limit?: number
+  // optional path, relative to the root of the repo, where config values
+  // are defined. Will be linked to in PRs which have overages.
+  limitConfigPath?: string
+}
+```
+
 Example:
 
 ```ts

packages/kbn-dev-utils/src/ci_stats_reporter/ci_stats_reporter.ts

@@ -29,7 +29,13 @@ interface Config {
   buildId: string;
 }
 
-export type CiStatsMetrics = Array<{ group: string; id: string; value: number }>;
+export type CiStatsMetrics = Array<{
+  group: string;
+  id: string;
+  value: number;
+  limit?: number;
+  limitConfigPath?: string;
+}>;
 
 function parseConfig(log: ToolingLog) {
   const configJson = process.env.KIBANA_CI_STATS_CONFIG;

packages/kbn-optimizer/limits.yml

@@ -0,0 +1,100 @@
+pageLoadAssetSize:
+  advancedSettings: 27_596
+  alerts: 106_936
+  apm: 64_385
+  apmOss: 18_996
+  beatsManagement: 188_135
+  bfetch: 41_874
+  canvas: 1_066_647
+  charts: 159_211
+  cloud: 21_076
+  console: 46_091
+  core: 692_106
+  crossClusterReplication: 65_408
+  dashboard: 374_194
+  dashboardEnhanced: 65_646
+  dashboardMode: 22_716
+  data: 1_170_713
+  dataEnhanced: 50_420
+  devTools: 38_637
+  discover: 105_145
+  discoverEnhanced: 42_730
+  embeddable: 312_874
+  embeddableEnhanced: 41_145
+  enterpriseSearch: 35_741
+  esUiShared: 326_654
+  expressions: 224_136
+  features: 31_211
+  fileUpload: 24_717
+  globalSearch: 43_548
+  globalSearchBar: 62_888
+  globalSearchProviders: 25_554
+  graph: 31_504
+  grokdebugger: 26_779
+  home: 41_661
+  indexLifecycleManagement: 107_090
+  indexManagement: 140_608
+  indexPatternManagement: 154_222
+  infra: 197_873
+  ingestManager: 415_829
+  ingestPipelines: 58_003
+  inputControlVis: 172_675
+  inspector: 148_711
+  kibanaLegacy: 107_711
+  kibanaOverview: 56_279
+  kibanaReact: 161_921
+  kibanaUtils: 198_829
+  lens: 96_624
+  licenseManagement: 41_817
+  licensing: 39_008
+  lists: 183_665
+  logstash: 53_548
+  management: 46_112
+  maps: 183_610
+  mapsLegacy: 116_817
+  mapsLegacyLicensing: 20_214
+  ml: 82_187
+  monitoring: 268_612
+  navigation: 37_269
+  newsfeed: 42_228
+  observability: 89_709
+  painlessLab: 179_748
+  regionMap: 66_098
+  remoteClusters: 51_327
+  reporting: 183_418
+  rollup: 97_204
+  savedObjects: 108_518
+  savedObjectsManagement: 100_503
+  searchprofiler: 67_080
+  security: 189_428
+  securityOss: 30_806
+  securitySolution: 622_387
+  share: 99_061
+  snapshotRestore: 79_032
+  spaces: 387_915
+  telemetry: 91_832
+  telemetryManagementSection: 52_443
+  tileMap: 65_337
+  timelion: 29_920
+  transform: 41_007
+  triggersActionsUi: 170_001
+  uiActions: 97_717
+  uiActionsEnhanced: 349_511
+  upgradeAssistant: 81_241
+  uptime: 40_825
+  urlDrilldown: 34_174
+  urlForwarding: 32_579
+  usageCollection: 39_762
+  visDefaultEditor: 50_178
+  visTypeMarkdown: 30_896
+  visTypeMetric: 42_790
+  visTypeTable: 94_934
+  visTypeTagcloud: 37_575
+  visTypeTimelion: 51_933
+  visTypeTimeseries: 155_203
+  visTypeVega: 153_573
+  visTypeVislib: 242_838
+  visTypeXy: 20_255
+  visualizations: 295_025
+  visualize: 57_431
+  watcher: 43_598

packages/kbn-optimizer/package.json

@@ -22,6 +22,7 @@
     "cpy": "^8.0.0",
     "core-js": "^3.6.5",
     "css-loader": "^3.4.2",
+    "dedent": "^0.7.0",
     "del": "^5.1.0",
     "execa": "^4.0.2",
     "file-loader": "^4.2.0",
@@ -38,6 +39,7 @@
     "postcss-loader": "^3.0.0",
     "raw-loader": "^3.1.0",
     "rxjs": "^6.5.5",
+    "js-yaml": "^3.14.0",
     "sass-loader": "^8.0.2",
     "source-map-support": "^0.5.19",
     "style-loader": "^1.1.3",

packages/kbn-optimizer/src/cli.ts

@@ -28,6 +28,7 @@ import { logOptimizerState } from './log_optimizer_state';
 import { OptimizerConfig } from './optimizer';
 import { reportOptimizerStats } from './report_optimizer_stats';
 import { runOptimizer } from './run_optimizer';
+import { validateLimitsForAllBundles, updateBundleLimits } from './limits';
 
 run(
   async ({ log, flags }) => {
@@ -93,21 +94,36 @@ run(
       throw createFlagError('expected --filter to be one or more strings');
     }
 
+    const validateLimits = flags['validate-limits'] ?? false;
+    if (typeof validateLimits !== 'boolean') {
+      throw createFlagError('expected --validate-limits to have no value');
+    }
+
+    const updateLimits = flags['update-limits'] ?? false;
+    if (typeof updateLimits !== 'boolean') {
+      throw createFlagError('expected --update-limits to have no value');
+    }
+
     const config = OptimizerConfig.create({
       repoRoot: REPO_ROOT,
       watch,
       maxWorkerCount,
-      oss,
-      dist,
+      oss: oss && !(validateLimits || updateLimits),
+      dist: dist || updateLimits,
       cache,
-      examples,
+      examples: examples && !(validateLimits || updateLimits),
       profileWebpack,
       extraPluginScanDirs,
       inspectWorkers,
       includeCoreBundle,
       filter,
     });
 
+    if (validateLimits) {
+      validateLimitsForAllBundles(log, config);
+      return;
+    }
+
     let update$ = runOptimizer(config);
 
     if (reportStats) {
@@ -121,6 +137,10 @@ run(
     }
 
     await update$.pipe(logOptimizerState(log, config)).toPromise();
+
+    if (updateLimits) {
+      updateBundleLimits(log, config);
+    }
   },
   {
     flags: {
@@ -134,6 +154,8 @@ run(
         'profile',
         'inspect-workers',
         'report-stats',
+        'validate-limits',
+        'update-limits',
       ],
       string: ['workers', 'scan-dir', 'filter'],
       default: {
@@ -152,10 +174,12 @@ run(
         --no-cache         disable the cache
         --filter           comma-separated list of bundle id filters, results from multiple flags are merged, * and ! are supported
         --no-examples      don't build the example plugins
-        --dist             create bundles that are suitable for inclusion in the Kibana distributable
+        --dist             create bundles that are suitable for inclusion in the Kibana distributable, enabled when running with --update-limits
         --scan-dir         add a directory to the list of directories scanned for plugins (specify as many times as necessary)
         --no-inspect-workers  when inspecting the parent process, don't inspect the workers
         --report-stats     attempt to report stats about this execution of the build to the kibana-ci-stats service using this name
+        --validate-limits  validate the limits.yml config to ensure that there are limits defined for every bundle
+        --update-limits    run a build and rewrite the limits file to include the current bundle sizes +5kb
       `,
     },
   }

packages/kbn-optimizer/src/index.ts

@@ -22,3 +22,4 @@ export * from './run_optimizer';
 export * from './log_optimizer_state';
 export * from './report_optimizer_stats';
 export * from './node';
+export * from './limits';

packages/kbn-optimizer/src/integration_tests/__snapshots__/basic_optimization.test.ts.snap

@@ -27,7 +27,13 @@ import del from 'del';
 import { toArray, tap, filter } from 'rxjs/operators';
 import { REPO_ROOT } from '@kbn/utils';
 import { ToolingLog } from '@kbn/dev-utils';
-import { runOptimizer, OptimizerConfig, OptimizerUpdate, logOptimizerState } from '@kbn/optimizer';
+import {
+  runOptimizer,
+  OptimizerConfig,
+  OptimizerUpdate,
+  logOptimizerState,
+  readLimits,
+} from '@kbn/optimizer';
 
 const TMP_DIR = Path.resolve(__dirname, '../__fixtures__/__tmp__');
 const MOCK_REPO_SRC = Path.resolve(__dirname, '../__fixtures__/mock_repo');
@@ -72,6 +78,9 @@ it('builds expected bundles, saves bundle counts to metadata', async () => {
     dist: false,
   });
 
+  expect(config.limits).toEqual(readLimits());
+  (config as any).limits = '<Limits>';
+
   expect(config).toMatchSnapshot('OptimizerConfig');
 
   const msgs = await runOptimizer(config)

packages/kbn-optimizer/src/integration_tests/basic_optimization.test.ts

@@ -0,0 +1,83 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import Fs from 'fs';
+
+import dedent from 'dedent';
+import Yaml from 'js-yaml';
+import { createFailError, ToolingLog } from '@kbn/dev-utils';
+
+import { OptimizerConfig, getMetrics } from './optimizer';
+
+const LIMITS_PATH = require.resolve('../limits.yml');
+const DEFAULT_BUDGET = 15000;
+
+const diff = <T>(a: T[], b: T[]): T[] => a.filter((item) => !b.includes(item));
+
+export function readLimits() {
+  return Yaml.safeLoad(Fs.readFileSync(LIMITS_PATH, 'utf8'));
+}
+
+export function validateLimitsForAllBundles(log: ToolingLog, config: OptimizerConfig) {
+  const limitBundleIds = Object.keys(config.limits.pageLoadAssetSize);
+  const configBundleIds = config.bundles.map((b) => b.id);
+
+  const missingBundleIds = diff(configBundleIds, limitBundleIds);
+  const extraBundleIds = diff(limitBundleIds, configBundleIds);
+
+  const issues = [];
+  if (missingBundleIds.length) {
+    issues.push(`missing: ${missingBundleIds.join(', ')}`);
+  }
+  if (extraBundleIds.length) {
+    issues.push(`extra: ${extraBundleIds.join(', ')}`);
+  }
+  if (issues.length) {
+    throw createFailError(
+      dedent`
+        The limits defined in packages/kbn-optimizer/limits.yml are outdated. Please update
+        this file with a limit (in bytes) for every production bundle.
+
+          ${issues.join('\n          ')}
+
+        To validate your changes locally, run:
+
+          node scripts/build_kibana_platform_plugins.js --validate-limits
+      ` + '\n'
+    );
+  }
+
+  log.success('limits.yml file valid');
+}
+
+export function updateBundleLimits(log: ToolingLog, config: OptimizerConfig) {
+  const metrics = getMetrics(log, config);
+
+  const number = (input: number) => input.toLocaleString('en').split(',').join('_');
+
+  let yaml = `pageLoadAssetSize:\n`;
+  for (const metric of metrics.sort((a, b) => a.id.localeCompare(b.id))) {
+    if (metric.group === 'page load bundle size') {
+      yaml += `  ${metric.id}: ${number(metric.value + DEFAULT_BUDGET)}\n`;
+    }
+  }
+
+  Fs.writeFileSync(LIMITS_PATH, yaml);
+  log.success(`wrote updated limits to ${LIMITS_PATH}`);
+}