[kbn/optimizer] report limits with ci metrics

Author: spalger

packages/kbn-dev-utils/src/ci_stats_reporter/README.md

@@ -8,10 +8,23 @@ This class integrates with the `ciStats.trackBuild {}` Jenkins Pipeline function
 
 To create an instance of the reporter, import the class and call `CiStatsReporter.fromEnv(log)` (passing it a tooling log).
 
-#### `CiStatsReporter#metrics(metrics: Array<{ group: string, id: string, value: number }>)`
+#### `CiStatsReporter#metrics(metrics: Metric[])`
 
 Use this method to record metrics in the Kibana CI Stats service.
 
+```ts
+interface Metric {
+  group: string,
+  id: string,
+  value: number,
+  // optional limit, values which exceed the limit will fail PRs
+  limit?: number
+  // optional path, relative to the root of the repo, where config values
+  // are defined. Will be linked to in PRs which have overages.
+  limitConfigPath?: string
+}
+```
+
 Example:
 
 ```ts

packages/kbn-dev-utils/src/ci_stats_reporter/ci_stats_reporter.ts

@@ -29,7 +29,13 @@ interface Config {
   buildId: string;
 }
 
-export type CiStatsMetrics = Array<{ group: string; id: string; value: number }>;
+export type CiStatsMetrics = Array<{
+  group: string;
+  id: string;
+  value: number;
+  limit?: number;
+  limitConfigPath?: string;
+}>;
 
 function parseConfig(log: ToolingLog) {
   const configJson = process.env.KIBANA_CI_STATS_CONFIG;

packages/kbn-optimizer/limits.yml

@@ -0,0 +1,99 @@
+pageLoadAssetSize:
+  advancedSettings: 17_596
+  alerts: 96_936
+  apm: 50_340
+  apmOss: 8_996
+  beatsManagement: 455_715
+  bfetch: 31_874
+  Bundle:
+  canvas: 1_055_965
+  charts: 149_211
+  cloud: 11_076
+  console: 36_091
+  core: 681_949
+  crossClusterReplication: 55_408
+  dashboard: 588_912
+  dashboardEnhanced: 49_143
+  dashboardMode: 12_716
+  data: 1_415_843
+  dataEnhanced: 40_162
+  devTools: 28_637
+  discover: 94_982
+  discoverEnhanced: 34_904
+  embeddable: 302_688
+  embeddableEnhanced: 31_145
+  enterpriseSearch: 25_484
+  esUiShared: 314_930
+  expressions: 214_136
+  features: 21_211
+  fileUpload: 14_717
+  globalSearch: 33_548
+  globalSearchBar: 34_422
+  globalSearchProviders: 15_554
+  graph: 21_215
+  grokdebugger: 16_779
+  home: 32_372
+  indexLifecycleManagement: 97_032
+  indexManagement: 130_599
+  indexPatternManagement: 144_222
+  infra: 187_885
+  ingestManager: 528_995
+  ingestPipelines: 48_003
+  inputControlVis: 162_675
+  inspector: 138_711
+  kibanaLegacy: 97_711
+  kibanaReact: 132_073
+  kibanaUtils: 188_829
+  lens: 82_860
+  licenseManagement: 31_817
+  licensing: 28_693
+  lists: 173_697
+  logstash: 43_548
+  management: 36_112
+  maps: 173_393
+  mapsLegacy: 106_817
+  mapsLegacyLicensing: 10_214
+  ml: 64_944
+  monitoring: 233_047
+  navigation: 27_443
+  newsfeed: 27_628
+  observability: 58_642
+  painlessLab: 169_748
+  regionMap: 56_105
+  remoteClusters: 41_327
+  reporting: 173_418
+  rollup: 87_204
+  savedObjects: 98_518
+  savedObjectsManagement: 90_503
+  searchprofiler: 57_080
+  security: 172_250
+  securitySolution: 606_205
+  share: 86_784
+  snapshotRestore: 69_032
+  spaces: 357_173
+  telemetry: 81_832
+  telemetryManagementSection: 42_443
+  tileMap: 55_257
+  timelion: 19_920
+  transform: 31_007
+  triggers_actions_ui: 156_543
+  uiActions: 87_533
+  uiActionsEnhanced: 333_430
+  upgradeAssistant: 71_251
+  uptime: 30_833
+  urlDrilldown: 24_174
+  urlForwarding: 22_554
+  usageCollection: 29_762
+  visDefaultEditor: 40_178
+  visTypeMarkdown: 20_896
+  visTypeMetric: 32_820
+  visTypeTable: 84_706
+  visTypeTagcloud: 27_605
+  visTypeTimelion: 41_933
+  visTypeTimeseries: 145_195
+  visTypeVega: 142_183
+  visTypeVislib: 233_049
+  visTypeXy: 10_255
+  visualizations: 283_763
+  visualize: 47_384
+  watcher: 33_598

packages/kbn-optimizer/package.json

@@ -22,6 +22,7 @@
     "cpy": "^8.0.0",
     "core-js": "^3.6.5",
     "css-loader": "^3.4.2",
+    "dedent": "^0.7.0",
     "del": "^5.1.0",
     "execa": "^4.0.2",
     "file-loader": "^4.2.0",
@@ -38,6 +39,7 @@
     "postcss-loader": "^3.0.0",
     "raw-loader": "^3.1.0",
     "rxjs": "^6.5.5",
+    "js-yaml": "^3.14.0",
     "sass-loader": "^8.0.2",
     "source-map-support": "^0.5.19",
     "style-loader": "^1.1.3",

packages/kbn-optimizer/src/cli.ts

@@ -28,6 +28,7 @@ import { logOptimizerState } from './log_optimizer_state';
 import { OptimizerConfig } from './optimizer';
 import { reportOptimizerStats } from './report_optimizer_stats';
 import { runOptimizer } from './run_optimizer';
+import { validateLimitsForAllBundles } from './limits';
 
 run(
   async ({ log, flags }) => {
@@ -93,21 +94,31 @@ run(
       throw createFlagError('expected --filter to be one or more strings');
     }
 
+    const validateLimits = flags['validate-limits'] ?? false;
+    if (typeof validateLimits !== 'boolean') {
+      throw createFlagError('expected --validate-limits to have no value');
+    }
+
     const config = OptimizerConfig.create({
       repoRoot: REPO_ROOT,
       watch,
       maxWorkerCount,
-      oss,
+      oss: oss && !validateLimits,
       dist,
       cache,
-      examples,
+      examples: examples && !validateLimits,
       profileWebpack,
       extraPluginScanDirs,
       inspectWorkers,
       includeCoreBundle,
       filter,
     });
 
+    if (validateLimits) {
+      validateLimitsForAllBundles(log, config);
+      return;
+    }
+
     let update$ = runOptimizer(config);
 
     if (reportStats) {
@@ -134,6 +145,7 @@ run(
         'profile',
         'inspect-workers',
         'report-stats',
+        'validate-limits',
       ],
       string: ['workers', 'scan-dir', 'filter'],
       default: {
@@ -156,6 +168,7 @@ run(
         --scan-dir         add a directory to the list of directories scanned for plugins (specify as many times as necessary)
         --no-inspect-workers  when inspecting the parent process, don't inspect the workers
         --report-stats     attempt to report stats about this execution of the build to the kibana-ci-stats service using this name
+        --validate-limits  validate the limits.yml config to ensure that there are limits defined for every bundle
       `,
     },
   }

packages/kbn-optimizer/src/index.ts

@@ -22,3 +22,4 @@ export * from './run_optimizer';
 export * from './log_optimizer_state';
 export * from './report_optimizer_stats';
 export * from './node';
+export * from './limits';

packages/kbn-optimizer/src/integration_tests/__snapshots__/basic_optimization.test.ts.snap

@@ -27,7 +27,13 @@ import del from 'del';
 import { toArray, tap, filter } from 'rxjs/operators';
 import { REPO_ROOT } from '@kbn/utils';
 import { ToolingLog } from '@kbn/dev-utils';
-import { runOptimizer, OptimizerConfig, OptimizerUpdate, logOptimizerState } from '@kbn/optimizer';
+import {
+  runOptimizer,
+  OptimizerConfig,
+  OptimizerUpdate,
+  logOptimizerState,
+  readLimits,
+} from '@kbn/optimizer';
 
 const TMP_DIR = Path.resolve(__dirname, '../__fixtures__/__tmp__');
 const MOCK_REPO_SRC = Path.resolve(__dirname, '../__fixtures__/mock_repo');
@@ -72,6 +78,9 @@ it('builds expected bundles, saves bundle counts to metadata', async () => {
     dist: false,
   });
 
+  expect(config.limits).toEqual(readLimits());
+  (config as any).limits = '<Limits>';
+
   expect(config).toMatchSnapshot('OptimizerConfig');
 
   const msgs = await runOptimizer(config)

packages/kbn-optimizer/src/integration_tests/basic_optimization.test.ts

@@ -0,0 +1,64 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import Fs from 'fs';
+
+import dedent from 'dedent';
+import Yaml from 'js-yaml';
+import { createFailError, ToolingLog } from '@kbn/dev-utils';
+
+import { OptimizerConfig } from './optimizer';
+
+const diff = <T>(a: T[], b: T[]): T[] => a.filter((item) => !b.includes(item));
+
+export function readLimits() {
+  return Yaml.safeLoad(Fs.readFileSync(require.resolve('../limits.yml'), 'utf8'));
+}
+
+export function validateLimitsForAllBundles(log: ToolingLog, config: OptimizerConfig) {
+  const limitBundleIds = Object.keys(config.limits.pageLoadAssetSize);
+  const configBundleIds = config.bundles.map((b) => b.id);
+
+  const missingBundleIds = diff(configBundleIds, limitBundleIds);
+  const extraBundleIds = diff(limitBundleIds, configBundleIds);
+
+  const issues = [];
+  if (missingBundleIds.length) {
+    issues.push(`missing: ${missingBundleIds.join(', ')}`);
+  }
+  if (extraBundleIds.length) {
+    issues.push(`extra: ${extraBundleIds.join(', ')}`);
+  }
+  if (issues.length) {
+    throw createFailError(
+      dedent`
+        The limits defined in packages/kbn-optimizer/limits.yml are outdated. Please update
+        this file with a limit (in bytes) for every production bundle.
+
+          ${issues.join('\n          ')}
+
+        To validate your changes locally, run:
+
+          node scripts/build_kibana_platform_plugins.js --validate-limits
+      ` + '\n'
+    );
+  }
+
+  log.success('limits.yml file valid');
+}

packages/kbn-optimizer/src/limits.ts

@@ -22,6 +22,7 @@ jest.mock('./kibana_platform_plugins.ts');
 jest.mock('./get_plugin_bundles.ts');
 jest.mock('../common/theme_tags.ts');
 jest.mock('./filter_by_id.ts');
+jest.mock('../limits.ts');
 
 jest.mock('os', () => {
   const realOs = jest.requireActual('os');
@@ -385,6 +386,7 @@ describe('OptimizerConfig::create()', () => {
     .findKibanaPlatformPlugins;
   const getPluginBundles: jest.Mock = jest.requireMock('./get_plugin_bundles.ts').getPluginBundles;
   const filterById: jest.Mock = jest.requireMock('./filter_by_id.ts').filterById;
+  const readLimits: jest.Mock = jest.requireMock('../limits.ts').readLimits;
 
   beforeEach(() => {
     if ('mock' in OptimizerConfig.parseOptions) {
@@ -398,6 +400,7 @@ describe('OptimizerConfig::create()', () => {
     findKibanaPlatformPlugins.mockReturnValue(Symbol('new platform plugins'));
     getPluginBundles.mockReturnValue([Symbol('bundle1'), Symbol('bundle2')]);
     filterById.mockReturnValue(Symbol('filtered bundles'));
+    readLimits.mockReturnValue(Symbol('limits'));
 
     jest.spyOn(OptimizerConfig, 'parseOptions').mockImplementation((): {
       [key in keyof ParsedOptions]: any;
@@ -429,6 +432,7 @@ describe('OptimizerConfig::create()', () => {
         "cache": Symbol(parsed cache),
         "dist": Symbol(parsed dist),
         "inspectWorkers": Symbol(parsed inspect workers),
+        "limits": Symbol(limits),
         "maxWorkerCount": Symbol(parsed max worker count),
         "plugins": Symbol(new platform plugins),
         "profileWebpack": Symbol(parsed profile webpack),