👷 [mob-session] make test-performances more resilient

scripts/lib/computeBundleSize.ts

@@ -79,15 +79,3 @@ export function calculateBundleSizes(): BundleSizes {
 
   return bundleSizes
 }
-
-export function formatSize(bytes: number | null): string {
-  if (bytes === null) {
-    return 'N/A'
-  }
-
-  if (bytes < 1024) {
-    return `${Math.round(bytes)} B`
-  }
-
-  return `${(bytes / 1024).toFixed(2)} KiB`
-}

scripts/lib/executionUtils.ts

@@ -39,6 +39,25 @@ export function printLog(...params: any[]): void {
   console.log(greenColor, ...params, resetColor)
 }
 
+export function formatSize(bytes: number | null, { includeSign = false } = {}): string {
+  if (bytes === null) {
+    return 'N/A'
+  }
+
+  const sign = includeSign && bytes > 0 ? '+' : ''
+
+  if (bytes < 1024) {
+    return `${sign}${Math.round(bytes)} B`
+  }
+
+  return `${sign}${(bytes / 1024).toFixed(2)} KiB`
+}
+
+export function formatPercentage(percentage: number, { includeSign = false } = {}): string {
+  const sign = includeSign && percentage > 0 ? '+' : ''
+  return `${sign}${(percentage * 100).toFixed(2)}%`
+}
+
 /**
  * Find an error of type T in the provided error or its causes.
  */

scripts/lib/gitUtils.ts

@@ -66,16 +66,6 @@ export async function createGitHubRelease({ version, body }: GitHubReleaseParams
   })
 }
 
-export async function getPrComments(prNumber: number): Promise<Array<{ id: number; body: string }>> {
-  using readToken = getGithubReadToken()
-  const response = await callGitHubApi<Array<{ id: number; body: string }>>(
-    'GET',
-    `issues/${prNumber}/comments`,
-    readToken
-  )
-  return response
-}
-
 export function createPullRequest(mainBranch: string) {
   using token = getGithubPullRequestToken()
   command`gh auth login --with-token`.withInput(token.value).run()

scripts/performance/index.ts

@@ -1,26 +1,24 @@
-import { runMain } from '../lib/executionUtils.ts'
-import { calculateBundleSizes } from '../lib/computeBundleSize.ts'
-import { reportAsPrComment } from './lib/reportAsAPrComment.ts'
-import { reportToDatadog } from './lib/reportToDatadog.ts'
-import { computeCpuPerformance } from './lib/computeCpuPerformance.ts'
-import { computeMemoryPerformance } from './lib/computeMemoryPerformance.ts'
-
-interface UncompressedBundleSizes {
-  [key: string]: number
-}
+import { printLog, runMain } from '../lib/executionUtils.ts'
+import { fetchPR, getLastCommonCommit, LOCAL_BRANCH } from '../lib/gitUtils.ts'
+import { PrComment } from './lib/reportAsAPrComment.ts'
+import { computeAndReportMemoryPerformance } from './lib/memoryPerformance.ts'
+import { computeAndReportBundleSizes } from './lib/bundleSizes.ts'
+import { computeAndReportCpuPerformance } from './lib/cpuPerformance.ts'
 
 runMain(async () => {
-  const localBundleSizes = extractUncompressedBundleSizes(calculateBundleSizes())
-  const localMemoryPerformance = await computeMemoryPerformance()
-  await computeCpuPerformance()
-  await reportToDatadog(localMemoryPerformance, 'memoryPerformance')
-  await reportToDatadog(localBundleSizes, 'bundleSizes')
-  await reportAsPrComment(localBundleSizes, localMemoryPerformance)
-})
+  const pr = await fetchPR(LOCAL_BRANCH!)
+  if (!pr) {
+    throw new Error('No pull requests found for the branch')
+  }
+  const prComment = new PrComment(pr.number)
+  const lastCommonCommit = getLastCommonCommit(pr.base.ref)
+
+  printLog('Bundle sizes...')
+  await computeAndReportBundleSizes(lastCommonCommit, prComment)
 
-// keep compatibility with the logs and PR comment format
-function extractUncompressedBundleSizes(
-  bundleSizes: Record<string, { uncompressed: number }>
-): UncompressedBundleSizes {
-  return Object.fromEntries(Object.entries(bundleSizes).map(([key, size]) => [key, size.uncompressed]))
-}
+  printLog('Memory performance...')
+  await computeAndReportMemoryPerformance(lastCommonCommit, prComment)
+
+  printLog('CPU performance...')
+  await computeAndReportCpuPerformance(lastCommonCommit, prComment)
+})

scripts/performance/lib/bundleSizes.ts

@@ -0,0 +1,90 @@
+import { formatPercentage, formatSize } from '../../lib/executionUtils.ts'
+import { calculateBundleSizes } from '../../lib/computeBundleSize.ts'
+import type { PerformanceMetric } from './fetchPerformanceMetrics.ts'
+import { fetchPerformanceMetrics } from './fetchPerformanceMetrics.ts'
+import { markdownArray, type PrComment } from './reportAsAPrComment.ts'
+import { reportToDatadog } from './reportToDatadog.ts'
+
+// The value is set to 5% as it's around 10 times the average value for small PRs.
+const SIZE_INCREASE_THRESHOLD = 5
+
+export async function computeAndReportBundleSizes(lastCommonCommit: string, prComment: PrComment) {
+  let localBundleSizes: PerformanceMetric[]
+  let baseBundleSizes: PerformanceMetric[]
+  try {
+    localBundleSizes = extractUncompressedBundleSizes(calculateBundleSizes())
+    baseBundleSizes = await fetchPerformanceMetrics(
+      'bundle',
+      localBundleSizes.map((bundleSize) => bundleSize.name),
+      lastCommonCommit
+    )
+  } catch (e) {
+    await prComment.setBundleSizes('Error computing bundle sizes')
+    throw e
+  }
+
+  await reportToDatadog({
+    message: 'Browser SDK bundles sizes',
+    bundle_sizes: Object.fromEntries(localBundleSizes.map(({ name, value }) => [name, value])),
+  })
+
+  await prComment.setBundleSizes(
+    formatBundleSizes({
+      baseBundleSizes,
+      localBundleSizes,
+    })
+  )
+}
+
+function extractUncompressedBundleSizes(bundleSizes: Record<string, { uncompressed: number }>): PerformanceMetric[] {
+  return Object.entries(bundleSizes).map(([key, size]) => ({ name: key, value: size.uncompressed }))
+}
+
+export function formatBundleSizes({
+  baseBundleSizes,
+  localBundleSizes,
+}: {
+  baseBundleSizes: PerformanceMetric[]
+  localBundleSizes: PerformanceMetric[]
+}) {
+  let highIncreaseDetected = false
+  let message = markdownArray({
+    headers: ['📦 Bundle Name', 'Base Size', 'Local Size', '𝚫', '𝚫%', 'Status'],
+    rows: localBundleSizes.map((localBundleSize) => {
+      const baseBundleSize = baseBundleSizes.find((baseBundleSize) => baseBundleSize.name === localBundleSize.name)
+
+      if (!baseBundleSize) {
+        return [formatBundleName(localBundleSize.name), 'N/A', formatSize(localBundleSize.value), 'N/A', 'N/A', 'N/A']
+      }
+
+      const percentageChange = (localBundleSize.value - baseBundleSize.value) / baseBundleSize.value
+
+      let status = '✅'
+      if (percentageChange > SIZE_INCREASE_THRESHOLD) {
+        status = '⚠️'
+        highIncreaseDetected = true
+      }
+      return [
+        formatBundleName(localBundleSize.name),
+        formatSize(baseBundleSize.value),
+        formatSize(localBundleSize.value),
+        formatSize(localBundleSize.value - baseBundleSize.value, { includeSign: true }),
+        formatPercentage(percentageChange, { includeSign: true }),
+        status,
+      ]
+    }),
+  })
+
+  if (highIncreaseDetected) {
+    message += `\n⚠️ The increase is particularly high and exceeds ${SIZE_INCREASE_THRESHOLD}%. Please check the changes.`
+  }
+
+  return message
+}
+
+function formatBundleName(bundleName: string): string {
+  return bundleName
+    .split('_')
+    .map((word) => word.charAt(0).toUpperCase() + word.slice(1))
+    .join(' ')
+}

scripts/performance/lib/constants.ts

@@ -0,0 +1,43 @@
+export interface Test {
+  name: string
+  button: string
+  property: string
+}
+
+export const TESTS: Test[] = [
+  {
+    name: 'RUM - add global context',
+    button: '#rum-add-global-context',
+    property: 'addglobalcontext',
+  },
+  {
+    name: 'RUM - add action',
+    button: '#rum-add-action',
+    property: 'addaction',
+  },
+  {
+    name: 'RUM - add error',
+    button: '#rum-add-error',
+    property: 'adderror',
+  },
+  {
+    name: 'RUM - add timing',
+    button: '#rum-add-timing',
+    property: 'addtiming',
+  },
+  {
+    name: 'RUM - start view',
+    button: '#rum-start-view',
+    property: 'startview',
+  },
+  {
+    name: 'RUM - start/stop session replay recording',
+    button: '#rum-start-stop-session-replay-recording',
+    property: 'startstopsessionreplayrecording',
+  },
+  {
+    name: 'Logs - log message',
+    button: '#logs-log-message',
+    property: 'logmessage',
+  },
+]

scripts/performance/lib/computeCpuPerformance.ts → scripts/performance/lib/cpuPerformance.ts

@@ -1,13 +1,18 @@
-import { fetchHandlingError, timeout } from '../../lib/executionUtils.ts'
+import { fetchHandlingError, formatPercentage, timeout } from '../../lib/executionUtils.ts'
 import { getOrg2ApiKey, getOrg2AppKey } from '../../lib/secrets.ts'
 import { fetchPR, LOCAL_BRANCH } from '../../lib/gitUtils.ts'
-import { LOCAL_COMMIT_SHA } from './reportAsAPrComment.ts'
+import type { PrComment } from './reportAsAPrComment.ts'
+import { markdownArray } from './reportAsAPrComment.ts'
+import type { PerformanceMetric } from './fetchPerformanceMetrics.ts'
+import { fetchPerformanceMetrics } from './fetchPerformanceMetrics.ts'
+import { TESTS } from './constants.ts'
 
 const API_KEY = getOrg2ApiKey()
 const APP_KEY = getOrg2AppKey()
 const TIMEOUT_IN_MS = 15000
 const TEST_PUBLIC_ID = 'vcg-7rk-5av'
 const RETRIES_NUMBER = 6
+const LOCAL_COMMIT_SHA = process.env.CI_COMMIT_SHORT_SHA
 
 interface SyntheticsTestResult {
   results: Array<{
@@ -20,13 +25,41 @@ interface SyntheticsTestStatus {
   status?: number
 }
 
-export async function computeCpuPerformance(): Promise<void> {
+export async function computeAndReportCpuPerformance(lastCommonCommit: string, prComment: PrComment) {
+  let baseCpuPerformances: PerformanceMetric[]
+  let localCpuPerformances: PerformanceMetric[]
+  try {
+    localCpuPerformances = await computeCpuPerformance()
+    baseCpuPerformances = await fetchPerformanceMetrics(
+      'cpu',
+      localCpuPerformances.map((cpuPerformance) => cpuPerformance.name),
+      lastCommonCommit
+    )
+  } catch (error) {
+    await prComment.setCpuPerformance('Error computing CPU performance')
+    throw error
+  }
+
+  await prComment.setCpuPerformance(
+    formatCpuPerformance({
+      baseCpuPerformances,
+      localCpuPerformances,
+    })
+  )
+}
+
+async function computeCpuPerformance(): Promise<PerformanceMetric[]> {
   const pr = LOCAL_BRANCH ? await fetchPR(LOCAL_BRANCH) : null
   const commitSha = LOCAL_COMMIT_SHA || ''
   const resultId = pr
     ? await triggerSyntheticsTest(pr.number.toString(), commitSha)
     : await triggerSyntheticsTest('', commitSha)
   await waitForSyntheticsTestToFinish(resultId, RETRIES_NUMBER)
+  return fetchPerformanceMetrics(
+    'cpu',
+    TESTS.map((test) => test.property),
+    LOCAL_COMMIT_SHA || ''
+  )
 }
 
 async function triggerSyntheticsTest(prNumber: string, commitId: string): Promise<string> {
@@ -76,3 +109,33 @@ async function waitForSyntheticsTestToFinish(resultId: string, retriesNumber: nu
   }
   throw new Error('Synthetics test did not finish within the specified number of retries')
 }
+
+function formatCpuPerformance({
+  baseCpuPerformances,
+  localCpuPerformances,
+}: {
+  baseCpuPerformances: PerformanceMetric[]
+  localCpuPerformances: PerformanceMetric[]
+}) {
+  return markdownArray({
+    headers: ['Action Name', 'Base CPU Time (ms)', 'Local CPU Time (ms)', '𝚫 (%)'],
+    rows: localCpuPerformances.map((localCpuPerformance) => {
+      const baseCpuPerformance = baseCpuPerformances.find(
+        (baseCpuPerformance) => baseCpuPerformance.name === localCpuPerformance.name
+      )
+
+      if (!baseCpuPerformance) {
+        return [localCpuPerformance.name, 'N/A', String(localCpuPerformance.value), 'N/A']
+      }
+
+      return [
+        TESTS.find((test) => test.property === localCpuPerformance.name)!.name,
+        String(baseCpuPerformance.value),
+        String(localCpuPerformance.value),
+        formatPercentage((localCpuPerformance.value - baseCpuPerformance.value) / baseCpuPerformance.value, {
+          includeSign: true,
+        }),
+      ]
+    }),
+  })
+}

scripts/performance/lib/fetchPerformanceMetrics.ts

@@ -3,22 +3,28 @@ import { fetchHandlingError } from '../../lib/executionUtils.ts'
 
 const ONE_DAY_IN_SECOND = 24 * 60 * 60
 
-interface Metric {
-  name: string
-  value: number | null
-}
-
 interface DatadogResponse {
   series?: Array<{
     pointlist?: Array<[number, number]>
   }>
 }
 
-export function fetchPerformanceMetrics(type: string, names: string[], commitId: string): Promise<Metric[]> {
-  return Promise.all(names.map((name) => fetchMetric(type, name, commitId)))
+export interface PerformanceMetric {
+  name: string
+  value: number
 }
 
-async function fetchMetric(type: string, name: string, commitId: string): Promise<Metric> {
+export async function fetchPerformanceMetrics(
+  type: string,
+  names: string[],
+  commitId: string
+): Promise<PerformanceMetric[]> {
+  return (await Promise.all(names.map((name) => fetchMetric(type, name, commitId)))).filter(
+    (metric): metric is PerformanceMetric => !!metric
+  )
+}
+
+async function fetchMetric(type: string, name: string, commitId: string): Promise<PerformanceMetric | undefined> {
   const now = Math.floor(Date.now() / 1000)
   const date = now - 30 * ONE_DAY_IN_SECOND
   let query = ''
@@ -50,8 +56,4 @@ async function fetchMetric(type: string, name: string, commitId: string): Promis
       value: data.series[0].pointlist[0][1],
     }
   }
-  return {
-    name,
-    value: null,
-  }
 }