fix: Prevent unbounded growth of transaction history

Compress transaction history down to 100 entries and truncate any
existing transaction historys to just 100 entries. This should fix
reports we've seen from production of extremely poor performance and
crashes caused by long transaction histories, and will also prevent
this problem from happening again.

Transaction history has been truncated because in the extreme cases,
it would be prohibitively expensive to compress. The downside is that
some state of how the transaction has changed may be lost. But this is
unlikely to impact users because we only show a limited number of
events from the transaction history in our UI, and these events are
more likely to be at the beginning of long transaction histories. Even
if a displayed event is lost, the impact on the UI is minimal (it's
only shown on the transaction details page under "Activity log", and
only for informational purposes).

For details on how the transaction compression works, and how it
prevents history size from growing unbouned, see MetaMask/core#4555

The transaction controller change has been applied using a patch. The
patch was generated from the core repository branch
`patch/transaction-controller-v32-compress-history`.

Relates to #9372

app/scripts/migrations/120.2.test.ts

@@ -0,0 +1,253 @@
+import { cloneDeep } from 'lodash';
+import { migrate, version } from './120.2';
+
+const sentryCaptureExceptionMock = jest.fn();
+
+global.sentry = {
+  captureException: sentryCaptureExceptionMock,
+};
+
+const oldVersion = 120.1;
+
+describe('migration #120.2', () => {
+  afterEach(() => {
+    jest.resetAllMocks();
+  });
+
+  it('updates the version metadata', async () => {
+    const oldStorage = {
+      meta: { version: oldVersion },
+      data: {},
+    };
+
+    const newStorage = await migrate(oldStorage);
+
+    expect(newStorage.meta).toStrictEqual({ version });
+  });
+
+  it('returns state unchanged if TransactionController state is missing', async () => {
+    const oldStorage = {
+      meta: { version: oldVersion },
+      data: {
+        PreferencesController: {},
+      },
+    };
+    const oldStorageDataClone = cloneDeep(oldStorage.data);
+
+    const newStorage = await migrate(oldStorage);
+
+    expect(newStorage.data).toStrictEqual(oldStorageDataClone);
+  });
+
+  it('reports error and returns state unchanged if TransactionController state is invalid', async () => {
+    const oldStorage = {
+      meta: { version: oldVersion },
+      data: {
+        PreferencesController: {},
+        TransactionController: 'invalid',
+      },
+    };
+    const oldStorageDataClone = cloneDeep(oldStorage.data);
+
+    const newStorage = await migrate(oldStorage);
+
+    expect(sentryCaptureExceptionMock).toHaveBeenCalledWith(
+      `Migration ${version}: Invalid TransactionController state of type 'string'`,
+    );
+    expect(newStorage.data).toStrictEqual(oldStorageDataClone);
+  });
+
+  it('returns state unchanged if transactions are missing', async () => {
+    const oldStorage = {
+      meta: { version: oldVersion },
+      data: {
+        PreferencesController: {},
+        TransactionController: {},
+      },
+    };
+    const oldStorageDataClone = cloneDeep(oldStorage.data);
+
+    const newStorage = await migrate(oldStorage);
+
+    expect(newStorage.data).toStrictEqual(oldStorageDataClone);
+  });
+
+  it('reports error and returns state unchanged if transactions property is invalid', async () => {
+    const oldStorage = {
+      meta: { version: oldVersion },
+      data: {
+        PreferencesController: {},
+        TransactionController: {
+          transactions: 'invalid',
+        },
+      },
+    };
+    const oldStorageDataClone = cloneDeep(oldStorage.data);
+
+    const newStorage = await migrate(oldStorage);
+
+    expect(sentryCaptureExceptionMock).toHaveBeenCalledWith(
+      `Migration ${version}: Invalid TransactionController transactions state of type 'string'`,
+    );
+    expect(newStorage.data).toStrictEqual(oldStorageDataClone);
+  });
+
+  it('reports error and returns state unchanged if there is an invalid transaction', async () => {
+    const oldStorage = {
+      meta: { version: oldVersion },
+      data: {
+        PreferencesController: {},
+        TransactionController: {
+          transactions: [
+            {}, // empty object is valid for the purposes of this migration
+            'invalid',
+            {},
+          ],
+        },
+      },
+    };
+    const oldStorageDataClone = cloneDeep(oldStorage.data);
+
+    const newStorage = await migrate(oldStorage);
+
+    expect(sentryCaptureExceptionMock).toHaveBeenCalledWith(
+      `Migration ${version}: Invalid transaction of type 'string'`,
+    );
+    expect(newStorage.data).toStrictEqual(oldStorageDataClone);
+  });
+
+  it('reports error and returns state unchanged if there is a transaction with an invalid history property', async () => {
+    const oldStorage = {
+      meta: { version: oldVersion },
+      data: {
+        PreferencesController: {},
+        TransactionController: {
+          transactions: [
+            {}, // empty object is valid for the purposes of this migration
+            {
+              history: 'invalid',
+            },
+            {},
+          ],
+        },
+      },
+    };
+    const oldStorageDataClone = cloneDeep(oldStorage.data);
+
+    const newStorage = await migrate(oldStorage);
+
+    expect(sentryCaptureExceptionMock).toHaveBeenCalledWith(
+      `Migration ${version}: Invalid transaction history of type 'string'`,
+    );
+    expect(newStorage.data).toStrictEqual(oldStorageDataClone);
+  });
+
+  it('returns state unchanged if there are no transactions', async () => {
+    const oldStorage = {
+      meta: { version: oldVersion },
+      data: {
+        PreferencesController: {},
+        TransactionController: {
+          transactions: [],
+        },
+      },
+    };
+    const oldStorageDataClone = cloneDeep(oldStorage.data);
+
+    const newStorage = await migrate(oldStorage);
+
+    expect(newStorage.data).toStrictEqual(oldStorageDataClone);
+  });
+
+  it('returns state unchanged if there are no transactions with history', async () => {
+    const oldStorage = {
+      meta: { version: oldVersion },
+      data: {
+        PreferencesController: {},
+        TransactionController: {
+          transactions: [{}, {}, {}],
+        },
+      },
+    };
+    const oldStorageDataClone = cloneDeep(oldStorage.data);
+
+    const newStorage = await migrate(oldStorage);
+
+    expect(newStorage.data).toStrictEqual(oldStorageDataClone);
+  });
+
+  it('returns state unchanged if there are no transactions with history exceeding max size', async () => {
+    const oldStorage = {
+      meta: { version: oldVersion },
+      data: {
+        PreferencesController: {},
+        TransactionController: {
+          transactions: [
+            {
+              history: [...Array(99).keys()],
+            },
+            {
+              history: [...Array(100).keys()],
+            },
+            {
+              history: [],
+            },
+          ],
+        },
+      },
+    };
+    const oldStorageDataClone = cloneDeep(oldStorage.data);
+
+    const newStorage = await migrate(oldStorage);
+
+    expect(newStorage.data).toStrictEqual(oldStorageDataClone);
+  });
+
+  it('trims histories exceeding max size', async () => {
+    const oldStorage = {
+      meta: { version: oldVersion },
+      data: {
+        PreferencesController: {},
+        TransactionController: {
+          transactions: [
+            {
+              history: [...Array(99).keys()],
+            },
+            {
+              history: [...Array(100).keys()],
+            },
+            {
+              history: [...Array(101).keys()],
+            },
+            {
+              history: [...Array(1000).keys()],
+            },
+          ],
+        },
+      },
+    };
+    const oldStorageDataClone = cloneDeep(oldStorage.data);
+
+    const newStorage = await migrate(oldStorage);
+
+    expect(newStorage.data).toStrictEqual({
+      ...oldStorageDataClone,
+      TransactionController: {
+        transactions: [
+          {
+            history: [...Array(99).keys()],
+          },
+          {
+            history: [...Array(100).keys()],
+          },
+          {
+            history: [...Array(100).keys()],
+          },
+          {
+            history: [...Array(100).keys()],
+          },
+        ],
+      },
+    });
+  });
+});

app/scripts/migrations/120.2.ts

@@ -0,0 +1,110 @@
+import { RuntimeObject, hasProperty, isObject } from '@metamask/utils';
+import { cloneDeep } from 'lodash';
+import log from 'loglevel';
+
+type VersionedData = {
+  meta: { version: number };
+  data: Record<string, unknown>;
+};
+
+export const version = 120.2;
+
+/**
+ * This migration trims the size of any large transaction histories. This will
+ * result in some loss of information, but the impact is minor. The lost data
+ * is only used in the "Activity log" on the transaction details page.
+ *
+ * @param originalVersionedData - Versioned MetaMask extension state, exactly
+ * what we persist to dist.
+ * @param originalVersionedData.meta - State metadata.
+ * @param originalVersionedData.meta.version - The current state version.
+ * @param originalVersionedData.data - The persisted MetaMask state, keyed by
+ * controller.
+ * @returns Updated versioned MetaMask extension state.
+ */
+export async function migrate(
+  originalVersionedData: VersionedData,
+): Promise<VersionedData> {
+  const versionedData = cloneDeep(originalVersionedData);
+  versionedData.meta.version = version;
+  transformState(versionedData.data);
+  return versionedData;
+}
+
+function transformState(state: Record<string, unknown>) {
+  if (!hasProperty(state, 'TransactionController')) {
+    log.warn(`Migration ${version}: Missing TransactionController state`);
+    return state;
+  } else if (!isObject(state.TransactionController)) {
+    global.sentry?.captureException(
+      `Migration ${version}: Invalid TransactionController state of type '${typeof state.TransactionController}'`,
+    );
+    return state;
+  }
+
+  const transactionControllerState = state.TransactionController;
+
+  if (!hasProperty(transactionControllerState, 'transactions')) {
+    log.warn(
+      `Migration ${version}: Missing TransactionController transactions`,
+    );
+    return state;
+  } else if (!Array.isArray(transactionControllerState.transactions)) {
+    global.sentry?.captureException(
+      `Migration ${version}: Invalid TransactionController transactions state of type '${typeof transactionControllerState.transactions}'`,
+    );
+    return state;
+  }
+
+  const validTransactions =
+    transactionControllerState.transactions.filter(isObject);
+  if (
+    transactionControllerState.transactions.length !== validTransactions.length
+  ) {
+    const invalidTransaction = transactionControllerState.transactions.find(
+      (transaction) => !isObject(transaction),
+    );
+    global.sentry?.captureException(
+      `Migration ${version}: Invalid transaction of type '${typeof invalidTransaction}'`,
+    );
+    return state;
+  }
+
+  const validHistoryTransactions = validTransactions.filter(
+    hasValidTransactionHistory,
+  );
+  if (validHistoryTransactions.length !== validTransactions.length) {
+    const invalidTransaction = validTransactions.find(
+      (transaction) => !hasValidTransactionHistory(transaction),
+    );
+    global.sentry?.captureException(
+      `Migration ${version}: Invalid transaction history of type '${typeof invalidTransaction?.history}'`,
+    );
+    return state;
+  }
+
+  for (const transaction of validHistoryTransactions) {
+    if (transaction.history && transaction.history.length > 100) {
+      transaction.history = transaction.history.slice(0, 100);
+    }
+  }
+
+  return state;
+}
+
+/**
+ * Check whether the given object has a valid `history` property, or no `history`
+ * property. We just check that it's an array, we don't validate the contents.
+ *
+ * @param transaction - The object to validate.
+ * @returns True if the given object was valid, false otherwise.
+ */
+function hasValidTransactionHistory(
+  transaction: RuntimeObject,
+): transaction is RuntimeObject & {
+  history: undefined | unknown[];
+} {
+  return (
+    !hasProperty(transaction, 'history') || Array.isArray(transaction.history)
+  );
+}

app/scripts/migrations/120.test.ts

@@ -1,6 +1,6 @@
 import { migrate, version } from './120';
 
-const oldVersion = 119;
+const oldVersion = 120.2;
 
 describe('migration #120', () => {
   afterEach(() => jest.resetAllMocks());

package.json

@@ -224,7 +224,7 @@
     "@trezor/schema-utils@npm:1.0.2": "patch:@trezor/schema-utils@npm%3A1.0.2#~/.yarn/patches/@trezor-schema-utils-npm-1.0.2-7dd48689b2.patch",
     "lavamoat-core@npm:^15.1.1": "patch:lavamoat-core@npm%3A15.1.1#~/.yarn/patches/lavamoat-core-npm-15.1.1-51fbe39988.patch",
     "@metamask/snaps-sdk": "^5.0.0",
-    "@metamask/transaction-controller": "^32.0.0",
+    "@metamask/transaction-controller": "patch:@metamask/transaction-controller@npm%3A32.0.0#~/.yarn/patches/@metamask-transaction-controller-npm-32.0.0-e23c2c3443.patch",
     "@babel/runtime@npm:^7.7.6": "patch:@babel/runtime@npm%3A7.24.0#~/.yarn/patches/@babel-runtime-npm-7.24.0-7eb1dd11a2.patch",
     "@babel/runtime@npm:^7.9.2": "patch:@babel/runtime@npm%3A7.24.0#~/.yarn/patches/@babel-runtime-npm-7.24.0-7eb1dd11a2.patch",
     "@babel/runtime@npm:^7.12.5": "patch:@babel/runtime@npm%3A7.24.0#~/.yarn/patches/@babel-runtime-npm-7.24.0-7eb1dd11a2.patch",
@@ -350,7 +350,7 @@
     "@metamask/snaps-rpc-methods": "^9.1.3",
     "@metamask/snaps-sdk": "^5.0.0",
     "@metamask/snaps-utils": "patch:@metamask/snaps-utils@npm%3A7.6.0#~/.yarn/patches/@metamask-snaps-utils-npm-7.6.0-5569b09766.patch",
-    "@metamask/transaction-controller": "^32.0.0",
+    "@metamask/transaction-controller": "patch:@metamask/transaction-controller@npm%3A32.0.0#~/.yarn/patches/@metamask-transaction-controller-npm-32.0.0-e23c2c3443.patch",
     "@metamask/user-operation-controller": "^10.0.0",
     "@metamask/utils": "^8.2.1",
     "@ngraveio/bc-ur": "^1.1.12",