fix: sanitize event observer strings (#2361)

* chore: test

* fix: manual event server

* fix: tests

* fix: remove extras

* fix: extra import

* fix: test

Author: rafaelcr

src/datastore/helpers.ts

@@ -1363,6 +1363,10 @@ export function newReOrgUpdatedEntities(): ReOrgUpdatedEntities {
   };
 }
 
+export function removeNullBytes(str: string): string {
+  return str.replace(/\x00/g, '');
+}
+
 /**
  * Priority queue for parallel Postgres write query execution. This helps performance because it
  * parallelizes the work postgres.js has to do when serializing JS types to PG types.

src/datastore/pg-write-store.ts

@@ -78,6 +78,7 @@ import {
   validateZonefileHash,
   newReOrgUpdatedEntities,
   PgWriteQueue,
+  removeNullBytes,
 } from './helpers';
 import { PgNotifier } from './pg-notifier';
 import { MIGRATIONS_DIR, PgStore } from './pg-store';
@@ -203,6 +204,13 @@ export class PgWriteStore extends PgStore {
   }
 
   async storeRawEventRequest(eventPath: string, payload: any): Promise<void> {
+    if (eventPath === '/new_block' && typeof payload === 'object') {
+      for (const tx of payload.transactions) {
+        if ('vm_error' in tx && tx.vm_error) {
+          tx.vm_error = removeNullBytes(tx.vm_error);
+        }
+      }
+    }
     await this.sqlWriteTransaction(async sql => {
       const insertResult = await sql<
         {
@@ -2028,7 +2036,9 @@ export class PgWriteStore extends PgStore {
       token_transfer_memo: tx.token_transfer_memo ?? null,
       smart_contract_clarity_version: tx.smart_contract_clarity_version ?? null,
       smart_contract_contract_id: tx.smart_contract_contract_id ?? null,
-      smart_contract_source_code: tx.smart_contract_source_code ?? null,
+      smart_contract_source_code: tx.smart_contract_source_code
+        ? removeNullBytes(tx.smart_contract_source_code)
+        : null,
       contract_call_contract_id: tx.contract_call_contract_id ?? null,
       contract_call_function_name: tx.contract_call_function_name ?? null,
       contract_call_function_args: tx.contract_call_function_args ?? null,
@@ -2051,7 +2061,7 @@ export class PgWriteStore extends PgStore {
       execution_cost_runtime: tx.execution_cost_runtime,
       execution_cost_write_count: tx.execution_cost_write_count,
       execution_cost_write_length: tx.execution_cost_write_length,
-      vm_error: tx.vm_error ?? null,
+      vm_error: tx.vm_error ? removeNullBytes(tx.vm_error) : null,
     }));
 
     let count = 0;

tests/api/parse-db-tx.test.ts

@@ -1,12 +1,77 @@
+import { migrate } from '../utils/test-helpers';
+import { importEventsFromTsv } from '../../src/event-replay/event-replay';
 import { decodeClarityValueList } from 'stacks-encoding-native-js';
+import * as fs from 'fs';
+import * as readline from 'readline';
+import { PgSqlClient, timeout } from '@hirosystems/api-toolkit';
+import { ChainID } from '@stacks/common';
+import { ApiServer, startApiServer } from '../../src/api/init';
+import { PgWriteStore } from '../../src/datastore/pg-write-store';
+import { EventStreamServer, startEventServer } from '../../src/event-stream/event-server';
+import { httpPostRequest } from '../../src/helpers';
 
-test('buggy parsing of contract-call args', () => {
-  // Contract-call args from tx 0xb066874942e97d6e7ecfedb999a788edf7fbdbe51ab4b172ea05e8ede9b0ae9c
-  const contractCallArgsWithEmptyList =
-    '000000030c00000007066865696768740100000000000000000000000000005cc20b6d65726b6c652d726f6f7402000000201302a1b1fa53c11ac01ce480937dcd1ef550b4799070878e20b3b97bb99b58ec056e62697473020000000424961417056e6f6e6365020000000434a32a2106706172656e740200000020d0c9ebf9784f0670eb9bfac963114a02cff58bdc2669000000000000000000000974696d657374616d70020000000452b003610776657273696f6e0200000004040000200c0000000403696e730b000000010c00000003086f7574706f696e740c0000000204686173680200000020e85fb1e809db705121d92f3ec7f9ea4bafd065a8651408f507223efa8ede146605696e64657802000000040300000009736372697074536967020000006b48304502210096638a2336c383a99b009025534ae00f0bb689888c3898e6723497324c92806802205a3eee656775961780676f48f2fc1514142cadcd49efe1893d75517ab47a7f620121022a0160b2ed13b803ddca6f6f04606f56dfadee571d683725c7a58d0ed199de790873657175656e63650200000004fdffffff086c6f636b74696d65020000000400000000046f7574730b000000000776657273696f6e0200000004010000000c00000003066861736865730b000000000a747265652d646570746801000000000000000000000000000000080874782d696e64657801ffffffffffffffffffffffffffffffff';
-  const fnArgs = Buffer.from(contractCallArgsWithEmptyList, 'hex');
-  const clarityVals = decodeClarityValueList(fnArgs);
-  clarityVals.map(c => {
-    expect(c.hex).not.toBe('');
+describe('transaction parsing', () => {
+  test('buggy parsing of contract-call args', () => {
+    // Contract-call args from tx 0xb066874942e97d6e7ecfedb999a788edf7fbdbe51ab4b172ea05e8ede9b0ae9c
+    const contractCallArgsWithEmptyList =
+      '000000030c00000007066865696768740100000000000000000000000000005cc20b6d65726b6c652d726f6f7402000000201302a1b1fa53c11ac01ce480937dcd1ef550b4799070878e20b3b97bb99b58ec056e62697473020000000424961417056e6f6e6365020000000434a32a2106706172656e740200000020d0c9ebf9784f0670eb9bfac963114a02cff58bdc2669000000000000000000000974696d657374616d70020000000452b003610776657273696f6e0200000004040000200c0000000403696e730b000000010c00000003086f7574706f696e740c0000000204686173680200000020e85fb1e809db705121d92f3ec7f9ea4bafd065a8651408f507223efa8ede146605696e64657802000000040300000009736372697074536967020000006b48304502210096638a2336c383a99b009025534ae00f0bb689888c3898e6723497324c92806802205a3eee656775961780676f48f2fc1514142cadcd49efe1893d75517ab47a7f620121022a0160b2ed13b803ddca6f6f04606f56dfadee571d683725c7a58d0ed199de790873657175656e63650200000004fdffffff086c6f636b74696d65020000000400000000046f7574730b000000000776657273696f6e0200000004010000000c00000003066861736865730b000000000a747265652d646570746801000000000000000000000000000000080874782d696e64657801ffffffffffffffffffffffffffffffff';
+    const fnArgs = Buffer.from(contractCallArgsWithEmptyList, 'hex');
+    const clarityVals = decodeClarityValueList(fnArgs);
+    clarityVals.map(c => {
+      expect(c.hex).not.toBe('');
+    });
+  });
+
+  describe('fuzzed tsvs', () => {
+    let db: PgWriteStore;
+    let client: PgSqlClient;
+    let api: ApiServer;
+
+    beforeEach(async () => {
+      await migrate('up');
+      db = await PgWriteStore.connect({
+        usageName: 'tests',
+        withNotifier: true,
+        skipMigrations: true,
+      });
+      client = db.sql;
+      api = await startApiServer({ datastore: db, chainId: ChainID.Testnet });
+
+      // set chainId env, because TSV import reads it manually
+      process.env['STACKS_CHAIN_ID'] = ChainID.Testnet.toString();
+    });
+
+    afterEach(async () => {
+      await api.terminate();
+      await db?.close();
+      await migrate('down');
+    });
+
+    test('parse fuzzed transactions', async () => {
+      const eventServer = await startEventServer({
+        datastore: db,
+        chainId: ChainID.Testnet,
+        serverHost: '127.0.0.1',
+        serverPort: 0,
+      });
+
+      const readStream = readline.createInterface({
+        input: fs.createReadStream('tests/api/tsv/fuzzed-transactions-1.tsv', { encoding: 'utf8' }),
+        crlfDelay: Infinity,
+      });
+      for await (const line of readStream) {
+        const [id, timestamp, eventPath, payload] = line.split('\t');
+        await httpPostRequest({
+          host: '127.0.0.1',
+          port: eventServer.serverAddress.port,
+          path: eventPath,
+          headers: { 'Content-Type': 'application/json' },
+          body: Buffer.from(payload, 'utf8'),
+          throwOnNotOK: true,
+        });
+      }
+
+      await eventServer.closeAsync();
+    });
   });
 });