cli: refactor build, dev, start commands (#111)

### Context This PR refactors how the build, dev, and start commands work. We leverage the Rollup's virtual plugin to generate a list of indexers to build time. This indexers are then imported by the dev/start command at runtime.
apibara · Nov 4, 2024 · c5c284f · c5c284f
2 parents d3aec5a + 7039ff0
commit c5c284f
Show file tree

Hide file tree

Showing 55 changed files with 828 additions and 339 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -37,13 +37,12 @@ jobs:
         run: pnpm install --strict-peer-dependencies=false
       - name: Run lint
         run: pnpm lint
-      - name: Run format
-        run: pnpm format
       - name: Run build
-        run: pnpm build
+        # Exclude examples/cli since it requires the cli to be built.
+        run: pnpm build --filter=!./examples/cli --filter=!./examples/cli-js
       - name: Run typecheck
         run: pnpm typecheck
       - name: Run test
-        run: pnpm test:ci
+        run: pnpm test:ci --filter=!./examples/cli --filter=!./examples/cli-js
       - name: Check change files
         run: pnpm beachball check
diff --git a/change/@apibara-indexer-843e6831-307c-49b1-bf59-833803a4a27f.json b/change/@apibara-indexer-843e6831-307c-49b1-bf59-833803a4a27f.json
@@ -0,0 +1,7 @@
+{
+  "type": "none",
+  "comment": "Remove unused scripts",
+  "packageName": "@apibara/indexer",
+  "email": "francesco@ceccon.me",
+  "dependentChangeType": "none"
+}
diff --git a/change/apibara-ed31dbbf-efbd-44c6-8eb0-f729dde0acb6.json b/change/apibara-ed31dbbf-efbd-44c6-8eb0-f729dde0acb6.json
@@ -0,0 +1,7 @@
+{
+  "type": "prerelease",
+  "comment": "Refactor dev, build, and start commands",
+  "packageName": "apibara",
+  "email": "francesco@ceccon.me",
+  "dependentChangeType": "patch"
+}
diff --git a/examples/cli-js/apibara.config.mjs b/examples/cli-js/apibara.config.mjs
@@ -0,0 +1,15 @@
+import { defineConfig } from "apibara/config";
+
+export default defineConfig({
+  runtimeConfig: {
+    test: 123,
+    check: "something",
+  },
+  presets: {
+    dev: {
+      runtimeConfig: {
+        test: 999,
+      },
+    },
+  },
+});
diff --git a/examples/cli-js/indexers/starknet.indexer.js b/examples/cli-js/indexers/starknet.indexer.js
@@ -0,0 +1,24 @@
+import { defineIndexer } from "@apibara/indexer";
+import { StarknetStream } from "@apibara/starknet";
+import { hash } from "starknet";
+
+export default defineIndexer(StarknetStream)({
+  streamUrl: "https://starknet.preview.apibara.org",
+  finality: "accepted",
+  startingCursor: {
+    orderKey: 800_000n,
+  },
+  filter: {
+    events: [
+      {
+        address:
+          "0x049d36570d4e46f48e99674bd3fcc84644ddd6b96f7c741b1562b82f9e004dc7",
+        keys: [hash.getSelectorFromName("Transfer")],
+        includeReceipt: true,
+      },
+    ],
+  },
+  async transform({ block: { header } }) {
+    console.log("Transforming block ", header?.blockNumber);
+  },
+});
diff --git a/examples/cli-js/package.json b/examples/cli-js/package.json
@@ -0,0 +1,27 @@
+{
+  "name": "example-cli-js",
+  "private": true,
+  "version": "1.0.0",
+  "description": "",
+  "scripts": {
+    "build": "apibara build",
+    "dev": "apibara dev",
+    "start": "apibara start",
+    "lint": "biome check .",
+    "lint:fix": "pnpm lint --write"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "devDependencies": {
+    "@types/node": "^20.5.2",
+    "typescript": "^5.6.2"
+  },
+  "dependencies": {
+    "@apibara/indexer": "workspace:*",
+    "@apibara/protocol": "workspace:*",
+    "@apibara/starknet": "workspace:*",
+    "apibara": "workspace:*",
+    "starknet": "^6.11.0"
+  }
+}
diff --git a/examples/cli/apibara.config.ts b/examples/cli/apibara.config.ts
@@ -1,4 +1,3 @@
-import { defaultSink } from "@apibara/indexer";
 import { defineConfig } from "apibara/config";
 
 export default defineConfig({
@@ -13,7 +12,4 @@ export default defineConfig({
       },
     },
   },
-  sink: {
-    default: () => defaultSink(),
-  },
 });
diff --git a/examples/cli/indexers/starknet.indexer.ts b/examples/cli/indexers/starknet.indexer.ts
@@ -1,16 +1,21 @@
 import { defineIndexer } from "@apibara/indexer";
+import { sqlitePersistence } from "@apibara/indexer/plugins/persistence";
 import { StarknetStream } from "@apibara/starknet";
 import type { ApibaraRuntimeConfig } from "apibara/types";
+import Database from "better-sqlite3";
 import { hash } from "starknet";
 
 export default function (runtimeConfig: ApibaraRuntimeConfig) {
   console.log("--> Starknet Indexer Runtime Config: ", runtimeConfig);
+  const database = new Database(":memory:");
+
   return defineIndexer(StarknetStream)({
     streamUrl: "https://starknet.preview.apibara.org",
     finality: "accepted",
     startingCursor: {
-      orderKey: 80_000n,
+      orderKey: 800_000n,
     },
+    plugins: [sqlitePersistence({ database })],
     filter: {
       events: [
         {
@@ -24,10 +29,5 @@ export default function (runtimeConfig: ApibaraRuntimeConfig) {
     async transform({ block: { header } }) {
       console.log("Transforming block ", header?.blockNumber);
     },
-    hooks: {
-      "handler:after": ({ endCursor }) => {
-        console.log("Handler After ", endCursor?.orderKey);
-      },
-    },
   });
 }
diff --git a/examples/cli/package.json b/examples/cli/package.json
@@ -3,11 +3,18 @@
   "private": true,
   "version": "1.0.0",
   "description": "",
-  "scripts": {},
+  "scripts": {
+    "build": "apibara build",
+    "dev": "apibara dev",
+    "start": "apibara start",
+    "lint": "biome check .",
+    "lint:fix": "pnpm lint --write"
+  },
   "keywords": [],
   "author": "",
   "license": "ISC",
   "devDependencies": {
+    "@types/better-sqlite3": "^7.6.11",
     "@types/node": "^20.5.2",
     "typescript": "^5.6.2"
   },
@@ -16,6 +23,7 @@
     "@apibara/protocol": "workspace:*",
     "@apibara/starknet": "workspace:*",
     "apibara": "workspace:*",
+    "better-sqlite3": "^11.5.0",
     "starknet": "^6.11.0"
   }
 }
diff --git a/examples/indexer/package.json b/examples/indexer/package.json
@@ -19,7 +19,7 @@
     "@opentelemetry/sdk-node": "^0.52.0",
     "@opentelemetry/sdk-trace-base": "^1.25.0",
     "@opentelemetry/semantic-conventions": "^1.25.0",
-    "better-sqlite3": "^11.1.2",
+    "better-sqlite3": "^11.5.0",
     "citty": "^0.1.6",
     "consola": "^3.2.3",
     "csv-stringify": "^6.5.0",

diff --git a/flake.nix b/flake.nix
@@ -13,6 +13,7 @@
       in
         {
           devShells.default = pkgs.mkShell {
+            MADNESS_ALLOW_LDD = "1";
             nativeBuildInputs = with pkgs; [
               protobuf
               nodejs

diff --git a/package.json b/package.json
@@ -9,9 +9,8 @@
     "test": "turbo run test",
     "test:ci": "turbo run test:ci",
     "test:postgres": "docker run --rm -p 5432:5432 -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=postgres postgres:16",
-    "lint": "turbo run lint --parallel",
-    "lint:fix": "turbo run lint:fix --parallel",
-    "format": "turbo run format --parallel",
+    "lint": "pnpm run -r lint",
+    "lint:fix": "pnpm run -r lint:fix",
     "release": "pnpm build"
   },
   "repository": {

diff --git a/packages/cli/build.config.ts b/packages/cli/build.config.ts
@@ -2,9 +2,17 @@ import { fileURLToPath } from "node:url";
 import { resolve } from "pathe";
 import { defineBuildConfig } from "unbuild";
 
-const modules = ["cli", "config", "core", "rollup", "types", "hooks "];
+const modules = [
+  "cli",
+  "config",
+  "core",
+  "rollup",
+  "types",
+  "hooks",
+  "runtime",
+];
 
-// @ts-ignore The 'import.meta' meta-property is only allowed when the '--module' option is 'es2020', 'es2022', 'esnext', 'system', 'node16', or 'nodenext'.
+// @ts-ignore
 const srcDir = fileURLToPath(new URL("src", import.meta.url));
 
 export default defineBuildConfig({
@@ -15,19 +23,22 @@ export default defineBuildConfig({
     { input: "./src/rollup/index.ts" },
     { input: "./src/types/index.ts" },
     { input: "./src/hooks/index.ts" },
-    { input: "./src/internal/consola/index.ts" },
-    { input: "./src/internal/citty/index.ts" },
+    { input: "./src/runtime/", outDir: "./dist/runtime", format: "esm" },
   ],
   clean: true,
   outDir: "./dist",
   declaration: true,
   alias: {
+    "apibara/runtime/meta": resolve(srcDir, "../runtime-meta.mjs"),
     ...Object.fromEntries(
       modules.map((module) => [
         `apibara/${module}`,
         resolve(srcDir, `${module}/index.ts`),
       ]),
     ),
   },
-  externals: [...modules.map((module) => `apibara/${module}`)],
+  externals: [
+    "apibara/runtime/meta",
+    ...modules.map((module) => `apibara/${module}`),
+  ],
 });
diff --git a/packages/cli/package.json b/packages/cli/package.json
@@ -32,13 +32,21 @@
       "types": "./dist/hooks/index.d.ts",
       "import": "./dist/hooks/index.mjs"
     },
-    "./internal/consola": {
-      "import": "./dist/internal/consola/index.mjs",
-      "types": "./dist/internal/consola/index.d.ts"
+    "./runtime": {
+      "types": "./dist/runtime/index.d.ts",
+      "import": "./dist/runtime/index.mjs"
     },
-    "./internal/citty": {
-      "import": "./dist/internal/citty/index.mjs",
-      "types": "./dist/internal/citty/index.d.ts"
+    "./runtime/meta": {
+      "types": "./runtime-meta.d.ts",
+      "import": "./runtime-meta.mjs"
+    },
+    "./runtime/*": {
+      "types": "./dist/runtime/*.d.ts",
+      "import": "./dist/runtime/*.mjs"
+    },
+    "./dist/runtime/*": {
+      "types": "./dist/runtime/*.d.ts",
+      "import": "./dist/runtime/*.mjs"
     }
   },
   "bin": {
@@ -55,13 +63,14 @@
     "typecheck": "tsc --noEmit",
     "lint:fix": "pnpm lint --write",
     "format": "biome format . --write",
-    "playground": "JITI_ESM_RESOLVE=1 NODE_OPTIONS=\"--enable-source-maps\" jiti ./src/cli/index.ts",
+    "playground": "JITI_ESM_RESOLVE=1 CONSOLA_LEVEL=debug NODE_OPTIONS=\"--enable-source-maps\" jiti ./src/cli/index.ts",
     "playground:prepare": "pnpm playground prepare --dir playground",
     "playground:dev": "pnpm playground dev --dir playground",
     "playground:build": "pnpm playground build --dir playground",
-    "playground:start": "JITI_ESM_RESOLVE=1 NODE_OPTIONS=\"--enable-source-maps\" jiti ./playground/.apibara/build/main.mjs"
+    "playground:start": "pnpm playground start --dir playground --indexer starknet"
   },
   "devDependencies": {
+    "@apibara/starknet": "workspace:*",
     "@types/fs-extra": "^11.0.4",
     "@types/node": "^20.14.0",
     "jiti": "^1.21.0",
@@ -71,14 +80,13 @@
     "vitest": "^1.6.0"
   },
   "dependencies": {
-    "@apibara/evm": "workspace:*",
     "@apibara/indexer": "workspace:*",
     "@apibara/protocol": "workspace:*",
-    "@apibara/starknet": "workspace:*",
     "@rollup/plugin-commonjs": "^26.0.1",
     "@rollup/plugin-json": "^6.1.0",
     "@rollup/plugin-node-resolve": "^15.2.3",
     "@rollup/plugin-typescript": "^11.1.6",
+    "@rollup/plugin-virtual": "^3.0.2",
     "c12": "^1.11.1",
     "chokidar": "^3.6.0",
     "citty": "^0.1.6",
@@ -88,10 +96,12 @@
     "fs-extra": "^11.2.0",
     "hookable": "^5.5.3",
     "klona": "^2.0.6",
+    "magic-string": "^0.30.12",
     "pathe": "^1.1.2",
     "perfect-debounce": "^1.0.0",
     "pkg-types": "^1.1.3",
     "rollup": "^4.18.1",
+    "rollup-plugin-esbuild": "^6.1.1",
     "tslib": "^2.6.3",
     "untyped": "^1.4.2"
   }

diff --git a/packages/cli/playground/apibara.config.ts b/packages/cli/playground/apibara.config.ts
@@ -1,10 +1,15 @@
-import { defaultSink } from "@apibara/indexer";
 import { defineConfig } from "apibara/config";
 
 export default defineConfig({
   runtimeConfig: {
     test: 123,
     check: "something",
+    nested: {
+      test: 456,
+    },
+    fromEnv: {
+      nodeEnv: process.env.NODE_ENV,
+    },
   },
   presets: {
     dev: {
@@ -13,7 +18,4 @@ export default defineConfig({
       },
     },
   },
-  sink: {
-    default: () => defaultSink(),
-  },
 });
diff --git a/packages/cli/playground/indexers/starknet.indexer.ts b/packages/cli/playground/indexers/starknet.indexer.ts
@@ -7,10 +7,10 @@ import { hash } from "starknet";
 export default function indexer(runtimeConfig: ApibaraRuntimeConfig) {
   consola.log("--> Starknet Indexer Runtime Config: ", runtimeConfig);
   return defineIndexer(StarknetStream)({
-    streamUrl: "http://mainnet-v2.starknet.a5a.ch:7007",
+    streamUrl: "https://starknet.preview.apibara.org",
     finality: "accepted",
     startingCursor: {
-      orderKey: 80_000n,
+      orderKey: 800_000n,
     },
     filter: {
       events: [
@@ -23,12 +23,9 @@ export default function indexer(runtimeConfig: ApibaraRuntimeConfig) {
       ],
     },
     async transform({ block: { header, events } }) {
-      consola.info("Transforming block ", header?.blockNumber);
-    },
-    hooks: {
-      "handler:after": ({ endCursor }) => {
-        consola.info("Handler After ", endCursor?.orderKey);
-      },
+      consola.info(
+        `Got block ${header?.blockNumber} with ${events.length} events`,
+      );
     },
   });
 }
diff --git a/packages/cli/playground/tsconfig.json b/packages/cli/playground/tsconfig.json
diff --git a/packages/cli/runtime-meta.d.ts b/packages/cli/runtime-meta.d.ts
@@ -0,0 +1,2 @@
+export declare const pkgDir: string;
+export declare const runtimeDir: string;