Skip to content

Commit

Permalink
cli: improve support for cjs modules like better-sqlite3
Browse files Browse the repository at this point in the history
  • Loading branch information
fracek committed Nov 3, 2024
1 parent 44326f4 commit 7039ff0
Show file tree
Hide file tree
Showing 9 changed files with 170 additions and 54 deletions.
5 changes: 5 additions & 0 deletions examples/cli/indexers/starknet.indexer.ts
Original file line number Diff line number Diff line change
@@ -1,16 +1,21 @@
import { defineIndexer } from "@apibara/indexer";
import { sqlitePersistence } from "@apibara/indexer/plugins/persistence";
import { StarknetStream } from "@apibara/starknet";
import type { ApibaraRuntimeConfig } from "apibara/types";
import Database from "better-sqlite3";
import { hash } from "starknet";

export default function (runtimeConfig: ApibaraRuntimeConfig) {
console.log("--> Starknet Indexer Runtime Config: ", runtimeConfig);
const database = new Database(":memory:");

return defineIndexer(StarknetStream)({
streamUrl: "https://starknet.preview.apibara.org",
finality: "accepted",
startingCursor: {
orderKey: 800_000n,
},
plugins: [sqlitePersistence({ database })],
filter: {
events: [
{
Expand Down
2 changes: 2 additions & 0 deletions examples/cli/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
"author": "",
"license": "ISC",
"devDependencies": {
"@types/better-sqlite3": "^7.6.11",
"@types/node": "^20.5.2",
"typescript": "^5.6.2"
},
Expand All @@ -22,6 +23,7 @@
"@apibara/protocol": "workspace:*",
"@apibara/starknet": "workspace:*",
"apibara": "workspace:*",
"better-sqlite3": "^11.5.0",
"starknet": "^6.11.0"
}
}
2 changes: 1 addition & 1 deletion examples/indexer/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
"@opentelemetry/sdk-node": "^0.52.0",
"@opentelemetry/sdk-trace-base": "^1.25.0",
"@opentelemetry/semantic-conventions": "^1.25.0",
"better-sqlite3": "^11.1.2",
"better-sqlite3": "^11.5.0",
"citty": "^0.1.6",
"consola": "^3.2.3",
"csv-stringify": "^6.5.0",
Expand Down
1 change: 1 addition & 0 deletions packages/cli/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,7 @@
"fs-extra": "^11.2.0",
"hookable": "^5.5.3",
"klona": "^2.0.6",
"magic-string": "^0.30.12",
"pathe": "^1.1.2",
"perfect-debounce": "^1.0.0",
"pkg-types": "^1.1.3",
Expand Down
76 changes: 46 additions & 30 deletions packages/cli/src/rollup/config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,13 @@ import { join } from "pathe";
import type { OutputPluginOption } from "rollup";
import esbuild from "rollup-plugin-esbuild";

import defu from "defu";
import { appConfig } from "./plugins/config";
import { esmShim } from "./plugins/esm-shim";
import { indexers } from "./plugins/indexers";

const runtimeDependencies = ["better-sqlite3"];

export function getRollupConfig(apibara: Apibara): RollupConfig {
const extensions: string[] = [
".ts",
Expand All @@ -21,42 +25,54 @@ export function getRollupConfig(apibara: Apibara): RollupConfig {
".jsx",
];

const rollupConfig: RollupConfig & { plugins: OutputPluginOption[] } = {
input: apibara.options.entry,
output: {
dir: join(apibara.options.outputDir || "./.apibara/build"),
format: "esm",
exports: "auto",
entryFileNames: "[name].mjs",
chunkFileNames: "chunks/[name]-[hash].mjs",
generatedCode: {
constBindings: true,
const rollupConfig: RollupConfig & { plugins: OutputPluginOption[] } = defu(
// biome-ignore lint/suspicious/noExplicitAny: apibara.options.rollupConfig is typed
apibara.options.rollupConfig as any,
<RollupConfig>{
input: apibara.options.entry,
output: {
dir: join(apibara.options.outputDir || "./.apibara/build"),
format: "esm",
exports: "auto",
entryFileNames: "[name].mjs",
chunkFileNames: "chunks/[name]-[hash].mjs",
generatedCode: {
constBindings: true,
},
sourcemap: true,
sourcemapExcludeSources: true,
sourcemapIgnoreList(relativePath, sourcemapPath) {
return relativePath.includes("node_modules");
},
},
sourcemap: true,
sourcemapExcludeSources: true,
sourcemapIgnoreList(relativePath, sourcemapPath) {
return relativePath.includes("node_modules");
plugins: [],
onwarn(warning, rollupWarn) {
if (
!["CIRCULAR_DEPENDENCY", "EVAL", "THIS_IS_UNDEFINED"].includes(
warning.code || "",
) &&
!warning.message.includes("Unsupported source map comment") &&
!warning.message.includes("@__PURE__")
) {
rollupWarn(warning);
}
},
treeshake: true,
external: [...builtinModules, ...runtimeDependencies],
},
plugins: [],
onwarn(warning, rollupWarn) {
if (
!["CIRCULAR_DEPENDENCY", "EVAL", "THIS_IS_UNDEFINED"].includes(
warning.code || "",
) &&
!warning.message.includes("Unsupported source map comment") &&
!warning.message.includes("@__PURE__")
) {
rollupWarn(warning);
}
},
treeshake: true,
external: [...builtinModules],
};
);

rollupConfig.plugins.push(commonjs());
rollupConfig.plugins.push(esmShim());
rollupConfig.plugins.push(json());

rollupConfig.plugins.push(
commonjs({
strictRequires: true,
requireReturnsDefault: "auto",
...apibara.options.commonJS,
}),
);

rollupConfig.plugins.push(
nodeResolve({
extensions,
Expand Down
69 changes: 69 additions & 0 deletions packages/cli/src/rollup/plugins/esm-shim.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
import MagicString from "magic-string";
import type { Plugin } from "rollup";

/**
* An alternative to @rollup/plugin-esm-shim
*/
export function esmShim(): Plugin {
const ESMShim = `
// -- Shims --
import cjsUrl from 'node:url';
import cjsPath from 'node:path';
const __filename = cjsUrl.fileURLToPath(import.meta.url);
const __dirname = cjsPath.dirname(__filename);
// -- End Shims --
`;

const CJSyntaxRegex = /__filename|__dirname/;

return {
name: "esm-shim",

renderChunk(code, _chunk, opts) {
if (opts.format === "es") {
if (code.includes(ESMShim) || !CJSyntaxRegex.test(code)) {
return null;
}

let endIndexOfLastImport = -1;

// Find the last import statement and its ending index
for (const match of code.matchAll(/^import\s.*';$/gm)) {
if (match.length === 0 || typeof match.index !== "number") {
continue;
}

endIndexOfLastImport = match.index + match[0].length;
}

const s = new MagicString(code);
s.appendRight(endIndexOfLastImport, ESMShim);

const sourceMap = s.generateMap({
includeContent: true,
});

let sourcesContent: string[] | undefined;
if (Array.isArray(sourceMap.sourcesContent)) {
sourcesContent = [];
for (let i = 0; i < sourceMap.sourcesContent.length; i++) {
const content = sourceMap.sourcesContent[i];
if (typeof content === "string") {
sourcesContent.push(content);
}
}
}

return {
code: s.toString(),
map: {
...sourceMap,
sourcesContent,
},
};
}

return null;
},
};
}
2 changes: 2 additions & 0 deletions packages/cli/src/types/config.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import type { RollupCommonJSOptions } from "@rollup/plugin-commonjs";
import type {
C12InputConfig,
ConfigWatcher,
Expand Down Expand Up @@ -73,6 +74,7 @@ export interface ApibaraOptions<
rollupConfig?: RollupConfig;
sourceMap?: boolean;
entry: string;
commonJS?: RollupCommonJSOptions;

// Advanced
typescript: {
Expand Down
4 changes: 2 additions & 2 deletions packages/indexer/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@
"@types/better-sqlite3": "^7.6.11",
"@types/node": "^20.14.0",
"@types/pg": "^8.11.10",
"better-sqlite3": "^11.1.2",
"better-sqlite3": "^11.5.0",
"csv-stringify": "^6.5.0",
"drizzle-orm": "^0.35.2",
"pg": "^8.12.0",
Expand All @@ -95,7 +95,7 @@
"unctx": "^2.3.1"
},
"peerDependencies": {
"better-sqlite3": "^11.1.2",
"better-sqlite3": "^11.5.0",
"csv-stringify": "^6.5.0",
"drizzle-orm": "^0.35.2",
"postgres-range": "^1.1.4",
Expand Down
Loading

0 comments on commit 7039ff0

Please sign in to comment.