Headless TUI

codegenerator/cli/README.md

@@ -19,7 +19,7 @@ HyperIndex is a fast, developer-friendly multichain indexer, optimized for both
 - **[Reorg support](https://docs.envio.dev/docs/HyperIndex/reorgs-support)** – Graceful handling of blockchain reorganizations
 - **GraphQL API** – Easy-to-query indexed data
 - **Flexible language support** – JavaScript, TypeScript, and ReScript
-- **Factory contract support** – Index data from 100,000+ factory contracts seamlessly
+- **Factory contract support** – Index data from 1M+ dynamically registered contracts seamlessly
 - **On-chain & off-chain data integration** – Easily combine multiple data sources
 - **[Self-hosted & managed options](https://docs.envio.dev/docs/HyperIndex/hosted-service)** – Run your own setup or use HyperIndex hosted services
 - **Detailed logging & error reporting** – Debug and optimize with clarity

codegenerator/cli/npm/envio/package.json

@@ -23,6 +23,11 @@
     "@envio-dev/hypersync-client": "0.6.5",
     "rescript": "11.1.3",
     "rescript-schema": "9.3.0",
+    "@rescript/react": "0.12.1",
+    "ink": "3.2.0",
+    "ink-big-text": "1.2.0",
+    "ink-spinner": "4.0.3",
+    "ink-use-stdout-dimensions": "1.0.5",
     "viem": "2.21.0",
     "bignumber.js": "9.1.2",
     "pino": "8.16.1",

codegenerator/cli/npm/envio/package.json.tmpl

@@ -34,6 +34,11 @@
     "@envio-dev/hypersync-client": "0.6.5",
     "rescript": "11.1.3",
     "rescript-schema": "9.3.0",
+    "@rescript/react": "0.12.1",
+    "ink": "3.2.0",
+    "ink-big-text": "1.2.0",
+    "ink-spinner": "4.0.3",
+    "ink-use-stdout-dimensions": "1.0.5",
     "viem": "2.21.0",
     "bignumber.js": "9.1.2",
     "pino": "8.16.1",

codegenerator/cli/npm/envio/rescript.json

@@ -13,8 +13,14 @@
     "in-source": true
   },
   "gentypeconfig": {
+    "shims": {
+      "Js": "Js"
+    },
     "generatedFileExtension": ".gen.ts"
   },
-  "bs-dependencies": ["rescript-schema"],
+  "jsx": {
+    "version": 4
+  },
+  "bs-dependencies": ["rescript-schema", "@rescript/react"],
   "bsc-flags": ["-open RescriptSchema"]
 }

codegenerator/cli/npm/envio/src/FetchState.res

@@ -818,20 +818,27 @@ let getNextQuery = (
       if (
         p->checkIsFetchingPartition->not && p.latestFetchedBlock.blockNumber < maxQueryBlockNumber
       ) {
-        switch p->makePartitionQuery(
-          ~indexingContracts,
-          ~endBlock=switch blockLag {
-          | 0 => endBlock
-          | _ =>
-            switch endBlock {
-            | Some(endBlock) => Some(Pervasives.min(headBlock, endBlock))
-            // Force head block as an endBlock when blockLag is set
-            // because otherwise HyperSync might return bigger range
-            | None => Some(headBlock)
-            }
-          },
-          ~mergeTarget,
-        ) {
+        let endBlock = switch blockLag {
+        | 0 => endBlock
+        | _ =>
+          switch endBlock {
+          | Some(endBlock) => Some(Pervasives.min(headBlock, endBlock))
+          // Force head block as an endBlock when blockLag is set
+          // because otherwise HyperSync might return bigger range
+          | None => Some(headBlock)
+          }
+        }
+        // Enforce the respose range up until target block
+        // Otherwise for indexers with 100+ partitions
+        // we might blow up the buffer size to more than 600k events
+        // simply because of HyperSync returning extra blocks
+        let endBlock = switch (endBlock, maxQueryBlockNumber < currentBlockHeight) {
+        | (Some(endBlock), true) => Some(Pervasives.min(maxQueryBlockNumber, endBlock))
+        | (None, true) => Some(maxQueryBlockNumber)
+        | (_, false) => endBlock
+        }
+
+        switch p->makePartitionQuery(~indexingContracts, ~endBlock, ~mergeTarget) {
         | Some(q) => queries->Array.push(q)
         | None => ()
         }
@@ -1061,10 +1068,6 @@ let make = (
   Prometheus.IndexingPartitions.set(~partitionsCount=partitions->Array.length, ~chainId)
   Prometheus.IndexingBufferSize.set(~bufferSize=0, ~chainId)
   Prometheus.IndexingBufferBlockNumber.set(~blockNumber=latestFetchedBlock.blockNumber, ~chainId)
-  switch endBlock {
-  | Some(endBlock) => Prometheus.IndexingEndBlock.set(~endBlock, ~chainId)
-  | None => ()
-  }
 
   {
     partitions,

codegenerator/cli/npm/envio/src/Hasura.res

@@ -222,24 +222,43 @@ let trackDatabase = async (
   ~endpoint,
   ~auth,
   ~pgSchema,
-  ~allStaticTables,
-  ~allEntityTables,
+  ~userEntities: array<Internal.entityConfig>,
   ~aggregateEntities,
   ~responseLimit,
   ~schema,
 ) => {
+  let trackOnlyInternalTableNames = [
+    InternalTable.Chains.table.tableName,
+    InternalTable.EventSyncState.table.tableName,
+    InternalTable.PersistedState.table.tableName,
+    InternalTable.EndOfBlockRangeScannedData.table.tableName,
+    InternalTable.DynamicContractRegistry.table.tableName,
+  ]
+  let exposedInternalTableNames = [
+    InternalTable.RawEvents.table.tableName,
+    InternalTable.Views.metaViewName,
+    InternalTable.Views.chainMetadataViewName,
+  ]
+  let userTableNames = userEntities->Js.Array2.map(entity => entity.table.tableName)
+
   Logging.info("Tracking tables in Hasura")
 
   let _ = await clearHasuraMetadata(~endpoint, ~auth)
-  let tableNames =
-    [allStaticTables, allEntityTables]
-    ->Belt.Array.concatMany
-    ->Js.Array2.map(({tableName}: Table.table) => tableName)
 
-  await trackTables(~endpoint, ~auth, ~pgSchema, ~tableNames)
+  await trackTables(
+    ~endpoint,
+    ~auth,
+    ~pgSchema,
+    ~tableNames=[
+      exposedInternalTableNames,
+      trackOnlyInternalTableNames,
+      userTableNames,
+    ]->Belt.Array.concatMany,
+  )
 
   let _ =
-    await tableNames
+    await [exposedInternalTableNames, userTableNames]
+    ->Belt.Array.concatMany
     ->Js.Array2.map(tableName =>
       createSelectPermissions(
         ~endpoint,
@@ -251,11 +270,11 @@ let trackDatabase = async (
       )
     )
     ->Js.Array2.concatMany(
-      allEntityTables->Js.Array2.map(table => {
-        let {tableName} = table
+      userEntities->Js.Array2.map(entityConfig => {
+        let {tableName} = entityConfig.table
         [
           //Set array relationships
-          table
+          entityConfig.table
           ->Table.getDerivedFromFields
           ->Js.Array2.map(derivedFromField => {
             //determines the actual name of the underlying relational field (if it's an entity mapping then suffixes _id for eg.)
@@ -275,7 +294,7 @@ let trackDatabase = async (
             )
           }),
           //Set object relationships
-          table
+          entityConfig.table
           ->Table.getLinkedEntityFields
           ->Js.Array2.map(((field, linkedEntityName)) => {
             createEntityRelationship(

codegenerator/cli/npm/envio/src/Internal.res

@@ -176,13 +176,16 @@ let fuelTransferParamsSchema = S.schema(s => {
 })
 
 type entity = private {id: string}
-type entityConfig = {
+type genericEntityConfig<'entity> = {
   name: string,
-  schema: S.t<entity>,
-  rowsSchema: S.t<array<entity>>,
+  schema: S.t<'entity>,
+  rowsSchema: S.t<array<'entity>>,
   table: Table.table,
-  entityHistory: EntityHistory.t<entity>,
+  entityHistory: EntityHistory.t<'entity>,
 }
+type entityConfig = genericEntityConfig<entity>
+external fromGenericEntityConfig: genericEntityConfig<'entity> => entityConfig = "%identity"
+
 type enum
 type enumConfig<'enum> = {
   name: string,

codegenerator/cli/npm/envio/src/InternalConfig.res

@@ -0,0 +1,22 @@
+// TODO: rename the file to Config.res after finishing the migration from codegen
+// And turn it into PublicConfig instead
+// For internal use we should create Indexer.res with a stateful type
+
+type contract = {
+  name: string,
+  abi: EvmTypes.Abi.t,
+  addresses: array<Address.t>,
+  events: array<Internal.eventConfig>,
+  startBlock: option<int>,
+}
+
+type chain = {
+  id: int,
+  startBlock: int,
+  endBlock?: int,
+  confirmedBlockThreshold: int,
+  contracts: array<contract>,
+  sources: array<Source.t>,
+}
+
+type ecosystem = | @as("evm") Evm | @as("fuel") Fuel

codegenerator/cli/npm/envio/src/Js.shim.ts

@@ -0,0 +1,11 @@
+export type Json_t =
+  | string
+  | boolean
+  | number
+  | null
+  | { [key: string]: Json_t }
+  | Json_t[];
+
+export type t = unknown;
+
+export type Exn_t = Error;

codegenerator/cli/npm/envio/src/LoadManager.res

@@ -66,26 +66,32 @@ let schedule = async loadManager => {
       }
     })
 
-    if inputsToLoad->Utils.Array.isEmpty->not {
+    let isSuccess = if inputsToLoad->Utils.Array.isEmpty->not {
       try {
         await group.load(inputsToLoad)
+        true
       } catch {
       | exn => {
           let exn = exn->Utils.prettifyExn
           currentInputKeys->Array.forEach(inputKey => {
             let call = calls->Js.Dict.unsafeGet(inputKey)
             call.reject(exn)
           })
+          false
         }
       }
+    } else {
+      true
     }
 
     if currentInputKeys->Utils.Array.isEmpty->not {
-      currentInputKeys->Js.Array2.forEach(inputKey => {
-        let call = calls->Js.Dict.unsafeGet(inputKey)
-        calls->Utils.Dict.deleteInPlace(inputKey)
-        call.resolve(group.getUnsafeInMemory(inputKey))
-      })
+      if isSuccess {
+        currentInputKeys->Js.Array2.forEach(inputKey => {
+          let call = calls->Js.Dict.unsafeGet(inputKey)
+          calls->Utils.Dict.deleteInPlace(inputKey)
+          call.resolve(group.getUnsafeInMemory(inputKey))
+        })
+      }
 
       // Clean up executed batch to reset
       // provided load function which