fix: do not force historical rpc and rate limits

.env.defaults

@@ -1,5 +1,5 @@
 SOLANA_RPC=http://solrpc1.aleph.cloud:7725/
-SOLANA_MAIN_PUBLIC_RPC=https://api.mainnet-beta.solana.com
+SOLANA_HISTORIC_RPC=https://api.mainnet-beta.solana.com
 
 # 16 GB RAM for node.js
 NODE_OPTIONS=--max-old-space-size=16384
@@ -41,8 +41,7 @@ NODE_OPTIONS=--max-old-space-size=16384
 # OASYS_INDEX_LOGS (default true)
 
 # SOLANA_RPC
-# SOLANA_PUBLIC_RPC
-# SOLANA_MAIN_PUBLIC_RPC
+# SOLANA_HISTORIC_RPC
 
 ## INDEXER EXAMPLE
 

README.md

@@ -88,12 +88,9 @@ ETHEREUM_INDEX_BLOCKS
 # For specifying a custom solana RPC node/cluster without rate limits (*mandatory*)
 SOLANA_RPC
 
-# For specifying a public solana RPC node/cluster rate limited (optional)
-SOLANA_PUBLIC_RPC
-
 # For specifying a main public solana RPC node/cluster rate limited 
 # that guarantees historical data access (default "https://api.mainnet-beta.solana.com")
-SOLANA_MAIN_PUBLIC_RPC
+SOLANA_HISTORIC_RPC
 
 # Other configuration vars
 

packages/framework/src/services/fetcher/src/baseHistoryFetcher.ts

@@ -180,7 +180,9 @@ export class BaseHistoryFetcher<C> {
    * Initialises the fetcherState class property of the Fetcher instance, could get
    * the data from the data access layer when the fetching progress is restarted.
    */
-  protected async getFetcherState(): Promise<BaseFetcherState<C>> {
+  protected async getFetcherState(
+    useHistoricRPC = true,
+  ): Promise<BaseFetcherState<C>> {
     if (this.fetcherState) return this.fetcherState
 
     const id = this.options.id
@@ -193,14 +195,14 @@ export class BaseHistoryFetcher<C> {
           lastRun: 0,
           numRuns: 0,
           complete: false,
-          useHistoricRPC: false,
+          useHistoricRPC,
         },
         backward: {
           frequency: undefined,
           lastRun: 0,
           numRuns: 0,
           complete: false,
-          useHistoricRPC: false,
+          useHistoricRPC,
         },
       },
     }

packages/framework/src/services/fetcher/src/entityHistoryFetcher.ts

@@ -47,7 +47,7 @@ export abstract class BaseEntityHistoryFetcher<
     this.pendingAccounts = new FetcherPool({
       id,
       interval: 0,
-      concurrency: 1,
+      concurrency: 100,
       dal: this.accountDAL,
       fetcherCache: true,
       getFetcher: ({ id, payload }) =>

packages/solana/src/sdk/index.ts

@@ -1,3 +1,2 @@
 export * from './client.js'
-export * from './instance.js'
 export * from './token/index.js'

packages/solana/src/services/fetcher/factory.ts

@@ -14,21 +14,57 @@ import {
   createRawEntityDAL,
   FetcherMsClient,
   IndexableEntityType,
+  getBlockchainEnv,
   // createAccountStateDAL,
 } from '@aleph-indexer/framework'
-import {
-  solanaMainPublicRPC,
-  solanaMainPublicRPCRoundRobin,
-  solanaPrivateRPC,
-  solanaPrivateRPCRoundRobin,
-} from '../../sdk/index.js'
+
 import { createSolanaAccountTransactionHistoryDAL } from './src/dal/accountTransactionHistory.js'
 import { SolanaRawTransaction } from '../../types.js'
 import { SolanaTransactionFetcher } from './src/transactionFetcher.js'
 import { SolanaTransactionHistoryFetcher } from './src/transactionHistoryFetcher.js'
+import { MAIN_SOLANA_CLUSTER_URL } from '../../utils/constants.js'
+import { SolanaRPC, SolanaRPCRoundRobin } from '../../sdk/client.js'
 // import { SolanaStateFetcher } from './src/stateFetcher.js'
 // import { SolanaAccountState } from './src/types.js'
 
+
+function getClusterConfig(blockchainId: BlockchainId): {
+  historyRpcRR: SolanaRPCRoundRobin
+  historyRpc: SolanaRPC,
+  privateRpcRR: SolanaRPCRoundRobin
+  privateRpc: SolanaRPC,
+} {
+  const historicEnv = getBlockchainEnv(blockchainId, 'HISTORIC_RPC', false) || `${MAIN_SOLANA_CLUSTER_URL}|true`
+  const [historicUrls, historicRateLimitStr] = historicEnv.split('|')
+  const historicUrlList = historicUrls.split(',')
+  const historicRateLimit = historicRateLimitStr === 'true'
+
+  const historyRpcRR = new SolanaRPCRoundRobin(historicUrlList, historicRateLimit)
+  const historyRpc = historyRpcRR.getProxy()
+
+
+  const privateEnv = getBlockchainEnv(blockchainId, 'RPC', true)
+  let privateRpcRR = historyRpcRR
+  let privateRpc = historyRpc
+
+  if (privateEnv !== historicEnv) {
+    const [privateUrls, privateRateLimitStr] = privateEnv.split('|')
+    const privateUrlList = privateUrls.split(',')
+    const privateRateLimit = privateRateLimitStr === 'true'
+
+    privateRpcRR = new SolanaRPCRoundRobin(privateUrlList, privateRateLimit)
+    privateRpc = privateRpcRR.getProxy()
+
+  }
+
+  return {
+    historyRpcRR,
+    historyRpc,
+    privateRpcRR,
+    privateRpc
+  }
+}
+
 export async function solanaFetcherFactory(
   blockchainId: BlockchainId,
   basePath: string,
@@ -37,11 +73,21 @@ export async function solanaFetcherFactory(
 ): Promise<BlockchainFetcherI> {
   if (basePath) await Utils.ensurePath(basePath)
 
+
+    const {
+      historyRpcRR,
+      historyRpc,
+      privateRpcRR,
+      privateRpc
+    } = getClusterConfig(blockchainId)
+
+
+
   // @note: Force resolve DNS and cache it before starting fetcher
   await Promise.allSettled(
     [
-      ...solanaPrivateRPCRoundRobin.getAllClients(),
-      ...solanaMainPublicRPCRoundRobin.getAllClients(),
+      ...privateRpcRR.getAllClients(),
+      ...historyRpcRR.getAllClients(),
     ].map(async (client) => {
       const conn = client.getConnection()
       const { result } = await (conn as any)._rpcRequest('getBlockHeight', [])
@@ -62,8 +108,8 @@ export async function solanaFetcherFactory(
 
   const transactionHistoryFetcher = new SolanaTransactionHistoryFetcher(
     blockchainId,
-    solanaPrivateRPC,
-    solanaMainPublicRPC,
+    privateRpc,
+    historyRpc,
     transactionHistoryFetcherStateDAL,
     fetcherClient,
     transactionHistoryPendingAccountDAL,
@@ -72,8 +118,8 @@ export async function solanaFetcherFactory(
 
   const transactionFetcher = new SolanaTransactionFetcher(
     blockchainId,
-    solanaPrivateRPC,
-    solanaMainPublicRPC,
+    privateRpc,
+    historyRpc,
     broker,
     pendingEntityDAL,
     pendingEntityCacheDAL,
@@ -83,8 +129,8 @@ export async function solanaFetcherFactory(
 
   // const accountStateFetcherMain = new SolanaStateFetcher(
   //   blockchainId,
-  //   solanaPrivateRPC,
-  //   solanaMainPublicRPC,
+  //   privateRpc,
+  //   historyRpc,
   //   accountStateDAL,
   //   accountStatePendingAccountDAL,
   // )

packages/solana/src/services/fetcher/src/accountTransactionHistoryFetcher.ts

@@ -3,6 +3,7 @@ import { ConfirmedSignatureInfo } from '@solana/web3.js'
 import { Utils } from '@aleph-indexer/core'
 import {
   BaseFetcherPaginationCursors,
+  BaseFetcherState,
   BaseHistoryFetcher,
   BlockchainId,
   FetcherJobRunnerHandleFetchResult,
@@ -70,6 +71,13 @@ export class SolanaAccountTransactionHistoryFetcher extends BaseHistoryFetcher<S
     )
   }
 
+  protected async getFetcherState(): Promise<
+    BaseFetcherState<SolanaAccountTransactionHistoryPaginationCursor>
+  > {
+    const useHistoricRPC = this.solanaMainPublicRpc === this.solanaRpc
+    return super.getFetcherState(useHistoricRPC)
+  }
+
   protected async fetchForward({
     firstRun,
     interval,

packages/solana/src/utils/constants.ts

@@ -1,6 +1,8 @@
 import { PublicKey } from '@solana/web3.js'
 import { ErrorCatalog } from '../types.js'
 
+export const MAIN_SOLANA_CLUSTER_URL = 'https://api.mainnet-beta.solana.com'
+
 // CONSTANTS from raydium-ui... might be a good idea to fetch those from their github?
 export const SYSTEM_PROGRAM_ID = new PublicKey(
   '11111111111111111111111111111111',