-
Notifications
You must be signed in to change notification settings - Fork 199
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
feat(store-sync): add util to fetch snapshot from indexer with SQL API #2996
Changes from 1 commit
8b76bdd
52dd46b
121479d
9abfffc
367daec
dc154d2
bf885cc
aca18b9
b36d6fe
175fe7c
a677f32
9c398d8
f22747a
0af11fa
fa2bbc7
25a2a77
6cb9b3f
f2fbaad
c92258e
494b0bb
11c5cdd
9adc159
403179f
6488f2c
c71eff1
1c77240
b819a2c
a005e3d
805f433
ed21c2f
e681e95
b04a4d8
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
- Loading branch information
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,6 +1,6 @@ | ||
import { DecodeDozerRecordsResult, DozerQueryResult, decodeDozerRecords } from "./decodeDozerRecords"; | ||
import { Hex } from "viem"; | ||
import { DozerTableQuery } from "./common"; | ||
import { TableQuery } from "./common"; | ||
import { Table } from "@latticexyz/config"; | ||
|
||
type DozerResponseSuccess = { | ||
|
@@ -12,13 +12,17 @@ type DozerResponseFail = { msg: string }; | |
|
||
type DozerResponse = DozerResponseSuccess | DozerResponseFail; | ||
|
||
type FetchDozerSqlArgs = { | ||
function isDozerResponseFail(response: DozerResponse): response is DozerResponseFail { | ||
return "msg" in response; | ||
} | ||
|
||
type FetchRecordsSqlArgs = { | ||
dozerUrl: string; | ||
storeAddress: Hex; | ||
queries: DozerTableQuery[]; | ||
queries: TableQuery[]; | ||
}; | ||
|
||
type FetchDozerSqlResult = | ||
type FetchRecordsSqlResult = | ||
| { | ||
blockHeight: bigint; | ||
result: { | ||
|
@@ -28,15 +32,11 @@ type FetchDozerSqlResult = | |
} | ||
| undefined; | ||
|
||
function isDozerResponseFail(response: DozerResponse): response is DozerResponseFail { | ||
return "msg" in response; | ||
} | ||
|
||
export async function fetchRecordsDozerSql({ | ||
export async function fetchRecordsSql({ | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. what about just |
||
dozerUrl, | ||
queries, | ||
storeAddress, | ||
}: FetchDozerSqlArgs): Promise<FetchDozerSqlResult> { | ||
}: FetchRecordsSqlArgs): Promise<FetchRecordsSqlResult> { | ||
const response: DozerResponse = await ( | ||
await fetch(dozerUrl, { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. since this URL we're fetching is tied to the shape of the request/response, I wonder if this arg should be the endpoint/hostname and we append the path like |
||
method: "POST", | ||
|
@@ -57,7 +57,7 @@ export async function fetchRecordsDozerSql({ | |
return; | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I find it a little weird that a function to fetch a thing returns undefined on a failure rather than throwing or something. I would usually reserve this behavior for the place where "failure is optional" e.g. no changes expected, just curious to hear your reasoning There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. no i think you're right, can't remember why i put it in here. moved it into |
||
} | ||
|
||
const result: FetchDozerSqlResult = { | ||
const result: FetchRecordsSqlResult = { | ||
blockHeight: BigInt(response.block_height), | ||
result: response.result.map((records, index) => ({ | ||
table: queries[index].table, | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,30 +1,30 @@ | ||
import { DozerLogFilter, DozerSyncFilter, DozerTableQuery } from "./common"; | ||
import { LogFilter, SyncFilter, TableQuery } from "./common"; | ||
import { Hex } from "viem"; | ||
import { StorageAdapterBlock, SyncFilter } from "../common"; | ||
import { fetchRecordsDozerSql } from "./fetchRecordsDozerSql"; | ||
import { StorageAdapterBlock, SyncFilter as LegacyLogFilter } from "../common"; | ||
import { fetchRecordsSql } from "./fetchRecordsSql"; | ||
import { recordToLog } from "../recordToLog"; | ||
import { getSnapshot } from "../getSnapshot"; | ||
import { getSnapshot as getSnapshotLogs } from "../getSnapshot"; | ||
import { bigIntMin, isDefined } from "@latticexyz/common/utils"; | ||
|
||
export type FetchInitialBlockLogsDozerArgs = { | ||
export type GetSnapshotArgs = { | ||
dozerUrl: string; | ||
storeAddress: Hex; | ||
filters?: DozerSyncFilter[]; | ||
filters?: SyncFilter[]; | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. would it better or worse to split this into two args: There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I kinda like the fact that |
||
startBlock?: bigint; | ||
chainId: number; | ||
}; | ||
|
||
export type FetchInitialBlockLogsDozerResult = { | ||
export type GetSnapshotResult = { | ||
initialBlockLogs: StorageAdapterBlock; | ||
}; | ||
|
||
export async function fetchInitialBlockLogsDozer({ | ||
export async function getSnapshot({ | ||
dozerUrl, | ||
storeAddress, | ||
filters, | ||
startBlock = 0n, | ||
chainId, | ||
}: FetchInitialBlockLogsDozerArgs): Promise<FetchInitialBlockLogsDozerResult> { | ||
}: GetSnapshotArgs): Promise<GetSnapshotResult> { | ||
const initialBlockLogs: StorageAdapterBlock = { blockNumber: startBlock, logs: [] }; | ||
|
||
// We execute the list of provided SQL queries for hydration. For performance | ||
|
@@ -36,13 +36,13 @@ export async function fetchInitialBlockLogsDozer({ | |
// partial updates), so we only notify consumers of state updates after the | ||
// initial hydration is complete. | ||
|
||
const sqlFilters = filters ? (filters.filter((filter) => "sql" in filter) as DozerTableQuery[]) : []; | ||
const sqlFilters = filters ? (filters.filter((filter) => "sql" in filter) as TableQuery[]) : []; | ||
|
||
// Execute individual SQL queries as separate requests to parallelize on the backend. | ||
// Each individual request is expected to be executed against the same db state so it | ||
// can't be parallelized. | ||
const dozerTables = ( | ||
await Promise.all(sqlFilters.map((filter) => fetchRecordsDozerSql({ dozerUrl, storeAddress, queries: [filter] }))) | ||
await Promise.all(sqlFilters.map((filter) => fetchRecordsSql({ dozerUrl, storeAddress, queries: [filter] }))) | ||
).filter(isDefined); | ||
|
||
if (dozerTables.length > 0) { | ||
|
@@ -54,30 +54,30 @@ export async function fetchInitialBlockLogsDozer({ | |
} | ||
|
||
// Fetch the tables without SQL filter from the snapshot logs API for better performance. | ||
const snapshotFilters = | ||
const logsFilters = | ||
filters && | ||
filters | ||
.filter((filter) => !("sql" in filter)) | ||
.map((filter) => { | ||
const { table, key0, key1 } = filter as DozerLogFilter; | ||
return { tableId: table.tableId, key0, key1 } as SyncFilter; | ||
const { table, key0, key1 } = filter as LogFilter; | ||
return { tableId: table.tableId, key0, key1 } as LegacyLogFilter; | ||
}); | ||
|
||
const snapshot = | ||
const logs = | ||
// If no filters are provided, the entire state is fetched | ||
!snapshotFilters || snapshotFilters.length > 0 | ||
? await getSnapshot({ | ||
!logsFilters || logsFilters.length > 0 | ||
? await getSnapshotLogs({ | ||
chainId, | ||
address: storeAddress, | ||
filters: snapshotFilters, | ||
filters: logsFilters, | ||
indexerUrl: dozerUrl, | ||
}) | ||
: undefined; | ||
|
||
// The block number passed in the overall result will be the min of all queries and the snapshot. | ||
if (snapshot) { | ||
initialBlockLogs.blockNumber = bigIntMin(initialBlockLogs.blockNumber, snapshot.blockNumber); | ||
initialBlockLogs.logs = [...initialBlockLogs.logs, ...snapshot.logs]; | ||
// The block number passed in the overall result will be the min of all queries and the logs. | ||
if (logs) { | ||
initialBlockLogs.blockNumber = bigIntMin(initialBlockLogs.blockNumber, logs.blockNumber); | ||
initialBlockLogs.logs = [...initialBlockLogs.logs, ...logs.logs]; | ||
} | ||
|
||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. ooc what happens if There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. yeah i think before that was the case, now all of |
||
return { initialBlockLogs }; | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,4 +1,4 @@ | ||
export * from "./common"; | ||
export * from "./fetchRecordsDozerSql"; | ||
export * from "./fetchRecordsSql"; | ||
export * from "./selectFrom"; | ||
export * from "./fetchInitialBlockLogsDozer"; | ||
export * from "./getSnapshot"; |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,12 +1,12 @@ | ||
import { Table } from "@latticexyz/config"; | ||
import { DozerTableQuery } from "./common"; | ||
import { TableQuery } from "./common"; | ||
|
||
// For autocompletion but still allowing all SQL strings | ||
export type Where<table extends Table> = `"${keyof table["schema"] & string}"` | (string & {}); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. arktype has |
||
|
||
export type SelectFromArgs<table extends Table> = { table: table; where?: Where<table>; limit?: number }; | ||
|
||
export function selectFrom<table extends Table>({ table, where, limit }: SelectFromArgs<table>): DozerTableQuery { | ||
export function selectFrom<table extends Table>({ table, where, limit }: SelectFromArgs<table>): TableQuery { | ||
const dozerTableLabel = table.namespace === "" ? table.name : `${table.namespace}__${table.name}`; | ||
return { | ||
table: table, | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
not blocking but fwiw I found it quite nice to model this sort of thing with arktype, then can parse/validate/return strongly typed response from a JSON API request