From 6d46576b75f67f3ca378fcb0e517bb7a789b2ddf Mon Sep 17 00:00:00 2001 From: Filipe Azevedo Date: Mon, 3 Mar 2025 16:20:23 +0000 Subject: [PATCH 001/150] fix substreams info endpoint (#5851) --- chain/ethereum/examples/firehose.rs | 1 + chain/substreams/examples/substreams.rs | 1 + graph/build.rs | 1 + graph/proto/substreams-rpc.proto | 157 ++-- graph/src/firehose/endpoints.rs | 83 +- graph/src/substreams_rpc/sf.firehose.v2.rs | 884 ++++++++++++++++++ .../substreams_rpc/sf.substreams.rpc.v2.rs | 378 +++++++- node/src/chain.rs | 2 + tests/src/fixture/mod.rs | 1 + 9 files changed, 1392 insertions(+), 116 deletions(-) create mode 100644 graph/src/substreams_rpc/sf.firehose.v2.rs diff --git a/chain/ethereum/examples/firehose.rs b/chain/ethereum/examples/firehose.rs index e5f85964fe1..5a70794dfe2 100644 --- a/chain/ethereum/examples/firehose.rs +++ b/chain/ethereum/examples/firehose.rs @@ -38,6 +38,7 @@ async fn main() -> Result<(), Error> { false, SubgraphLimit::Unlimited, metrics, + false, )); loop { diff --git a/chain/substreams/examples/substreams.rs b/chain/substreams/examples/substreams.rs index eac9397b893..a5af2bbe25c 100644 --- a/chain/substreams/examples/substreams.rs +++ b/chain/substreams/examples/substreams.rs @@ -57,6 +57,7 @@ async fn main() -> Result<(), Error> { false, SubgraphLimit::Unlimited, Arc::new(endpoint_metrics), + true, )); let client = Arc::new(ChainClient::new_firehose(FirehoseEndpoints::for_testing( diff --git a/graph/build.rs b/graph/build.rs index 3cc00c0dc07..4ad3c03a459 100644 --- a/graph/build.rs +++ b/graph/build.rs @@ -22,6 +22,7 @@ fn main() { tonic_build::configure() .protoc_arg("--experimental_allow_proto3_optional") .extern_path(".sf.substreams.v1", "crate::substreams") + .extern_path(".sf.firehose.v2", "crate::firehose") .out_dir("src/substreams_rpc") .compile(&["proto/substreams-rpc.proto"], &["proto"]) .expect("Failed to compile Substreams RPC proto(s)"); diff --git a/graph/proto/substreams-rpc.proto b/graph/proto/substreams-rpc.proto index a0ba1b72037..28298458480 100644 --- a/graph/proto/substreams-rpc.proto +++ b/graph/proto/substreams-rpc.proto @@ -4,39 +4,46 @@ package sf.substreams.rpc.v2; import "google/protobuf/any.proto"; import "substreams.proto"; +import "firehose.proto"; -service Stream { - rpc Blocks(Request) returns (stream Response); +service EndpointInfo { + rpc Info(sf.firehose.v2.InfoRequest) returns (sf.firehose.v2.InfoResponse); } +service Stream { rpc Blocks(Request) returns (stream Response); } + message Request { int64 start_block_num = 1; string start_cursor = 2; uint64 stop_block_num = 3; // With final_block_only, you only receive blocks that are irreversible: - // 'final_block_height' will be equal to current block and no 'undo_signal' will ever be sent + // 'final_block_height' will be equal to current block and no 'undo_signal' + // will ever be sent bool final_blocks_only = 4; - // Substreams has two mode when executing your module(s) either development mode or production - // mode. Development and production modes impact the execution of Substreams, important aspects - // of execution include: + // Substreams has two mode when executing your module(s) either development + // mode or production mode. Development and production modes impact the + // execution of Substreams, important aspects of execution include: // * The time required to reach the first byte. // * The speed that large ranges get executed. // * The module logs and outputs sent back to the client. // - // By default, the engine runs in developer mode, with richer and deeper output. Differences - // between production and development modes include: - // * Forward parallel execution is enabled in production mode and disabled in development mode - // * The time required to reach the first byte in development mode is faster than in production mode. + // By default, the engine runs in developer mode, with richer and deeper + // output. Differences between production and development modes include: + // * Forward parallel execution is enabled in production mode and disabled in + // development mode + // * The time required to reach the first byte in development mode is faster + // than in production mode. // // Specific attributes of development mode include: // * The client will receive all of the executed module's logs. - // * It's possible to request specific store snapshots in the execution tree (via `debug_initial_store_snapshot_for_modules`). + // * It's possible to request specific store snapshots in the execution tree + // (via `debug_initial_store_snapshot_for_modules`). // * Multiple module's output is possible. // - // With production mode`, however, you trade off functionality for high speed enabling forward - // parallel execution of module ahead of time. + // With production mode`, however, you trade off functionality for high speed + // enabling forward parallel execution of module ahead of time. bool production_mode = 5; string output_module = 6; @@ -47,23 +54,24 @@ message Request { repeated string debug_initial_store_snapshot_for_modules = 10; } - message Response { oneof message { - SessionInit session = 1; // Always sent first - ModulesProgress progress = 2; // Progress of data preparation, before sending in the stream of `data` events. + SessionInit session = 1; // Always sent first + ModulesProgress progress = 2; // Progress of data preparation, before + // sending in the stream of `data` events. BlockScopedData block_scoped_data = 3; BlockUndoSignal block_undo_signal = 4; Error fatal_error = 5; - // Available only in developer mode, and only if `debug_initial_store_snapshot_for_modules` is set. + // Available only in developer mode, and only if + // `debug_initial_store_snapshot_for_modules` is set. InitialSnapshotData debug_snapshot_data = 10; - // Available only in developer mode, and only if `debug_initial_store_snapshot_for_modules` is set. + // Available only in developer mode, and only if + // `debug_initial_store_snapshot_for_modules` is set. InitialSnapshotComplete debug_snapshot_complete = 11; } } - // BlockUndoSignal informs you that every bit of data // with a block number above 'last_valid_block' has been reverted // on-chain. Delete that data and restart from 'last_valid_cursor' @@ -84,16 +92,14 @@ message BlockScopedData { repeated StoreModuleOutput debug_store_outputs = 11; } -message SessionInit { +message SessionInit { string trace_id = 1; uint64 resolved_start_block = 2; uint64 linear_handoff_block = 3; uint64 max_parallel_workers = 4; } -message InitialSnapshotComplete { - string cursor = 1; -} +message InitialSnapshotComplete { string cursor = 1; } message InitialSnapshotData { string module_name = 1; @@ -110,9 +116,9 @@ message MapModuleOutput { } // StoreModuleOutput are produced for store modules in development mode. -// It is not possible to retrieve store models in production, with parallelization -// enabled. If you need the deltas directly, write a pass through mapper module -// that will get them down to you. +// It is not possible to retrieve store models in production, with +// parallelization enabled. If you need the deltas directly, write a pass +// through mapper module that will get them down to you. message StoreModuleOutput { string name = 1; repeated StoreDelta debug_store_deltas = 2; @@ -121,8 +127,9 @@ message StoreModuleOutput { message OutputDebugInfo { repeated string logs = 1; - // LogsTruncated is a flag that tells you if you received all the logs or if they - // were truncated because you logged too much (fixed limit currently is set to 128 KiB). + // LogsTruncated is a flag that tells you if you received all the logs or if + // they were truncated because you logged too much (fixed limit currently is + // set to 128 KiB). bool logs_truncated = 2; bool cached = 3; } @@ -130,7 +137,8 @@ message OutputDebugInfo { // ModulesProgress is a message that is sent every 500ms message ModulesProgress { // previously: repeated ModuleProgress modules = 1; - // these previous `modules` messages were sent in bursts and are not sent anymore. + // these previous `modules` messages were sent in bursts and are not sent + // anymore. reserved 1; // List of jobs running on tier2 servers repeated Job running_jobs = 2; @@ -147,73 +155,82 @@ message ProcessedBytes { uint64 total_bytes_written = 2; } - message Error { string module = 1; string reason = 2; repeated string logs = 3; - // FailureLogsTruncated is a flag that tells you if you received all the logs or if they - // were truncated because you logged too much (fixed limit currently is set to 128 KiB). + // FailureLogsTruncated is a flag that tells you if you received all the logs + // or if they were truncated because you logged too much (fixed limit + // currently is set to 128 KiB). bool logs_truncated = 4; } - message Job { - uint32 stage = 1; - uint64 start_block = 2; - uint64 stop_block = 3; - uint64 processed_blocks = 4; - uint64 duration_ms = 5; + uint32 stage = 1; + uint64 start_block = 2; + uint64 stop_block = 3; + uint64 processed_blocks = 4; + uint64 duration_ms = 5; } message Stage { - repeated string modules = 1; - repeated BlockRange completed_ranges = 2; + repeated string modules = 1; + repeated BlockRange completed_ranges = 2; } -// ModuleStats gathers metrics and statistics from each module, running on tier1 or tier2 -// All the 'count' and 'time_ms' values may include duplicate for each stage going over that module +// ModuleStats gathers metrics and statistics from each module, running on tier1 +// or tier2 All the 'count' and 'time_ms' values may include duplicate for each +// stage going over that module message ModuleStats { - // name of the module - string name = 1; + // name of the module + string name = 1; - // total_processed_blocks is the sum of blocks sent to that module code - uint64 total_processed_block_count = 2; - // total_processing_time_ms is the sum of all time spent running that module code - uint64 total_processing_time_ms = 3; + // total_processed_blocks is the sum of blocks sent to that module code + uint64 total_processed_block_count = 2; + // total_processing_time_ms is the sum of all time spent running that module + // code + uint64 total_processing_time_ms = 3; - //// external_calls are chain-specific intrinsics, like "Ethereum RPC calls". - repeated ExternalCallMetric external_call_metrics = 4; + //// external_calls are chain-specific intrinsics, like "Ethereum RPC calls". + repeated ExternalCallMetric external_call_metrics = 4; - // total_store_operation_time_ms is the sum of all time spent running that module code waiting for a store operation (ex: read, write, delete...) - uint64 total_store_operation_time_ms = 5; - // total_store_read_count is the sum of all the store Read operations called from that module code - uint64 total_store_read_count = 6; + // total_store_operation_time_ms is the sum of all time spent running that + // module code waiting for a store operation (ex: read, write, delete...) + uint64 total_store_operation_time_ms = 5; + // total_store_read_count is the sum of all the store Read operations called + // from that module code + uint64 total_store_read_count = 6; - // total_store_write_count is the sum of all store Write operations called from that module code (store-only) - uint64 total_store_write_count = 10; + // total_store_write_count is the sum of all store Write operations called + // from that module code (store-only) + uint64 total_store_write_count = 10; - // total_store_deleteprefix_count is the sum of all store DeletePrefix operations called from that module code (store-only) - // note that DeletePrefix can be a costly operation on large stores - uint64 total_store_deleteprefix_count = 11; + // total_store_deleteprefix_count is the sum of all store DeletePrefix + // operations called from that module code (store-only) note that DeletePrefix + // can be a costly operation on large stores + uint64 total_store_deleteprefix_count = 11; - // store_size_bytes is the uncompressed size of the full KV store for that module, from the last 'merge' operation (store-only) - uint64 store_size_bytes = 12; + // store_size_bytes is the uncompressed size of the full KV store for that + // module, from the last 'merge' operation (store-only) + uint64 store_size_bytes = 12; - // total_store_merging_time_ms is the time spent merging partial stores into a full KV store for that module (store-only) - uint64 total_store_merging_time_ms = 13; + // total_store_merging_time_ms is the time spent merging partial stores into a + // full KV store for that module (store-only) + uint64 total_store_merging_time_ms = 13; - // store_currently_merging is true if there is a merging operation (partial store to full KV store) on the way. - bool store_currently_merging = 14; + // store_currently_merging is true if there is a merging operation (partial + // store to full KV store) on the way. + bool store_currently_merging = 14; - // highest_contiguous_block is the highest block in the highest merged full KV store of that module (store-only) - uint64 highest_contiguous_block = 15; + // highest_contiguous_block is the highest block in the highest merged full KV + // store of that module (store-only) + uint64 highest_contiguous_block = 15; } message ExternalCallMetric { - string name = 1; - uint64 count = 2; - uint64 time_ms = 3; + string name = 1; + uint64 count = 2; + uint64 time_ms = 3; } message StoreDelta { diff --git a/graph/src/firehose/endpoints.rs b/graph/src/firehose/endpoints.rs index ebf27faa5a1..00b87ea21a4 100644 --- a/graph/src/firehose/endpoints.rs +++ b/graph/src/firehose/endpoints.rs @@ -1,3 +1,4 @@ +use crate::firehose::codec::InfoRequest; use crate::firehose::fetch_client::FetchClient; use crate::firehose::interceptors::AuthInterceptor; use crate::{ @@ -51,6 +52,7 @@ pub struct FirehoseEndpoint { pub filters_enabled: bool, pub compression_enabled: bool, pub subgraph_limit: SubgraphLimit, + is_substreams: bool, endpoint_metrics: Arc, channel: Channel, @@ -79,8 +81,15 @@ impl NetworkDetails for Arc { async fn provides_extended_blocks(&self) -> anyhow::Result { let info = self.clone().info().await?; + let pred = if info.chain_name.contains("arbitrum-one") + || info.chain_name.contains("optimism-mainnet") + { + |x: &String| x.starts_with("extended") || x == "hybrid" + } else { + |x: &String| x == "extended" + }; - Ok(info.block_features.iter().all(|x| x == "extended")) + Ok(info.block_features.iter().any(pred)) } } @@ -175,6 +184,7 @@ impl FirehoseEndpoint { compression_enabled: bool, subgraph_limit: SubgraphLimit, endpoint_metrics: Arc, + is_substreams_endpoint: bool, ) -> Self { let uri = url .as_ref() @@ -238,6 +248,7 @@ impl FirehoseEndpoint { subgraph_limit, endpoint_metrics, info_response: OnceCell::new(), + is_substreams: is_substreams_endpoint, } } @@ -306,7 +317,44 @@ impl FirehoseEndpoint { client } - fn new_substreams_client( + fn new_firehose_info_client(&self) -> crate::firehose::endpoint_info::Client { + let metrics = self.metrics_interceptor(); + let auth = self.auth.clone(); + + let mut client = crate::firehose::endpoint_info::Client::new(metrics, auth); + + if self.compression_enabled { + client = client.with_compression(); + } + + client = client.with_max_message_size(self.max_message_size()); + client + } + + fn new_substreams_info_client( + &self, + ) -> crate::substreams_rpc::endpoint_info_client::EndpointInfoClient< + InterceptedService, impl tonic::service::Interceptor>, + > { + let metrics = self.metrics_interceptor(); + + let mut client = + crate::substreams_rpc::endpoint_info_client::EndpointInfoClient::with_interceptor( + metrics, + self.auth.clone(), + ) + .accept_compressed(CompressionEncoding::Gzip); + + if self.compression_enabled { + client = client.send_compressed(CompressionEncoding::Gzip); + } + + client = client.max_decoding_message_size(self.max_message_size()); + + client + } + + fn new_substreams_streaming_client( &self, ) -> substreams_rpc::stream_client::StreamClient< InterceptedService, impl tonic::service::Interceptor>, @@ -595,7 +643,7 @@ impl FirehoseEndpoint { request: substreams_rpc::Request, headers: &ConnectionHeaders, ) -> Result, anyhow::Error> { - let mut client = self.new_substreams_client(); + let mut client = self.new_substreams_streaming_client(); let request = headers.add_to_request(request); let response_stream = client.blocks(request).await?; let block_stream = response_stream.into_inner(); @@ -610,18 +658,20 @@ impl FirehoseEndpoint { self.info_response .get_or_try_init(move || async move { - let metrics = endpoint.metrics_interceptor(); - let auth = endpoint.auth.clone(); - - let mut client = crate::firehose::endpoint_info::Client::new(metrics, auth); + if endpoint.is_substreams { + let mut client = endpoint.new_substreams_info_client(); + + client + .info(InfoRequest {}) + .await + .map(|r| r.into_inner()) + .map_err(anyhow::Error::from) + .and_then(|e| e.try_into()) + } else { + let mut client = endpoint.new_firehose_info_client(); - if endpoint.compression_enabled { - client = client.with_compression(); + client.info().await } - - client = client.with_max_message_size(endpoint.max_message_size()); - - client.info().await }) .await .map(ToOwned::to_owned) @@ -709,6 +759,7 @@ mod test { false, SubgraphLimit::Unlimited, Arc::new(EndpointMetrics::mock()), + false, ))]; let endpoints = FirehoseEndpoints::for_testing(endpoint); @@ -741,6 +792,7 @@ mod test { false, SubgraphLimit::Limit(2), Arc::new(EndpointMetrics::mock()), + false, ))]; let endpoints = FirehoseEndpoints::for_testing(endpoint); @@ -768,6 +820,7 @@ mod test { false, SubgraphLimit::Disabled, Arc::new(EndpointMetrics::mock()), + false, ))]; let endpoints = FirehoseEndpoints::for_testing(endpoint); @@ -794,6 +847,7 @@ mod test { false, SubgraphLimit::Unlimited, endpoint_metrics.clone(), + false, )); let high_error_adapter2 = Arc::new(FirehoseEndpoint::new( "high_error".to_string(), @@ -804,6 +858,7 @@ mod test { false, SubgraphLimit::Unlimited, endpoint_metrics.clone(), + false, )); let low_availability = Arc::new(FirehoseEndpoint::new( "low availability".to_string(), @@ -814,6 +869,7 @@ mod test { false, SubgraphLimit::Limit(2), endpoint_metrics.clone(), + false, )); let high_availability = Arc::new(FirehoseEndpoint::new( "high availability".to_string(), @@ -824,6 +880,7 @@ mod test { false, SubgraphLimit::Unlimited, endpoint_metrics.clone(), + false, )); endpoint_metrics.report_for_test(&high_error_adapter1.provider, false); diff --git a/graph/src/substreams_rpc/sf.firehose.v2.rs b/graph/src/substreams_rpc/sf.firehose.v2.rs new file mode 100644 index 00000000000..ac86a47e505 --- /dev/null +++ b/graph/src/substreams_rpc/sf.firehose.v2.rs @@ -0,0 +1,884 @@ +// This file is @generated by prost-build. +/// Generated client implementations. +pub mod stream_client { + #![allow(unused_variables, dead_code, missing_docs, clippy::let_unit_value)] + use tonic::codegen::*; + use tonic::codegen::http::Uri; + #[derive(Debug, Clone)] + pub struct StreamClient { + inner: tonic::client::Grpc, + } + impl StreamClient { + /// Attempt to create a new client by connecting to a given endpoint. + pub async fn connect(dst: D) -> Result + where + D: TryInto, + D::Error: Into, + { + let conn = tonic::transport::Endpoint::new(dst)?.connect().await?; + Ok(Self::new(conn)) + } + } + impl StreamClient + where + T: tonic::client::GrpcService, + T::Error: Into, + T::ResponseBody: Body + Send + 'static, + ::Error: Into + Send, + { + pub fn new(inner: T) -> Self { + let inner = tonic::client::Grpc::new(inner); + Self { inner } + } + pub fn with_origin(inner: T, origin: Uri) -> Self { + let inner = tonic::client::Grpc::with_origin(inner, origin); + Self { inner } + } + pub fn with_interceptor( + inner: T, + interceptor: F, + ) -> StreamClient> + where + F: tonic::service::Interceptor, + T::ResponseBody: Default, + T: tonic::codegen::Service< + http::Request, + Response = http::Response< + >::ResponseBody, + >, + >, + , + >>::Error: Into + Send + Sync, + { + StreamClient::new(InterceptedService::new(inner, interceptor)) + } + /// Compress requests with the given encoding. + /// + /// This requires the server to support it otherwise it might respond with an + /// error. + #[must_use] + pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.inner = self.inner.send_compressed(encoding); + self + } + /// Enable decompressing responses. + #[must_use] + pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.inner = self.inner.accept_compressed(encoding); + self + } + /// Limits the maximum size of a decoded message. + /// + /// Default: `4MB` + #[must_use] + pub fn max_decoding_message_size(mut self, limit: usize) -> Self { + self.inner = self.inner.max_decoding_message_size(limit); + self + } + /// Limits the maximum size of an encoded message. + /// + /// Default: `usize::MAX` + #[must_use] + pub fn max_encoding_message_size(mut self, limit: usize) -> Self { + self.inner = self.inner.max_encoding_message_size(limit); + self + } + pub async fn blocks( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response>, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/sf.firehose.v2.Stream/Blocks", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("sf.firehose.v2.Stream", "Blocks")); + self.inner.server_streaming(req, path, codec).await + } + } +} +/// Generated client implementations. +pub mod fetch_client { + #![allow(unused_variables, dead_code, missing_docs, clippy::let_unit_value)] + use tonic::codegen::*; + use tonic::codegen::http::Uri; + #[derive(Debug, Clone)] + pub struct FetchClient { + inner: tonic::client::Grpc, + } + impl FetchClient { + /// Attempt to create a new client by connecting to a given endpoint. + pub async fn connect(dst: D) -> Result + where + D: TryInto, + D::Error: Into, + { + let conn = tonic::transport::Endpoint::new(dst)?.connect().await?; + Ok(Self::new(conn)) + } + } + impl FetchClient + where + T: tonic::client::GrpcService, + T::Error: Into, + T::ResponseBody: Body + Send + 'static, + ::Error: Into + Send, + { + pub fn new(inner: T) -> Self { + let inner = tonic::client::Grpc::new(inner); + Self { inner } + } + pub fn with_origin(inner: T, origin: Uri) -> Self { + let inner = tonic::client::Grpc::with_origin(inner, origin); + Self { inner } + } + pub fn with_interceptor( + inner: T, + interceptor: F, + ) -> FetchClient> + where + F: tonic::service::Interceptor, + T::ResponseBody: Default, + T: tonic::codegen::Service< + http::Request, + Response = http::Response< + >::ResponseBody, + >, + >, + , + >>::Error: Into + Send + Sync, + { + FetchClient::new(InterceptedService::new(inner, interceptor)) + } + /// Compress requests with the given encoding. + /// + /// This requires the server to support it otherwise it might respond with an + /// error. + #[must_use] + pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.inner = self.inner.send_compressed(encoding); + self + } + /// Enable decompressing responses. + #[must_use] + pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.inner = self.inner.accept_compressed(encoding); + self + } + /// Limits the maximum size of a decoded message. + /// + /// Default: `4MB` + #[must_use] + pub fn max_decoding_message_size(mut self, limit: usize) -> Self { + self.inner = self.inner.max_decoding_message_size(limit); + self + } + /// Limits the maximum size of an encoded message. + /// + /// Default: `usize::MAX` + #[must_use] + pub fn max_encoding_message_size(mut self, limit: usize) -> Self { + self.inner = self.inner.max_encoding_message_size(limit); + self + } + pub async fn block( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/sf.firehose.v2.Fetch/Block", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("sf.firehose.v2.Fetch", "Block")); + self.inner.unary(req, path, codec).await + } + } +} +/// Generated client implementations. +pub mod endpoint_info_client { + #![allow(unused_variables, dead_code, missing_docs, clippy::let_unit_value)] + use tonic::codegen::*; + use tonic::codegen::http::Uri; + #[derive(Debug, Clone)] + pub struct EndpointInfoClient { + inner: tonic::client::Grpc, + } + impl EndpointInfoClient { + /// Attempt to create a new client by connecting to a given endpoint. + pub async fn connect(dst: D) -> Result + where + D: TryInto, + D::Error: Into, + { + let conn = tonic::transport::Endpoint::new(dst)?.connect().await?; + Ok(Self::new(conn)) + } + } + impl EndpointInfoClient + where + T: tonic::client::GrpcService, + T::Error: Into, + T::ResponseBody: Body + Send + 'static, + ::Error: Into + Send, + { + pub fn new(inner: T) -> Self { + let inner = tonic::client::Grpc::new(inner); + Self { inner } + } + pub fn with_origin(inner: T, origin: Uri) -> Self { + let inner = tonic::client::Grpc::with_origin(inner, origin); + Self { inner } + } + pub fn with_interceptor( + inner: T, + interceptor: F, + ) -> EndpointInfoClient> + where + F: tonic::service::Interceptor, + T::ResponseBody: Default, + T: tonic::codegen::Service< + http::Request, + Response = http::Response< + >::ResponseBody, + >, + >, + , + >>::Error: Into + Send + Sync, + { + EndpointInfoClient::new(InterceptedService::new(inner, interceptor)) + } + /// Compress requests with the given encoding. + /// + /// This requires the server to support it otherwise it might respond with an + /// error. + #[must_use] + pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.inner = self.inner.send_compressed(encoding); + self + } + /// Enable decompressing responses. + #[must_use] + pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.inner = self.inner.accept_compressed(encoding); + self + } + /// Limits the maximum size of a decoded message. + /// + /// Default: `4MB` + #[must_use] + pub fn max_decoding_message_size(mut self, limit: usize) -> Self { + self.inner = self.inner.max_decoding_message_size(limit); + self + } + /// Limits the maximum size of an encoded message. + /// + /// Default: `usize::MAX` + #[must_use] + pub fn max_encoding_message_size(mut self, limit: usize) -> Self { + self.inner = self.inner.max_encoding_message_size(limit); + self + } + pub async fn info( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/sf.firehose.v2.EndpointInfo/Info", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("sf.firehose.v2.EndpointInfo", "Info")); + self.inner.unary(req, path, codec).await + } + } +} +/// Generated server implementations. +pub mod stream_server { + #![allow(unused_variables, dead_code, missing_docs, clippy::let_unit_value)] + use tonic::codegen::*; + /// Generated trait containing gRPC methods that should be implemented for use with StreamServer. + #[async_trait] + pub trait Stream: Send + Sync + 'static { + /// Server streaming response type for the Blocks method. + type BlocksStream: tonic::codegen::tokio_stream::Stream< + Item = std::result::Result, + > + + Send + + 'static; + async fn blocks( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; + } + #[derive(Debug)] + pub struct StreamServer { + inner: _Inner, + accept_compression_encodings: EnabledCompressionEncodings, + send_compression_encodings: EnabledCompressionEncodings, + max_decoding_message_size: Option, + max_encoding_message_size: Option, + } + struct _Inner(Arc); + impl StreamServer { + pub fn new(inner: T) -> Self { + Self::from_arc(Arc::new(inner)) + } + pub fn from_arc(inner: Arc) -> Self { + let inner = _Inner(inner); + Self { + inner, + accept_compression_encodings: Default::default(), + send_compression_encodings: Default::default(), + max_decoding_message_size: None, + max_encoding_message_size: None, + } + } + pub fn with_interceptor( + inner: T, + interceptor: F, + ) -> InterceptedService + where + F: tonic::service::Interceptor, + { + InterceptedService::new(Self::new(inner), interceptor) + } + /// Enable decompressing requests with the given encoding. + #[must_use] + pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.accept_compression_encodings.enable(encoding); + self + } + /// Compress responses with the given encoding, if the client supports it. + #[must_use] + pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.send_compression_encodings.enable(encoding); + self + } + /// Limits the maximum size of a decoded message. + /// + /// Default: `4MB` + #[must_use] + pub fn max_decoding_message_size(mut self, limit: usize) -> Self { + self.max_decoding_message_size = Some(limit); + self + } + /// Limits the maximum size of an encoded message. + /// + /// Default: `usize::MAX` + #[must_use] + pub fn max_encoding_message_size(mut self, limit: usize) -> Self { + self.max_encoding_message_size = Some(limit); + self + } + } + impl tonic::codegen::Service> for StreamServer + where + T: Stream, + B: Body + Send + 'static, + B::Error: Into + Send + 'static, + { + type Response = http::Response; + type Error = std::convert::Infallible; + type Future = BoxFuture; + fn poll_ready( + &mut self, + _cx: &mut Context<'_>, + ) -> Poll> { + Poll::Ready(Ok(())) + } + fn call(&mut self, req: http::Request) -> Self::Future { + let inner = self.inner.clone(); + match req.uri().path() { + "/sf.firehose.v2.Stream/Blocks" => { + #[allow(non_camel_case_types)] + struct BlocksSvc(pub Arc); + impl< + T: Stream, + > tonic::server::ServerStreamingService + for BlocksSvc { + type Response = crate::firehose::Response; + type ResponseStream = T::BlocksStream; + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::blocks(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let inner = inner.0; + let method = BlocksSvc(inner); + let codec = tonic::codec::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.server_streaming(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + _ => { + Box::pin(async move { + Ok( + http::Response::builder() + .status(200) + .header("grpc-status", "12") + .header("content-type", "application/grpc") + .body(empty_body()) + .unwrap(), + ) + }) + } + } + } + } + impl Clone for StreamServer { + fn clone(&self) -> Self { + let inner = self.inner.clone(); + Self { + inner, + accept_compression_encodings: self.accept_compression_encodings, + send_compression_encodings: self.send_compression_encodings, + max_decoding_message_size: self.max_decoding_message_size, + max_encoding_message_size: self.max_encoding_message_size, + } + } + } + impl Clone for _Inner { + fn clone(&self) -> Self { + Self(Arc::clone(&self.0)) + } + } + impl std::fmt::Debug for _Inner { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{:?}", self.0) + } + } + impl tonic::server::NamedService for StreamServer { + const NAME: &'static str = "sf.firehose.v2.Stream"; + } +} +/// Generated server implementations. +pub mod fetch_server { + #![allow(unused_variables, dead_code, missing_docs, clippy::let_unit_value)] + use tonic::codegen::*; + /// Generated trait containing gRPC methods that should be implemented for use with FetchServer. + #[async_trait] + pub trait Fetch: Send + Sync + 'static { + async fn block( + &self, + request: tonic::Request, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; + } + #[derive(Debug)] + pub struct FetchServer { + inner: _Inner, + accept_compression_encodings: EnabledCompressionEncodings, + send_compression_encodings: EnabledCompressionEncodings, + max_decoding_message_size: Option, + max_encoding_message_size: Option, + } + struct _Inner(Arc); + impl FetchServer { + pub fn new(inner: T) -> Self { + Self::from_arc(Arc::new(inner)) + } + pub fn from_arc(inner: Arc) -> Self { + let inner = _Inner(inner); + Self { + inner, + accept_compression_encodings: Default::default(), + send_compression_encodings: Default::default(), + max_decoding_message_size: None, + max_encoding_message_size: None, + } + } + pub fn with_interceptor( + inner: T, + interceptor: F, + ) -> InterceptedService + where + F: tonic::service::Interceptor, + { + InterceptedService::new(Self::new(inner), interceptor) + } + /// Enable decompressing requests with the given encoding. + #[must_use] + pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.accept_compression_encodings.enable(encoding); + self + } + /// Compress responses with the given encoding, if the client supports it. + #[must_use] + pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.send_compression_encodings.enable(encoding); + self + } + /// Limits the maximum size of a decoded message. + /// + /// Default: `4MB` + #[must_use] + pub fn max_decoding_message_size(mut self, limit: usize) -> Self { + self.max_decoding_message_size = Some(limit); + self + } + /// Limits the maximum size of an encoded message. + /// + /// Default: `usize::MAX` + #[must_use] + pub fn max_encoding_message_size(mut self, limit: usize) -> Self { + self.max_encoding_message_size = Some(limit); + self + } + } + impl tonic::codegen::Service> for FetchServer + where + T: Fetch, + B: Body + Send + 'static, + B::Error: Into + Send + 'static, + { + type Response = http::Response; + type Error = std::convert::Infallible; + type Future = BoxFuture; + fn poll_ready( + &mut self, + _cx: &mut Context<'_>, + ) -> Poll> { + Poll::Ready(Ok(())) + } + fn call(&mut self, req: http::Request) -> Self::Future { + let inner = self.inner.clone(); + match req.uri().path() { + "/sf.firehose.v2.Fetch/Block" => { + #[allow(non_camel_case_types)] + struct BlockSvc(pub Arc); + impl< + T: Fetch, + > tonic::server::UnaryService + for BlockSvc { + type Response = crate::firehose::SingleBlockResponse; + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::block(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let inner = inner.0; + let method = BlockSvc(inner); + let codec = tonic::codec::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + _ => { + Box::pin(async move { + Ok( + http::Response::builder() + .status(200) + .header("grpc-status", "12") + .header("content-type", "application/grpc") + .body(empty_body()) + .unwrap(), + ) + }) + } + } + } + } + impl Clone for FetchServer { + fn clone(&self) -> Self { + let inner = self.inner.clone(); + Self { + inner, + accept_compression_encodings: self.accept_compression_encodings, + send_compression_encodings: self.send_compression_encodings, + max_decoding_message_size: self.max_decoding_message_size, + max_encoding_message_size: self.max_encoding_message_size, + } + } + } + impl Clone for _Inner { + fn clone(&self) -> Self { + Self(Arc::clone(&self.0)) + } + } + impl std::fmt::Debug for _Inner { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{:?}", self.0) + } + } + impl tonic::server::NamedService for FetchServer { + const NAME: &'static str = "sf.firehose.v2.Fetch"; + } +} +/// Generated server implementations. +pub mod endpoint_info_server { + #![allow(unused_variables, dead_code, missing_docs, clippy::let_unit_value)] + use tonic::codegen::*; + /// Generated trait containing gRPC methods that should be implemented for use with EndpointInfoServer. + #[async_trait] + pub trait EndpointInfo: Send + Sync + 'static { + async fn info( + &self, + request: tonic::Request, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; + } + #[derive(Debug)] + pub struct EndpointInfoServer { + inner: _Inner, + accept_compression_encodings: EnabledCompressionEncodings, + send_compression_encodings: EnabledCompressionEncodings, + max_decoding_message_size: Option, + max_encoding_message_size: Option, + } + struct _Inner(Arc); + impl EndpointInfoServer { + pub fn new(inner: T) -> Self { + Self::from_arc(Arc::new(inner)) + } + pub fn from_arc(inner: Arc) -> Self { + let inner = _Inner(inner); + Self { + inner, + accept_compression_encodings: Default::default(), + send_compression_encodings: Default::default(), + max_decoding_message_size: None, + max_encoding_message_size: None, + } + } + pub fn with_interceptor( + inner: T, + interceptor: F, + ) -> InterceptedService + where + F: tonic::service::Interceptor, + { + InterceptedService::new(Self::new(inner), interceptor) + } + /// Enable decompressing requests with the given encoding. + #[must_use] + pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.accept_compression_encodings.enable(encoding); + self + } + /// Compress responses with the given encoding, if the client supports it. + #[must_use] + pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.send_compression_encodings.enable(encoding); + self + } + /// Limits the maximum size of a decoded message. + /// + /// Default: `4MB` + #[must_use] + pub fn max_decoding_message_size(mut self, limit: usize) -> Self { + self.max_decoding_message_size = Some(limit); + self + } + /// Limits the maximum size of an encoded message. + /// + /// Default: `usize::MAX` + #[must_use] + pub fn max_encoding_message_size(mut self, limit: usize) -> Self { + self.max_encoding_message_size = Some(limit); + self + } + } + impl tonic::codegen::Service> for EndpointInfoServer + where + T: EndpointInfo, + B: Body + Send + 'static, + B::Error: Into + Send + 'static, + { + type Response = http::Response; + type Error = std::convert::Infallible; + type Future = BoxFuture; + fn poll_ready( + &mut self, + _cx: &mut Context<'_>, + ) -> Poll> { + Poll::Ready(Ok(())) + } + fn call(&mut self, req: http::Request) -> Self::Future { + let inner = self.inner.clone(); + match req.uri().path() { + "/sf.firehose.v2.EndpointInfo/Info" => { + #[allow(non_camel_case_types)] + struct InfoSvc(pub Arc); + impl< + T: EndpointInfo, + > tonic::server::UnaryService + for InfoSvc { + type Response = crate::firehose::InfoResponse; + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::info(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let inner = inner.0; + let method = InfoSvc(inner); + let codec = tonic::codec::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + _ => { + Box::pin(async move { + Ok( + http::Response::builder() + .status(200) + .header("grpc-status", "12") + .header("content-type", "application/grpc") + .body(empty_body()) + .unwrap(), + ) + }) + } + } + } + } + impl Clone for EndpointInfoServer { + fn clone(&self) -> Self { + let inner = self.inner.clone(); + Self { + inner, + accept_compression_encodings: self.accept_compression_encodings, + send_compression_encodings: self.send_compression_encodings, + max_decoding_message_size: self.max_decoding_message_size, + max_encoding_message_size: self.max_encoding_message_size, + } + } + } + impl Clone for _Inner { + fn clone(&self) -> Self { + Self(Arc::clone(&self.0)) + } + } + impl std::fmt::Debug for _Inner { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{:?}", self.0) + } + } + impl tonic::server::NamedService for EndpointInfoServer { + const NAME: &'static str = "sf.firehose.v2.EndpointInfo"; + } +} diff --git a/graph/src/substreams_rpc/sf.substreams.rpc.v2.rs b/graph/src/substreams_rpc/sf.substreams.rpc.v2.rs index 19b8d0493f0..38a793e666f 100644 --- a/graph/src/substreams_rpc/sf.substreams.rpc.v2.rs +++ b/graph/src/substreams_rpc/sf.substreams.rpc.v2.rs @@ -9,28 +9,32 @@ pub struct Request { #[prost(uint64, tag = "3")] pub stop_block_num: u64, /// With final_block_only, you only receive blocks that are irreversible: - /// 'final_block_height' will be equal to current block and no 'undo_signal' will ever be sent + /// 'final_block_height' will be equal to current block and no 'undo_signal' + /// will ever be sent #[prost(bool, tag = "4")] pub final_blocks_only: bool, - /// Substreams has two mode when executing your module(s) either development mode or production - /// mode. Development and production modes impact the execution of Substreams, important aspects - /// of execution include: + /// Substreams has two mode when executing your module(s) either development + /// mode or production mode. Development and production modes impact the + /// execution of Substreams, important aspects of execution include: /// * The time required to reach the first byte. /// * The speed that large ranges get executed. /// * The module logs and outputs sent back to the client. /// - /// By default, the engine runs in developer mode, with richer and deeper output. Differences - /// between production and development modes include: - /// * Forward parallel execution is enabled in production mode and disabled in development mode - /// * The time required to reach the first byte in development mode is faster than in production mode. + /// By default, the engine runs in developer mode, with richer and deeper + /// output. Differences between production and development modes include: + /// * Forward parallel execution is enabled in production mode and disabled in + /// development mode + /// * The time required to reach the first byte in development mode is faster + /// than in production mode. /// /// Specific attributes of development mode include: /// * The client will receive all of the executed module's logs. - /// * It's possible to request specific store snapshots in the execution tree (via `debug_initial_store_snapshot_for_modules`). + /// * It's possible to request specific store snapshots in the execution tree + /// (via `debug_initial_store_snapshot_for_modules`). /// * Multiple module's output is possible. /// - /// With production mode`, however, you trade off functionality for high speed enabling forward - /// parallel execution of module ahead of time. + /// With production mode`, however, you trade off functionality for high speed + /// enabling forward parallel execution of module ahead of time. #[prost(bool, tag = "5")] pub production_mode: bool, #[prost(string, tag = "6")] @@ -57,19 +61,22 @@ pub mod response { /// Always sent first #[prost(message, tag = "1")] Session(super::SessionInit), - /// Progress of data preparation, before sending in the stream of `data` events. + /// Progress of data preparation, before #[prost(message, tag = "2")] Progress(super::ModulesProgress), + /// sending in the stream of `data` events. #[prost(message, tag = "3")] BlockScopedData(super::BlockScopedData), #[prost(message, tag = "4")] BlockUndoSignal(super::BlockUndoSignal), #[prost(message, tag = "5")] FatalError(super::Error), - /// Available only in developer mode, and only if `debug_initial_store_snapshot_for_modules` is set. + /// Available only in developer mode, and only if + /// `debug_initial_store_snapshot_for_modules` is set. #[prost(message, tag = "10")] DebugSnapshotData(super::InitialSnapshotData), - /// Available only in developer mode, and only if `debug_initial_store_snapshot_for_modules` is set. + /// Available only in developer mode, and only if + /// `debug_initial_store_snapshot_for_modules` is set. #[prost(message, tag = "11")] DebugSnapshotComplete(super::InitialSnapshotComplete), } @@ -144,9 +151,9 @@ pub struct MapModuleOutput { pub debug_info: ::core::option::Option, } /// StoreModuleOutput are produced for store modules in development mode. -/// It is not possible to retrieve store models in production, with parallelization -/// enabled. If you need the deltas directly, write a pass through mapper module -/// that will get them down to you. +/// It is not possible to retrieve store models in production, with +/// parallelization enabled. If you need the deltas directly, write a pass +/// through mapper module that will get them down to you. #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct StoreModuleOutput { @@ -162,8 +169,9 @@ pub struct StoreModuleOutput { pub struct OutputDebugInfo { #[prost(string, repeated, tag = "1")] pub logs: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, - /// LogsTruncated is a flag that tells you if you received all the logs or if they - /// were truncated because you logged too much (fixed limit currently is set to 128 KiB). + /// LogsTruncated is a flag that tells you if you received all the logs or if + /// they were truncated because you logged too much (fixed limit currently is + /// set to 128 KiB). #[prost(bool, tag = "2")] pub logs_truncated: bool, #[prost(bool, tag = "3")] @@ -202,8 +210,9 @@ pub struct Error { pub reason: ::prost::alloc::string::String, #[prost(string, repeated, tag = "3")] pub logs: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, - /// FailureLogsTruncated is a flag that tells you if you received all the logs or if they - /// were truncated because you logged too much (fixed limit currently is set to 128 KiB). + /// FailureLogsTruncated is a flag that tells you if you received all the logs + /// or if they were truncated because you logged too much (fixed limit + /// currently is set to 128 KiB). #[prost(bool, tag = "4")] pub logs_truncated: bool, } @@ -229,8 +238,9 @@ pub struct Stage { #[prost(message, repeated, tag = "2")] pub completed_ranges: ::prost::alloc::vec::Vec, } -/// ModuleStats gathers metrics and statistics from each module, running on tier1 or tier2 -/// All the 'count' and 'time_ms' values may include duplicate for each stage going over that module +/// ModuleStats gathers metrics and statistics from each module, running on tier1 +/// or tier2 All the 'count' and 'time_ms' values may include duplicate for each +/// stage going over that module #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct ModuleStats { @@ -240,35 +250,44 @@ pub struct ModuleStats { /// total_processed_blocks is the sum of blocks sent to that module code #[prost(uint64, tag = "2")] pub total_processed_block_count: u64, - /// total_processing_time_ms is the sum of all time spent running that module code + /// total_processing_time_ms is the sum of all time spent running that module + /// code #[prost(uint64, tag = "3")] pub total_processing_time_ms: u64, /// // external_calls are chain-specific intrinsics, like "Ethereum RPC calls". #[prost(message, repeated, tag = "4")] pub external_call_metrics: ::prost::alloc::vec::Vec, - /// total_store_operation_time_ms is the sum of all time spent running that module code waiting for a store operation (ex: read, write, delete...) + /// total_store_operation_time_ms is the sum of all time spent running that + /// module code waiting for a store operation (ex: read, write, delete...) #[prost(uint64, tag = "5")] pub total_store_operation_time_ms: u64, - /// total_store_read_count is the sum of all the store Read operations called from that module code + /// total_store_read_count is the sum of all the store Read operations called + /// from that module code #[prost(uint64, tag = "6")] pub total_store_read_count: u64, - /// total_store_write_count is the sum of all store Write operations called from that module code (store-only) + /// total_store_write_count is the sum of all store Write operations called + /// from that module code (store-only) #[prost(uint64, tag = "10")] pub total_store_write_count: u64, - /// total_store_deleteprefix_count is the sum of all store DeletePrefix operations called from that module code (store-only) - /// note that DeletePrefix can be a costly operation on large stores + /// total_store_deleteprefix_count is the sum of all store DeletePrefix + /// operations called from that module code (store-only) note that DeletePrefix + /// can be a costly operation on large stores #[prost(uint64, tag = "11")] pub total_store_deleteprefix_count: u64, - /// store_size_bytes is the uncompressed size of the full KV store for that module, from the last 'merge' operation (store-only) + /// store_size_bytes is the uncompressed size of the full KV store for that + /// module, from the last 'merge' operation (store-only) #[prost(uint64, tag = "12")] pub store_size_bytes: u64, - /// total_store_merging_time_ms is the time spent merging partial stores into a full KV store for that module (store-only) + /// total_store_merging_time_ms is the time spent merging partial stores into a + /// full KV store for that module (store-only) #[prost(uint64, tag = "13")] pub total_store_merging_time_ms: u64, - /// store_currently_merging is true if there is a merging operation (partial store to full KV store) on the way. + /// store_currently_merging is true if there is a merging operation (partial + /// store to full KV store) on the way. #[prost(bool, tag = "14")] pub store_currently_merging: bool, - /// highest_contiguous_block is the highest block in the highest merged full KV store of that module (store-only) + /// highest_contiguous_block is the highest block in the highest merged full KV + /// store of that module (store-only) #[prost(uint64, tag = "15")] pub highest_contiguous_block: u64, } @@ -350,6 +369,118 @@ pub struct BlockRange { pub end_block: u64, } /// Generated client implementations. +pub mod endpoint_info_client { + #![allow(unused_variables, dead_code, missing_docs, clippy::let_unit_value)] + use tonic::codegen::*; + use tonic::codegen::http::Uri; + #[derive(Debug, Clone)] + pub struct EndpointInfoClient { + inner: tonic::client::Grpc, + } + impl EndpointInfoClient { + /// Attempt to create a new client by connecting to a given endpoint. + pub async fn connect(dst: D) -> Result + where + D: TryInto, + D::Error: Into, + { + let conn = tonic::transport::Endpoint::new(dst)?.connect().await?; + Ok(Self::new(conn)) + } + } + impl EndpointInfoClient + where + T: tonic::client::GrpcService, + T::Error: Into, + T::ResponseBody: Body + Send + 'static, + ::Error: Into + Send, + { + pub fn new(inner: T) -> Self { + let inner = tonic::client::Grpc::new(inner); + Self { inner } + } + pub fn with_origin(inner: T, origin: Uri) -> Self { + let inner = tonic::client::Grpc::with_origin(inner, origin); + Self { inner } + } + pub fn with_interceptor( + inner: T, + interceptor: F, + ) -> EndpointInfoClient> + where + F: tonic::service::Interceptor, + T::ResponseBody: Default, + T: tonic::codegen::Service< + http::Request, + Response = http::Response< + >::ResponseBody, + >, + >, + , + >>::Error: Into + Send + Sync, + { + EndpointInfoClient::new(InterceptedService::new(inner, interceptor)) + } + /// Compress requests with the given encoding. + /// + /// This requires the server to support it otherwise it might respond with an + /// error. + #[must_use] + pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.inner = self.inner.send_compressed(encoding); + self + } + /// Enable decompressing responses. + #[must_use] + pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.inner = self.inner.accept_compressed(encoding); + self + } + /// Limits the maximum size of a decoded message. + /// + /// Default: `4MB` + #[must_use] + pub fn max_decoding_message_size(mut self, limit: usize) -> Self { + self.inner = self.inner.max_decoding_message_size(limit); + self + } + /// Limits the maximum size of an encoded message. + /// + /// Default: `usize::MAX` + #[must_use] + pub fn max_encoding_message_size(mut self, limit: usize) -> Self { + self.inner = self.inner.max_encoding_message_size(limit); + self + } + pub async fn info( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/sf.substreams.rpc.v2.EndpointInfo/Info", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("sf.substreams.rpc.v2.EndpointInfo", "Info")); + self.inner.unary(req, path, codec).await + } + } +} +/// Generated client implementations. pub mod stream_client { #![allow(unused_variables, dead_code, missing_docs, clippy::let_unit_value)] use tonic::codegen::*; @@ -462,6 +593,187 @@ pub mod stream_client { } } /// Generated server implementations. +pub mod endpoint_info_server { + #![allow(unused_variables, dead_code, missing_docs, clippy::let_unit_value)] + use tonic::codegen::*; + /// Generated trait containing gRPC methods that should be implemented for use with EndpointInfoServer. + #[async_trait] + pub trait EndpointInfo: Send + Sync + 'static { + async fn info( + &self, + request: tonic::Request, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; + } + #[derive(Debug)] + pub struct EndpointInfoServer { + inner: _Inner, + accept_compression_encodings: EnabledCompressionEncodings, + send_compression_encodings: EnabledCompressionEncodings, + max_decoding_message_size: Option, + max_encoding_message_size: Option, + } + struct _Inner(Arc); + impl EndpointInfoServer { + pub fn new(inner: T) -> Self { + Self::from_arc(Arc::new(inner)) + } + pub fn from_arc(inner: Arc) -> Self { + let inner = _Inner(inner); + Self { + inner, + accept_compression_encodings: Default::default(), + send_compression_encodings: Default::default(), + max_decoding_message_size: None, + max_encoding_message_size: None, + } + } + pub fn with_interceptor( + inner: T, + interceptor: F, + ) -> InterceptedService + where + F: tonic::service::Interceptor, + { + InterceptedService::new(Self::new(inner), interceptor) + } + /// Enable decompressing requests with the given encoding. + #[must_use] + pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.accept_compression_encodings.enable(encoding); + self + } + /// Compress responses with the given encoding, if the client supports it. + #[must_use] + pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.send_compression_encodings.enable(encoding); + self + } + /// Limits the maximum size of a decoded message. + /// + /// Default: `4MB` + #[must_use] + pub fn max_decoding_message_size(mut self, limit: usize) -> Self { + self.max_decoding_message_size = Some(limit); + self + } + /// Limits the maximum size of an encoded message. + /// + /// Default: `usize::MAX` + #[must_use] + pub fn max_encoding_message_size(mut self, limit: usize) -> Self { + self.max_encoding_message_size = Some(limit); + self + } + } + impl tonic::codegen::Service> for EndpointInfoServer + where + T: EndpointInfo, + B: Body + Send + 'static, + B::Error: Into + Send + 'static, + { + type Response = http::Response; + type Error = std::convert::Infallible; + type Future = BoxFuture; + fn poll_ready( + &mut self, + _cx: &mut Context<'_>, + ) -> Poll> { + Poll::Ready(Ok(())) + } + fn call(&mut self, req: http::Request) -> Self::Future { + let inner = self.inner.clone(); + match req.uri().path() { + "/sf.substreams.rpc.v2.EndpointInfo/Info" => { + #[allow(non_camel_case_types)] + struct InfoSvc(pub Arc); + impl< + T: EndpointInfo, + > tonic::server::UnaryService + for InfoSvc { + type Response = crate::firehose::InfoResponse; + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::info(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let inner = inner.0; + let method = InfoSvc(inner); + let codec = tonic::codec::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + _ => { + Box::pin(async move { + Ok( + http::Response::builder() + .status(200) + .header("grpc-status", "12") + .header("content-type", "application/grpc") + .body(empty_body()) + .unwrap(), + ) + }) + } + } + } + } + impl Clone for EndpointInfoServer { + fn clone(&self) -> Self { + let inner = self.inner.clone(); + Self { + inner, + accept_compression_encodings: self.accept_compression_encodings, + send_compression_encodings: self.send_compression_encodings, + max_decoding_message_size: self.max_decoding_message_size, + max_encoding_message_size: self.max_encoding_message_size, + } + } + } + impl Clone for _Inner { + fn clone(&self) -> Self { + Self(Arc::clone(&self.0)) + } + } + impl std::fmt::Debug for _Inner { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{:?}", self.0) + } + } + impl tonic::server::NamedService for EndpointInfoServer { + const NAME: &'static str = "sf.substreams.rpc.v2.EndpointInfo"; + } +} +/// Generated server implementations. pub mod stream_server { #![allow(unused_variables, dead_code, missing_docs, clippy::let_unit_value)] use tonic::codegen::*; diff --git a/node/src/chain.rs b/node/src/chain.rs index 289c0580c2e..beeb366ad61 100644 --- a/node/src/chain.rs +++ b/node/src/chain.rs @@ -90,6 +90,7 @@ pub fn create_substreams_networks( firehose.compression_enabled(), SubgraphLimit::Unlimited, endpoint_metrics.clone(), + true, ))); } } @@ -157,6 +158,7 @@ pub fn create_firehose_networks( firehose.compression_enabled(), firehose.limit_for(&config.node), endpoint_metrics.cheap_clone(), + false, ))); } } diff --git a/tests/src/fixture/mod.rs b/tests/src/fixture/mod.rs index 4e8127875a0..1fc43e495b8 100644 --- a/tests/src/fixture/mod.rs +++ b/tests/src/fixture/mod.rs @@ -108,6 +108,7 @@ impl CommonChainConfig { false, SubgraphLimit::Unlimited, Arc::new(EndpointMetrics::mock()), + false, ))]); Self { From 717e007dc1d066471a344a22f0ea7e4513523f55 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Tue, 18 Feb 2025 11:37:10 -0800 Subject: [PATCH 002/150] all: Remove websocket server --- Cargo.lock | 46 +-- Cargo.toml | 17 +- node/Cargo.toml | 1 - node/src/main.rs | 6 - server/websocket/Cargo.toml | 11 - server/websocket/src/connection.rs | 436 ----------------------------- server/websocket/src/lib.rs | 4 - server/websocket/src/server.rs | 217 -------------- 8 files changed, 15 insertions(+), 723 deletions(-) delete mode 100644 server/websocket/Cargo.toml delete mode 100644 server/websocket/src/connection.rs delete mode 100644 server/websocket/src/lib.rs delete mode 100644 server/websocket/src/server.rs diff --git a/Cargo.lock b/Cargo.lock index b2ae24e0169..0d20d8900d3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -394,7 +394,7 @@ dependencies = [ "sha1", "sync_wrapper 1.0.1", "tokio", - "tokio-tungstenite 0.21.0", + "tokio-tungstenite", "tower 0.4.13 (registry+https://github.com/rust-lang/crates.io-index)", "tower-layer 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "tower-service 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2021,7 +2021,6 @@ dependencies = [ "graph-server-index-node", "graph-server-json-rpc", "graph-server-metrics", - "graph-server-websocket", "graph-store-postgres", "graphman", "graphman-server", @@ -2120,17 +2119,6 @@ dependencies = [ "graph", ] -[[package]] -name = "graph-server-websocket" -version = "0.36.0" -dependencies = [ - "graph", - "serde", - "serde_derive", - "tokio-tungstenite 0.23.1", - "uuid", -] - [[package]] name = "graph-store-postgres" version = "0.36.0" @@ -5291,19 +5279,7 @@ dependencies = [ "futures-util", "log", "tokio", - "tungstenite 0.21.0", -] - -[[package]] -name = "tokio-tungstenite" -version = "0.23.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6989540ced10490aaf14e6bad2e3d33728a2813310a0c71d1574304c49631cd" -dependencies = [ - "futures-util", - "log", - "tokio", - "tungstenite 0.23.0", + "tungstenite", ] [[package]] @@ -5598,24 +5574,6 @@ dependencies = [ "utf-8", ] -[[package]] -name = "tungstenite" -version = "0.23.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e2e2ce1e47ed2994fd43b04c8f618008d4cabdd5ee34027cf14f9d918edd9c8" -dependencies = [ - "byteorder", - "bytes", - "data-encoding", - "http 1.1.0", - "httparse", - "log", - "rand", - "sha1", - "thiserror", - "utf-8", -] - [[package]] name = "typenum" version = "1.17.0" diff --git a/Cargo.toml b/Cargo.toml index 751a19d6213..4cf78a3eea0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,10 +7,19 @@ members = [ "chain/*", "graphql", "node", - "runtime/*", - "server/*", - "store/*", - "substreams/*", + "runtime/derive", + "runtime/test", + "runtime/wasm", + "server/graphman", + "server/http", + "server/index-node", + "server/json-rpc", + "server/metrics", + "store/postgres", + "store/test-store", + "substreams/substreams-head-tracker", + "substreams/substreams-trigger-filter", + "substreams/trigger-filters", "graph", "tests", "graph/derive", diff --git a/node/Cargo.toml b/node/Cargo.toml index 820ed8405a8..812ab5fc3e0 100644 --- a/node/Cargo.toml +++ b/node/Cargo.toml @@ -31,7 +31,6 @@ graph-graphql = { path = "../graphql" } graph-server-http = { path = "../server/http" } graph-server-index-node = { path = "../server/index-node" } graph-server-json-rpc = { path = "../server/json-rpc" } -graph-server-websocket = { path = "../server/websocket" } graph-server-metrics = { path = "../server/metrics" } graph-store-postgres = { path = "../store/postgres" } graphman-server = { workspace = true } diff --git a/node/src/main.rs b/node/src/main.rs index 870cce97318..81aceb921d2 100644 --- a/node/src/main.rs +++ b/node/src/main.rs @@ -28,7 +28,6 @@ use graph_server_http::GraphQLServer as GraphQLQueryServer; use graph_server_index_node::IndexNodeServer; use graph_server_json_rpc::JsonRpcServer; use graph_server_metrics::PrometheusMetricsServer; -use graph_server_websocket::SubscriptionServer as GraphQLSubscriptionServer; use graph_store_postgres::connection_pool::ConnectionPool; use graph_store_postgres::Store; use graph_store_postgres::{register_jobs as register_store_jobs, NotificationSender}; @@ -362,8 +361,6 @@ async fn main() { graphql_metrics_registry, )); let graphql_server = GraphQLQueryServer::new(&logger_factory, graphql_runner.clone()); - let subscription_server = - GraphQLSubscriptionServer::new(&logger, graphql_runner.clone(), network_store.clone()); let index_node_server = IndexNodeServer::new( &logger_factory, @@ -510,9 +507,6 @@ async fn main() { // Serve GraphQL queries over HTTP graph::spawn(async move { graphql_server.start(http_port, ws_port).await }); - // Serve GraphQL subscriptions over WebSockets - graph::spawn(subscription_server.serve(ws_port)); - // Run the index node server graph::spawn(async move { index_node_server.start(index_node_port).await }); diff --git a/server/websocket/Cargo.toml b/server/websocket/Cargo.toml deleted file mode 100644 index 622682732c1..00000000000 --- a/server/websocket/Cargo.toml +++ /dev/null @@ -1,11 +0,0 @@ -[package] -name = "graph-server-websocket" -version.workspace = true -edition.workspace = true - -[dependencies] -graph = { path = "../../graph" } -serde = { workspace = true } -serde_derive = { workspace = true } -tokio-tungstenite = "0.23" -uuid = { version = "1.9.1", features = ["v4"] } diff --git a/server/websocket/src/connection.rs b/server/websocket/src/connection.rs deleted file mode 100644 index 571817703f9..00000000000 --- a/server/websocket/src/connection.rs +++ /dev/null @@ -1,436 +0,0 @@ -use graph::futures01::sync::mpsc; -use graph::futures01::{Future, IntoFuture, Sink as _, Stream as _}; -use graph::futures03::future::TryFutureExt; -use graph::futures03::sink::SinkExt; -use graph::futures03::stream::{SplitStream, StreamExt, TryStreamExt}; -use std::collections::HashMap; -use tokio::io::{AsyncRead, AsyncWrite}; -use tokio_tungstenite::tungstenite::{ - http::Response as WsResponse, http::StatusCode, Error as WsError, Message as WsMessage, -}; -use tokio_tungstenite::WebSocketStream; -use uuid::Uuid; - -use graph::futures03::compat::Future01CompatExt; -use graph::{data::query::QueryTarget, prelude::*}; - -#[derive(Debug, Deserialize, Serialize)] -#[serde(rename_all = "camelCase")] -struct StartPayload { - query: String, - variables: Option, - operation_name: Option, -} - -/// GraphQL/WebSocket message received from a client. -#[derive(Debug, Deserialize)] -#[serde(tag = "type", rename_all = "snake_case")] -enum IncomingMessage { - ConnectionInit { - #[allow(dead_code)] - payload: Option, - }, - ConnectionTerminate, - Start { - id: String, - payload: StartPayload, - }, - Stop { - id: String, - }, -} - -impl IncomingMessage { - pub fn from_ws_message(msg: WsMessage) -> Result { - let text = msg.into_text()?; - serde_json::from_str(text.as_str()).map_err(|e| { - let msg = - format!("Invalid GraphQL over WebSocket message: {}: {}", text, e).into_bytes(); - WsError::Http(WsResponse::new(Some(msg))) - }) - } -} - -/// GraphQL/WebSocket message to be sent to the client. -#[derive(Debug, Serialize)] -#[serde(tag = "type", rename_all = "snake_case")] -enum OutgoingMessage { - ConnectionAck, - Error { - id: String, - payload: String, - }, - Data { - id: String, - payload: Arc, - }, - Complete { - id: String, - }, -} - -impl OutgoingMessage { - pub fn from_query_result(id: String, result: Arc) -> Self { - OutgoingMessage::Data { - id, - payload: result, - } - } - - pub fn from_error_string(id: String, s: String) -> Self { - OutgoingMessage::Error { id, payload: s } - } -} - -impl From for WsMessage { - fn from(msg: OutgoingMessage) -> Self { - WsMessage::text(serde_json::to_string(&msg).expect("invalid GraphQL/WebSocket message")) - } -} - -/// Helper function to send outgoing messages. -fn send_message( - sink: &mpsc::UnboundedSender, - msg: OutgoingMessage, -) -> Result<(), WsError> { - sink.unbounded_send(msg.into()).map_err(|_| { - let mut response = WsResponse::new(None); - *response.status_mut() = StatusCode::INTERNAL_SERVER_ERROR; - WsError::Http(response) - }) -} - -/// Helper function to send error messages. -fn send_error_string( - sink: &mpsc::UnboundedSender, - operation_id: String, - error: String, -) -> Result<(), WsError> { - sink.unbounded_send(OutgoingMessage::from_error_string(operation_id, error).into()) - .map_err(|_| { - let mut response = WsResponse::new(None); - *response.status_mut() = StatusCode::INTERNAL_SERVER_ERROR; - WsError::Http(response) - }) -} - -/// Responsible for recording operation ids and stopping them. -/// On drop, cancels all operations. -struct Operations { - operations: HashMap, - msg_sink: mpsc::UnboundedSender, -} - -impl Operations { - fn new(msg_sink: mpsc::UnboundedSender) -> Self { - Self { - operations: HashMap::new(), - msg_sink, - } - } - - fn contains(&self, id: &str) -> bool { - self.operations.contains_key(id) - } - - fn insert(&mut self, id: String, guard: CancelGuard) { - self.operations.insert(id, guard); - } - - fn stop(&mut self, operation_id: String) -> Result<(), WsError> { - // Remove the operation with this ID from the known operations. - match self.operations.remove(&operation_id) { - Some(stopper) => { - // Cancel the subscription result stream. - stopper.cancel(); - - // Send a GQL_COMPLETE to indicate the operation is been completed. - send_message( - &self.msg_sink, - OutgoingMessage::Complete { - id: operation_id.clone(), - }, - ) - } - None => send_error_string( - &self.msg_sink, - operation_id.clone(), - format!("Unknown operation ID: {}", operation_id), - ), - } - } -} - -impl Drop for Operations { - fn drop(&mut self) { - let ids = Vec::from_iter(self.operations.keys().cloned()); - for id in ids { - // Discard errors, the connection is being shutdown anyways. - let _ = self.stop(id); - } - } -} - -/// A WebSocket connection implementing the GraphQL over WebSocket protocol. -pub struct GraphQlConnection { - id: String, - logger: Logger, - graphql_runner: Arc, - stream: WebSocketStream, - deployment: DeploymentHash, -} - -impl GraphQlConnection -where - Q: GraphQlRunner, - S: AsyncRead + AsyncWrite + Send + 'static + Unpin, -{ - /// Creates a new GraphQL subscription service. - pub(crate) fn new( - logger: &Logger, - deployment: DeploymentHash, - stream: WebSocketStream, - graphql_runner: Arc, - ) -> Self { - GraphQlConnection { - id: Uuid::new_v4().to_string(), - logger: logger.new(o!("component" => "GraphQlConnection")), - graphql_runner, - stream, - deployment, - } - } - - async fn handle_incoming_messages( - mut ws_stream: SplitStream>, - mut msg_sink: mpsc::UnboundedSender, - logger: Logger, - connection_id: String, - deployment: DeploymentHash, - graphql_runner: Arc, - ) -> Result<(), WsError> { - let mut operations = Operations::new(msg_sink.clone()); - - // Process incoming messages as long as the WebSocket is open - while let Some(ws_msg) = ws_stream.try_next().await? { - use self::IncomingMessage::*; - use self::OutgoingMessage::*; - - debug!(logger, "Received message"; - "connection" => &connection_id, - "msg" => format!("{}", ws_msg).as_str()); - - let msg = IncomingMessage::from_ws_message(ws_msg.clone())?; - - debug!(logger, "GraphQL/WebSocket message"; - "connection" => &connection_id, - "msg" => format!("{:?}", msg).as_str()); - - match msg { - // Always accept connection init requests - ConnectionInit { payload: _ } => send_message(&msg_sink, ConnectionAck), - - // When receiving a connection termination request - ConnectionTerminate => { - // Close the message sink - msg_sink.close().unwrap(); - - // Return an error here to terminate the connection - Err(WsError::ConnectionClosed) - } - - // When receiving a stop request - Stop { id } => operations.stop(id), - - // When receiving a start request - Start { id, payload } => { - // Respond with a GQL_ERROR if we already have an operation with this ID - if operations.contains(&id) { - return send_error_string( - &msg_sink, - id.clone(), - format!("Operation with ID already started: {}", id), - ); - } - - let max_ops = ENV_VARS.graphql.max_operations_per_connection; - if operations.operations.len() >= max_ops { - return send_error_string( - &msg_sink, - id, - format!("Reached the limit of {} operations per connection", max_ops), - ); - } - - // Parse the GraphQL query document; respond with a GQL_ERROR if - // the query is invalid - let query = match q::parse_query(&payload.query) { - Ok(query) => query.into_static(), - Err(e) => { - return send_error_string( - &msg_sink, - id, - format!("Invalid query: {}: {}", payload.query, e), - ); - } - }; - - // Parse the query variables, if present - let variables = match payload.variables { - None | Some(serde_json::Value::Null) => None, - Some(variables @ serde_json::Value::Object(_)) => { - match serde_json::from_value(variables.clone()) { - Ok(variables) => Some(variables), - Err(e) => { - return send_error_string( - &msg_sink, - id, - format!("Invalid variables provided: {}", e), - ); - } - } - } - _ => { - return send_error_string( - &msg_sink, - id, - "Invalid variables provided (must be an object)".to_string(), - ); - } - }; - - // Construct a subscription - let target = QueryTarget::Deployment(deployment.clone(), Default::default()); - let subscription = Subscription { - // Subscriptions currently do not benefit from the generational cache - // anyways, so don't bother passing a network. - query: Query::new(query, variables, false), - }; - - debug!(logger, "Start operation"; - "connection" => &connection_id, - "id" => &id); - - // Execute the GraphQL subscription - let error_sink = msg_sink.clone(); - let result_sink = msg_sink.clone(); - let result_id = id.clone(); - let err_id = id.clone(); - let err_connection_id = connection_id.clone(); - let err_logger = logger.clone(); - let run_subscription = graphql_runner - .cheap_clone() - .run_subscription(subscription, target) - .compat() - .map_err(move |e| { - debug!(err_logger, "Subscription error"; - "connection" => &err_connection_id, - "id" => &err_id, - "error" => format!("{:?}", e)); - - // Send errors back to the client as GQL_DATA - match e { - SubscriptionError::GraphQLError(e) => { - // Don't bug clients with transient `TooExpensive` errors, - // simply skip updating them - if !e - .iter() - .any(|err| matches!(err, QueryExecutionError::TooExpensive)) - { - let result = Arc::new(QueryResult::from(e)); - let msg = OutgoingMessage::from_query_result( - err_id.clone(), - result, - ); - - // An error means the client closed the websocket, ignore - // and let it be handled in the websocket loop above. - let _ = error_sink.unbounded_send(msg.into()); - } - } - }; - }) - .and_then(move |result_stream| { - // Send results back to the client as GQL_DATA - result_stream - .map(move |result| { - OutgoingMessage::from_query_result(result_id.clone(), result) - }) - .map(WsMessage::from) - .map(Ok) - .compat() - .forward(result_sink.sink_map_err(|_| ())) - .map(|_| ()) - }); - - // Setup cancelation. - let guard = CancelGuard::new(); - let logger = logger.clone(); - let cancel_id = id.clone(); - let connection_id = connection_id.clone(); - let run_subscription = - run_subscription.compat().cancelable(&guard, move || { - debug!(logger, "Stopped operation"; - "connection" => &connection_id, - "id" => &cancel_id); - Ok(()) - }); - operations.insert(id, guard); - - graph::spawn_allow_panic(run_subscription); - Ok(()) - } - }? - } - Ok(()) - } -} - -impl IntoFuture for GraphQlConnection -where - Q: GraphQlRunner, - S: AsyncRead + AsyncWrite + Send + 'static + Unpin, -{ - type Future = Box + Send>; - type Item = (); - type Error = (); - - fn into_future(self) -> Self::Future { - debug!(self.logger, "GraphQL over WebSocket connection opened"; "id" => &self.id); - - // Obtain sink/stream pair to send and receive WebSocket messages - let (ws_sink, ws_stream) = self.stream.split(); - - // Allocate a channel for writing - let (msg_sink, msg_stream) = mpsc::unbounded(); - - // Handle incoming messages asynchronously - let ws_reader = Self::handle_incoming_messages( - ws_stream, - msg_sink, - self.logger.clone(), - self.id.clone(), - self.deployment.clone(), - self.graphql_runner.clone(), - ); - - // Send outgoing messages asynchronously - let ws_writer = msg_stream.forward(ws_sink.compat().sink_map_err(|_| ())); - - // Silently swallow internal send results and errors. There is nothing - // we can do about these errors ourselves. Clients will be disconnected - // as a result of this but most will try to reconnect (GraphiQL for sure, - // Apollo maybe). - let ws_writer = ws_writer.map(|_| ()); - let ws_reader = Box::pin(ws_reader.map_err(|_| ())); - - // Return a future that is fulfilled when either we or the client close - // our/their end of the WebSocket stream - let logger = self.logger.clone(); - let id = self.id.clone(); - Box::new(ws_reader.compat().select(ws_writer).then(move |_| { - debug!(logger, "GraphQL over WebSocket connection closed"; "connection" => id); - Ok(()) - })) - } -} diff --git a/server/websocket/src/lib.rs b/server/websocket/src/lib.rs deleted file mode 100644 index 887fed506fe..00000000000 --- a/server/websocket/src/lib.rs +++ /dev/null @@ -1,4 +0,0 @@ -mod connection; -mod server; - -pub use self::server::SubscriptionServer; diff --git a/server/websocket/src/server.rs b/server/websocket/src/server.rs deleted file mode 100644 index 9e1178cf0d0..00000000000 --- a/server/websocket/src/server.rs +++ /dev/null @@ -1,217 +0,0 @@ -use crate::connection::GraphQlConnection; -use graph::futures01::IntoFuture as _; -use graph::futures03::compat::Future01CompatExt; -use graph::futures03::future::FutureExt; -use graph::{ - data::query::QueryTarget, - prelude::{SubscriptionServer as SubscriptionServerTrait, *}, -}; -use std::net::{IpAddr, Ipv4Addr, SocketAddr}; -use std::sync::Mutex; -use tokio::net::TcpListener; -use tokio_tungstenite::accept_hdr_async; -use tokio_tungstenite::tungstenite::handshake::server::Request; -use tokio_tungstenite::tungstenite::http::{ - header::ACCESS_CONTROL_ALLOW_ORIGIN, header::CONTENT_TYPE, HeaderValue, Response, StatusCode, -}; - -/// A GraphQL subscription server based on Hyper / Websockets. -pub struct SubscriptionServer { - logger: Logger, - graphql_runner: Arc, - store: Arc, -} - -impl SubscriptionServer -where - Q: GraphQlRunner, - S: QueryStoreManager, -{ - pub fn new(logger: &Logger, graphql_runner: Arc, store: Arc) -> Self { - SubscriptionServer { - logger: logger.new(o!("component" => "SubscriptionServer")), - graphql_runner, - store, - } - } - - async fn subgraph_id_from_url_path( - store: Arc, - path: &str, - ) -> Result, Error> { - fn target_from_name(name: String, api_version: ApiVersion) -> Option { - SubgraphName::new(name) - .ok() - .map(|sub_name| QueryTarget::Name(sub_name, api_version)) - } - - fn target_from_id(id: &str, api_version: ApiVersion) -> Option { - DeploymentHash::new(id) - .ok() - .map(|hash| QueryTarget::Deployment(hash, api_version)) - } - - async fn state( - store: Arc, - target: Option, - ) -> Option { - let target = match target { - Some(target) => target, - None => return None, - }; - match store.query_store(target, false).await.ok() { - Some(query_store) => query_store.deployment_state().await.ok(), - None => None, - } - } - - let path_segments = { - let mut segments = path.split('/'); - - // Remove leading '/' - let first_segment = segments.next(); - if first_segment != Some("") { - return Ok(None); - } - - segments.collect::>() - }; - - match path_segments.as_slice() { - &["subgraphs", "id", subgraph_id] => { - Ok(state(store, target_from_id(subgraph_id, ApiVersion::default())).await) - } - &["subgraphs", "name", _] | &["subgraphs", "name", _, _] => Ok(state( - store, - target_from_name(path_segments[2..].join("/"), ApiVersion::default()), // TODO: version - ) - .await), - &["subgraphs", "network", _, _] => Ok(state( - store, - target_from_name(path_segments[1..].join("/"), ApiVersion::default()), // TODO: version - ) - .await), - _ => Ok(None), - } - } -} - -#[async_trait] -impl SubscriptionServerTrait for SubscriptionServer -where - Q: GraphQlRunner, - S: QueryStoreManager, -{ - async fn serve(self, port: u16) { - info!( - self.logger, - "Starting GraphQL WebSocket server at: ws://localhost:{}", port - ); - - let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), port); - let socket = TcpListener::bind(&addr) - .await - .expect("Failed to bind WebSocket port"); - - loop { - let stream = match socket.accept().await { - Ok((stream, _)) => stream, - Err(e) => { - trace!(self.logger, "Connection error: {}", e); - continue; - } - }; - let logger = self.logger.clone(); - let logger2 = self.logger.clone(); - let graphql_runner = self.graphql_runner.clone(); - let store = self.store.clone(); - - // Subgraph that the request is resolved to (if any) - let subgraph_id = Arc::new(Mutex::new(None)); - let accept_subgraph_id = subgraph_id.clone(); - - accept_hdr_async(stream, move |request: &Request, mut response: Response<()>| { - // Try to obtain the subgraph ID or name from the URL path. - // Return a 404 if the URL path contains no name/ID segment. - let path = request.uri().path(); - - // `block_in_place` is not recommended but in this case we have no alternative since - // we're in an async context but `tokio_tungstenite` doesn't allow this callback - // to be a future. - let state = tokio::task::block_in_place(|| { - graph::block_on(Self::subgraph_id_from_url_path( - store.clone(), - path, - )) - }) - .map_err(|e| { - error!( - logger, - "Error resolving subgraph ID from URL path"; - "error" => e.to_string() - ); - - Response::builder() - .status(StatusCode::INTERNAL_SERVER_ERROR) - .header(ACCESS_CONTROL_ALLOW_ORIGIN, "*") - .header(CONTENT_TYPE, "text/plain") - .body(None) - .unwrap() - }) - .and_then(|state| { - state.ok_or_else(|| { - Response::builder() - .status(StatusCode::NOT_FOUND) - .header(ACCESS_CONTROL_ALLOW_ORIGIN, "*") - .header(CONTENT_TYPE, "text/plain") - .body(None) - .unwrap() - }) - })?; - - // Check if the subgraph is deployed - if !state.is_deployed() { - error!(logger, "Failed to establish WS connection, no data found for subgraph"; - "subgraph_id" => state.id.to_string(), - ); - return Err(Response::builder() - .status(StatusCode::NOT_FOUND) - .header(ACCESS_CONTROL_ALLOW_ORIGIN, "*") - .header(CONTENT_TYPE, "text/plain") - .body(None) - .unwrap()); - } - - *accept_subgraph_id.lock().unwrap() = Some(state.id); - response.headers_mut().insert( - "Sec-WebSocket-Protocol", - HeaderValue::from_static("graphql-ws"), - ); - Ok(response) - }) - .then(move |result| async move { - match result { - Ok(ws_stream) => { - // Obtain the subgraph ID or name that we resolved the request to - let subgraph_id = subgraph_id.lock().unwrap().clone().unwrap(); - - // Spawn a GraphQL over WebSocket connection - let service = GraphQlConnection::new( - &logger2, - subgraph_id, - ws_stream, - graphql_runner.clone(), - ); - - graph::spawn_allow_panic(service.into_future().compat()); - } - Err(e) => { - // We gracefully skip over failed connection attempts rather - // than tearing down the entire stream - trace!(logger2, "Failed to establish WebSocket connection: {}", e); - } - } - }).await - } - } -} From 39efbe48b97079c1ce30fde8d49589f1a7383b41 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Tue, 18 Feb 2025 11:52:34 -0800 Subject: [PATCH 003/150] all: Remove subscriptions from GraphQlRunner This also removes all testing of subscription functionality --- graph/src/components/graphql.rs | 9 +- graphql/src/lib.rs | 4 - graphql/src/runner.rs | 75 +---- graphql/src/store/query.rs | 59 +--- graphql/src/store/resolver.rs | 54 +--- graphql/src/subscription/mod.rs | 256 ----------------- node/src/bin/manager.rs | 15 +- node/src/main.rs | 1 - node/src/manager/commands/query.rs | 4 +- server/http/src/service.rs | 8 - server/http/tests/server.rs | 8 - store/test-store/src/store.rs | 2 - store/test-store/tests/graphql/query.rs | 143 +--------- store/test-store/tests/postgres/store.rs | 334 +---------------------- tests/src/fixture/mod.rs | 4 +- 15 files changed, 31 insertions(+), 945 deletions(-) delete mode 100644 graphql/src/subscription/mod.rs diff --git a/graph/src/components/graphql.rs b/graph/src/components/graphql.rs index c5abf39b275..9ff8fc62261 100644 --- a/graph/src/components/graphql.rs +++ b/graph/src/components/graphql.rs @@ -1,6 +1,6 @@ use crate::data::query::QueryResults; use crate::data::query::{Query, QueryTarget}; -use crate::data::subscription::{Subscription, SubscriptionError, SubscriptionResult}; +use crate::data::subscription::{SubscriptionError, SubscriptionResult}; use crate::prelude::DeploymentHash; use async_trait::async_trait; @@ -33,13 +33,6 @@ pub trait GraphQlRunner: Send + Sync + 'static { max_skip: Option, ) -> QueryResults; - /// Runs a GraphQL subscription and returns a stream of results. - async fn run_subscription( - self: Arc, - subscription: Subscription, - target: QueryTarget, - ) -> Result; - fn metrics(&self) -> Arc; } diff --git a/graphql/src/lib.rs b/graphql/src/lib.rs index 7a3070b3844..03626eb907e 100644 --- a/graphql/src/lib.rs +++ b/graphql/src/lib.rs @@ -7,9 +7,6 @@ mod execution; /// Utilities for executing GraphQL queries and working with query ASTs. pub mod query; -/// Utilities for executing GraphQL subscriptions. -pub mod subscription; - /// Utilities for working with GraphQL values. mod values; @@ -28,7 +25,6 @@ pub mod prelude { pub use super::introspection::IntrospectionResolver; pub use super::query::{execute_query, ext::BlockConstraint, QueryExecutionOptions}; pub use super::store::StoreResolver; - pub use super::subscription::SubscriptionExecutionOptions; pub use super::values::MaybeCoercible; pub use super::metrics::GraphQLMetrics; diff --git a/graphql/src/runner.rs b/graphql/src/runner.rs index 79a13b0e04e..a70f1e752ac 100644 --- a/graphql/src/runner.rs +++ b/graphql/src/runner.rs @@ -2,18 +2,13 @@ use std::sync::Arc; use std::time::Instant; use crate::metrics::GraphQLMetrics; -use crate::prelude::{QueryExecutionOptions, StoreResolver, SubscriptionExecutionOptions}; +use crate::prelude::{QueryExecutionOptions, StoreResolver}; use crate::query::execute_query; -use crate::subscription::execute_prepared_subscription; use graph::futures03::future; use graph::prelude::MetricsRegistry; -use graph::{ - components::store::SubscriptionManager, - prelude::{ - async_trait, o, CheapClone, DeploymentState, GraphQLMetrics as GraphQLMetricsTrait, - GraphQlRunner as GraphQlRunnerTrait, Logger, Query, QueryExecutionError, Subscription, - SubscriptionError, SubscriptionResult, ENV_VARS, - }, +use graph::prelude::{ + async_trait, o, CheapClone, DeploymentState, GraphQLMetrics as GraphQLMetricsTrait, + GraphQlRunner as GraphQlRunnerTrait, Logger, Query, QueryExecutionError, ENV_VARS, }; use graph::{data::graphql::load_manager::LoadManager, prelude::QueryStoreManager}; use graph::{ @@ -22,10 +17,9 @@ use graph::{ }; /// GraphQL runner implementation for The Graph. -pub struct GraphQlRunner { +pub struct GraphQlRunner { logger: Logger, store: Arc, - subscription_manager: Arc, load_manager: Arc, graphql_metrics: Arc, } @@ -36,16 +30,14 @@ lazy_static::lazy_static! { pub static ref INITIAL_DEPLOYMENT_STATE_FOR_TESTS: std::sync::Mutex> = std::sync::Mutex::new(None); } -impl GraphQlRunner +impl GraphQlRunner where S: QueryStoreManager, - SM: SubscriptionManager, { /// Creates a new query runner. pub fn new( logger: &Logger, store: Arc, - subscription_manager: Arc, load_manager: Arc, registry: Arc, ) -> Self { @@ -54,7 +46,6 @@ where GraphQlRunner { logger, store, - subscription_manager, load_manager, graphql_metrics, } @@ -173,7 +164,6 @@ where &self.logger, store.cheap_clone(), &state, - self.subscription_manager.cheap_clone(), ptr, error_policy, query.schema.id().clone(), @@ -220,10 +210,9 @@ where } #[async_trait] -impl GraphQlRunnerTrait for GraphQlRunner +impl GraphQlRunnerTrait for GraphQlRunner where S: QueryStoreManager, - SM: SubscriptionManager, { async fn run_query(self: Arc, query: Query, target: QueryTarget) -> QueryResults { self.run_query_with_complexity( @@ -259,56 +248,6 @@ where .unwrap_or_else(|e| e) } - async fn run_subscription( - self: Arc, - subscription: Subscription, - target: QueryTarget, - ) -> Result { - let store = self.store.query_store(target.clone(), true).await?; - let schema = store.api_schema()?; - let network = store.network_name().to_string(); - - let query = crate::execution::Query::new( - &self.logger, - schema, - Some(network), - subscription.query, - ENV_VARS.graphql.max_complexity, - ENV_VARS.graphql.max_depth, - self.graphql_metrics.cheap_clone(), - )?; - - if let Err(err) = self - .load_manager - .decide( - &store.wait_stats().map_err(QueryExecutionError::from)?, - store.shard(), - store.deployment_id(), - query.shape_hash, - query.query_text.as_ref(), - ) - .to_result() - { - return Err(SubscriptionError::GraphQLError(vec![err])); - } - - execute_prepared_subscription( - query, - SubscriptionExecutionOptions { - logger: self.logger.clone(), - store, - subscription_manager: self.subscription_manager.cheap_clone(), - timeout: ENV_VARS.graphql.query_timeout, - max_complexity: ENV_VARS.graphql.max_complexity, - max_depth: ENV_VARS.graphql.max_depth, - max_first: ENV_VARS.graphql.max_first, - max_skip: ENV_VARS.graphql.max_skip, - graphql_metrics: self.graphql_metrics.clone(), - load_manager: self.load_manager.cheap_clone(), - }, - ) - } - fn metrics(&self) -> Arc { self.graphql_metrics.clone() } diff --git a/graphql/src/store/query.rs b/graphql/src/store/query.rs index 2c139152f86..bd74141c421 100644 --- a/graphql/src/store/query.rs +++ b/graphql/src/store/query.rs @@ -1,4 +1,3 @@ -use std::collections::{BTreeSet, HashSet, VecDeque}; use std::mem::discriminant; use graph::cheap_clone::CheapClone; @@ -8,12 +7,12 @@ use graph::components::store::{ }; use graph::data::graphql::TypeExt as _; use graph::data::query::QueryExecutionError; -use graph::data::store::{Attribute, SubscriptionFilter, Value, ValueType}; +use graph::data::store::{Attribute, Value, ValueType}; use graph::data::value::Object; use graph::data::value::Value as DataValue; -use graph::prelude::{r, s, TryFromValue, ENV_VARS}; +use graph::prelude::{r, TryFromValue, ENV_VARS}; use graph::schema::ast::{self as sast, FilterOp}; -use graph::schema::{ApiSchema, EntityType, InputSchema, ObjectOrInterface}; +use graph::schema::{EntityType, InputSchema, ObjectOrInterface}; use crate::execution::ast as a; @@ -652,58 +651,6 @@ fn build_order_direction(field: &a::Field) -> Result Result, QueryExecutionError> { - // Output entities - let mut entities = HashSet::new(); - - // List of objects/fields to visit next - let mut queue = VecDeque::new(); - queue.push_back((object_type, field)); - - while let Some((object_type, field)) = queue.pop_front() { - // Check if the field exists on the object type - if let Some(field_type) = sast::get_field(&object_type, &field.name) { - // Check if the field type corresponds to a type definition (in a valid schema, - // this should always be the case) - if let Some(type_definition) = schema.get_type_definition_from_field(field_type) { - // If the field's type definition is an object type, extract that type - if let s::TypeDefinition::Object(object_type) = type_definition { - // Only collect whether the field's type has an @entity directive - if sast::get_object_type_directive(object_type, String::from("entity")) - .is_some() - { - entities - .insert((input_schema.id().cheap_clone(), object_type.name.clone())); - } - - // If the query field has a non-empty selection set, this means we - // need to recursively process it - let object_type = schema.object_type(object_type).into(); - for sub_field in field.selection_set.fields_for(&object_type)? { - queue.push_back((object_type.cheap_clone(), sub_field)) - } - } - } - } - } - - entities - .into_iter() - .map(|(id, entity_type)| { - input_schema - .entity_type(&entity_type) - .map(|entity_type| SubscriptionFilter::Entities(id, entity_type)) - }) - .collect::>() - .map_err(Into::into) -} - #[cfg(test)] mod tests { use graph::components::store::EntityQuery; diff --git a/graphql/src/store/resolver.rs b/graphql/src/store/resolver.rs index a112fc97ae3..82c40420fa6 100644 --- a/graphql/src/store/resolver.rs +++ b/graphql/src/store/resolver.rs @@ -1,9 +1,8 @@ use std::collections::BTreeMap; -use std::result; use std::sync::Arc; use graph::components::graphql::GraphQLMetrics as _; -use graph::components::store::{QueryPermit, SubscriptionManager, UnitStream}; +use graph::components::store::QueryPermit; use graph::data::graphql::load_manager::LoadManager; use graph::data::graphql::{object, ObjectOrInterface}; use graph::data::query::{CacheStatus, QueryResults, Trace}; @@ -12,8 +11,8 @@ use graph::data::value::{Object, Word}; use graph::derive::CheapClone; use graph::prelude::*; use graph::schema::{ - ast as sast, ApiSchema, INTROSPECTION_SCHEMA_FIELD_NAME, INTROSPECTION_TYPE_FIELD_NAME, - META_FIELD_NAME, META_FIELD_TYPE, + ast as sast, INTROSPECTION_SCHEMA_FIELD_NAME, INTROSPECTION_TYPE_FIELD_NAME, META_FIELD_NAME, + META_FIELD_TYPE, }; use graph::schema::{ErrorPolicy, BLOCK_FIELD_TYPE}; @@ -21,7 +20,6 @@ use crate::execution::{ast as a, Query}; use crate::metrics::GraphQLMetrics; use crate::prelude::{ExecutionContext, Resolver}; use crate::query::ext::BlockConstraint; -use crate::store::query::collect_entities_from_query_field; /// A resolver that fetches entities from a `Store`. #[derive(Clone, CheapClone)] @@ -29,7 +27,6 @@ pub struct StoreResolver { #[allow(dead_code)] logger: Logger, pub(crate) store: Arc, - subscription_manager: Arc, pub(crate) block_ptr: Option, deployment: DeploymentHash, has_non_fatal_errors: bool, @@ -39,33 +36,6 @@ pub struct StoreResolver { } impl StoreResolver { - /// Create a resolver that looks up entities at whatever block is the - /// latest when the query is run. That means that multiple calls to find - /// entities into this resolver might return entities from different - /// blocks - pub fn for_subscription( - logger: &Logger, - deployment: DeploymentHash, - store: Arc, - subscription_manager: Arc, - graphql_metrics: Arc, - load_manager: Arc, - ) -> Self { - StoreResolver { - logger: logger.new(o!("component" => "StoreResolver")), - store, - subscription_manager, - block_ptr: None, - deployment, - - // Checking for non-fatal errors does not work with subscriptions. - has_non_fatal_errors: false, - error_policy: ErrorPolicy::Deny, - graphql_metrics, - load_manager, - } - } - /// Create a resolver that looks up entities at the block specified /// by `bc`. Any calls to find objects will always return entities as /// of that block. Note that if `bc` is `BlockConstraint::Latest` we use @@ -75,7 +45,6 @@ impl StoreResolver { logger: &Logger, store: Arc, state: &DeploymentState, - subscription_manager: Arc, block_ptr: BlockPtr, error_policy: ErrorPolicy, deployment: DeploymentHash, @@ -90,7 +59,6 @@ impl StoreResolver { let resolver = StoreResolver { logger: logger.new(o!("component" => "StoreResolver")), store, - subscription_manager, block_ptr: Some(block_ptr), deployment, has_non_fatal_errors, @@ -380,22 +348,6 @@ impl Resolver for StoreResolver { } } - fn resolve_field_stream( - &self, - schema: &ApiSchema, - object_type: &s::ObjectType, - field: &a::Field, - ) -> result::Result { - // Collect all entities involved in the query field - let object_type = schema.object_type(object_type).into(); - let input_schema = self.store.input_schema()?; - let entities = - collect_entities_from_query_field(&input_schema, schema, object_type, field)?; - - // Subscribe to the store and return the entity change stream - Ok(self.subscription_manager.subscribe_no_payload(entities)) - } - fn post_process(&self, result: &mut QueryResult) -> Result<(), anyhow::Error> { // Post-processing is only necessary for queries with indexing errors, and no query errors. if !self.has_non_fatal_errors || result.has_errors() { diff --git a/graphql/src/subscription/mod.rs b/graphql/src/subscription/mod.rs deleted file mode 100644 index ef0fc7b53ce..00000000000 --- a/graphql/src/subscription/mod.rs +++ /dev/null @@ -1,256 +0,0 @@ -use std::result::Result; -use std::time::{Duration, Instant}; - -use graph::components::store::UnitStream; -use graph::data::graphql::load_manager::LoadManager; -use graph::futures03::future::FutureExt; -use graph::futures03::stream::StreamExt; -use graph::schema::ApiSchema; -use graph::{components::store::SubscriptionManager, prelude::*, schema::ErrorPolicy}; - -use crate::metrics::GraphQLMetrics; -use crate::{execution::ast as a, execution::*, prelude::StoreResolver}; - -/// Options available for subscription execution. -pub struct SubscriptionExecutionOptions { - /// The logger to use during subscription execution. - pub logger: Logger, - - /// The store to use. - pub store: Arc, - - pub subscription_manager: Arc, - - /// Individual timeout for each subscription query. - pub timeout: Option, - - /// Maximum complexity for a subscription query. - pub max_complexity: Option, - - /// Maximum depth for a subscription query. - pub max_depth: u8, - - /// Maximum value for the `first` argument. - pub max_first: u32, - - /// Maximum value for the `skip` argument. - pub max_skip: u32, - - pub graphql_metrics: Arc, - - pub load_manager: Arc, -} - -pub fn execute_subscription( - subscription: Subscription, - schema: Arc, - options: SubscriptionExecutionOptions, -) -> Result { - let query = crate::execution::Query::new( - &options.logger, - schema, - None, - subscription.query, - options.max_complexity, - options.max_depth, - options.graphql_metrics.cheap_clone(), - )?; - execute_prepared_subscription(query, options) -} - -pub(crate) fn execute_prepared_subscription( - query: Arc, - options: SubscriptionExecutionOptions, -) -> Result { - if !query.is_subscription() { - return Err(SubscriptionError::from(QueryExecutionError::NotSupported( - "Only subscriptions are supported".to_string(), - ))); - } - - info!( - options.logger, - "Execute subscription"; - "query" => &query.query_text, - ); - - let source_stream = create_source_event_stream(query.clone(), &options)?; - let response_stream = map_source_to_response_stream(query, options, source_stream); - Ok(response_stream) -} - -fn create_source_event_stream( - query: Arc, - options: &SubscriptionExecutionOptions, -) -> Result { - let resolver = StoreResolver::for_subscription( - &options.logger, - query.schema.id().clone(), - options.store.clone(), - options.subscription_manager.cheap_clone(), - options.graphql_metrics.cheap_clone(), - options.load_manager.cheap_clone(), - ); - let ctx = ExecutionContext { - logger: options.logger.cheap_clone(), - resolver, - query, - deadline: None, - max_first: options.max_first, - max_skip: options.max_skip, - cache_status: Default::default(), - trace: ENV_VARS.log_sql_timing(), - }; - - let subscription_type = ctx - .query - .schema - .subscription_type - .as_ref() - .ok_or(QueryExecutionError::NoRootSubscriptionObjectType)?; - - let field = if ctx.query.selection_set.is_empty() { - return Err(SubscriptionError::from(QueryExecutionError::EmptyQuery)); - } else { - match ctx.query.selection_set.single_field() { - Some(field) => field, - None => { - return Err(SubscriptionError::from( - QueryExecutionError::MultipleSubscriptionFields, - )); - } - } - }; - - resolve_field_stream(&ctx, subscription_type, field) -} - -fn resolve_field_stream( - ctx: &ExecutionContext, - object_type: &s::ObjectType, - field: &a::Field, -) -> Result { - ctx.resolver - .resolve_field_stream(&ctx.query.schema, object_type, field) - .map_err(SubscriptionError::from) -} - -fn map_source_to_response_stream( - query: Arc, - options: SubscriptionExecutionOptions, - source_stream: UnitStream, -) -> QueryResultStream { - // Create a stream with a single empty event. By chaining this in front - // of the real events, we trick the subscription into executing its query - // at least once. This satisfies the GraphQL over Websocket protocol - // requirement of "respond[ing] with at least one GQL_DATA message", see - // https://github.com/apollographql/subscriptions-transport-ws/blob/master/PROTOCOL.md#gql_data - let trigger_stream = graph::futures03::stream::once(async {}); - - let SubscriptionExecutionOptions { - logger, - store, - subscription_manager, - timeout, - max_complexity: _, - max_depth: _, - max_first, - max_skip, - graphql_metrics, - load_manager, - } = options; - - trigger_stream - .chain(source_stream) - .then(move |()| { - execute_subscription_event( - logger.clone(), - store.clone(), - subscription_manager.cheap_clone(), - query.clone(), - timeout, - max_first, - max_skip, - graphql_metrics.cheap_clone(), - load_manager.cheap_clone(), - ) - .boxed() - }) - .boxed() -} - -async fn execute_subscription_event( - logger: Logger, - store: Arc, - subscription_manager: Arc, - query: Arc, - timeout: Option, - max_first: u32, - max_skip: u32, - metrics: Arc, - load_manager: Arc, -) -> Arc { - async fn make_resolver( - store: Arc, - logger: &Logger, - subscription_manager: Arc, - query: &Arc, - metrics: Arc, - load_manager: Arc, - ) -> Result { - let state = store.deployment_state().await?; - StoreResolver::at_block( - logger, - store, - &state, - subscription_manager, - state.latest_block.clone(), - ErrorPolicy::Deny, - query.schema.id().clone(), - metrics, - load_manager, - ) - .await - } - - let resolver = match make_resolver( - store, - &logger, - subscription_manager, - &query, - metrics, - load_manager, - ) - .await - { - Ok(resolver) => resolver, - Err(e) => return Arc::new(e.into()), - }; - - let block_ptr = resolver.block_ptr.clone(); - - // Create a fresh execution context with deadline. - let ctx = Arc::new(ExecutionContext { - logger, - resolver, - query, - deadline: timeout.map(|t| Instant::now() + t), - max_first, - max_skip, - cache_status: Default::default(), - trace: ENV_VARS.log_sql_timing(), - }); - - let subscription_type = match ctx.query.schema.subscription_type.as_ref() { - Some(t) => t.cheap_clone(), - None => return Arc::new(QueryExecutionError::NoRootSubscriptionObjectType.into()), - }; - - execute_root_selection_set( - ctx.cheap_clone(), - ctx.query.selection_set.cheap_clone(), - subscription_type.into(), - block_ptr, - ) - .await -} diff --git a/node/src/bin/manager.rs b/node/src/bin/manager.rs index 902241d3d54..8024bacb183 100644 --- a/node/src/bin/manager.rs +++ b/node/src/bin/manager.rs @@ -24,9 +24,7 @@ use graph_node::manager::color::Terminal; use graph_node::manager::commands; use graph_node::network_setup::Networks; use graph_node::{ - manager::{deployment::DeploymentSearch, PanicSubscriptionManager}, - store_builder::StoreBuilder, - MetricsContext, + manager::deployment::DeploymentSearch, store_builder::StoreBuilder, MetricsContext, }; use graph_store_postgres::connection_pool::PoolCoordinator; use graph_store_postgres::ChainStore; @@ -998,22 +996,15 @@ impl Context { (store.block_store(), primary.clone()) } - fn graphql_runner(self) -> Arc> { + fn graphql_runner(self) -> Arc> { let logger = self.logger.clone(); let registry = self.registry.clone(); let store = self.store(); - let subscription_manager = Arc::new(PanicSubscriptionManager); let load_manager = Arc::new(LoadManager::new(&logger, vec![], vec![], registry.clone())); - Arc::new(GraphQlRunner::new( - &logger, - store, - subscription_manager, - load_manager, - registry, - )) + Arc::new(GraphQlRunner::new(&logger, store, load_manager, registry)) } async fn networks(&self) -> anyhow::Result { diff --git a/node/src/main.rs b/node/src/main.rs index 81aceb921d2..72ff4f9ee27 100644 --- a/node/src/main.rs +++ b/node/src/main.rs @@ -356,7 +356,6 @@ async fn main() { let graphql_runner = Arc::new(GraphQlRunner::new( &logger, network_store.clone(), - subscription_manager.clone(), load_manager, graphql_metrics_registry, )); diff --git a/node/src/manager/commands/query.rs b/node/src/manager/commands/query.rs index 879e0eaf4a4..6339b7bf9cc 100644 --- a/node/src/manager/commands/query.rs +++ b/node/src/manager/commands/query.rs @@ -16,10 +16,8 @@ use graph::{ use graph_graphql::prelude::GraphQlRunner; use graph_store_postgres::Store; -use crate::manager::PanicSubscriptionManager; - pub async fn run( - runner: Arc>, + runner: Arc>, target: String, query: String, vars: Vec, diff --git a/server/http/src/service.rs b/server/http/src/service.rs index c806b9f2b65..ca03893b534 100644 --- a/server/http/src/service.rs +++ b/server/http/src/service.rs @@ -449,14 +449,6 @@ mod tests { )) } - async fn run_subscription( - self: Arc, - _subscription: Subscription, - _target: QueryTarget, - ) -> Result { - unreachable!(); - } - fn metrics(&self) -> Arc { Arc::new(TestGraphQLMetrics) } diff --git a/server/http/tests/server.rs b/server/http/tests/server.rs index a62a27a6c59..9970420ed2d 100644 --- a/server/http/tests/server.rs +++ b/server/http/tests/server.rs @@ -63,14 +63,6 @@ impl GraphQlRunner for TestGraphQlRunner { .into() } - async fn run_subscription( - self: Arc, - _subscription: Subscription, - _target: QueryTarget, - ) -> Result { - unreachable!(); - } - fn metrics(&self) -> Arc { Arc::new(TestGraphQLMetrics) } diff --git a/store/test-store/src/store.rs b/store/test-store/src/store.rs index 70fc26a3dde..ca0e97f7c2f 100644 --- a/store/test-store/src/store.rs +++ b/store/test-store/src/store.rs @@ -68,7 +68,6 @@ lazy_static! { pub static ref PRIMARY_POOL: ConnectionPool = STORE_POOL_CONFIG.1.clone(); pub static ref STORE: Arc = STORE_POOL_CONFIG.0.clone(); static ref CONFIG: Config = STORE_POOL_CONFIG.2.clone(); - pub static ref SUBSCRIPTION_MANAGER: Arc = STORE_POOL_CONFIG.3.clone(); pub static ref NODE_ID: NodeId = NodeId::new("test").unwrap(); pub static ref SUBGRAPH_STORE: Arc = STORE.subgraph_store(); static ref BLOCK_STORE: Arc = STORE.block_store(); @@ -544,7 +543,6 @@ async fn execute_subgraph_query_internal( &logger, store.clone(), &state, - SUBSCRIPTION_MANAGER.clone(), ptr, error_policy, query.schema.id().clone(), diff --git a/store/test-store/tests/graphql/query.rs b/store/test-store/tests/graphql/query.rs index d7e7dec8f55..d5e82834a1e 100644 --- a/store/test-store/tests/graphql/query.rs +++ b/store/test-store/tests/graphql/query.rs @@ -4,12 +4,12 @@ use graph::data::store::scalar::Timestamp; use graph::data::subgraph::schema::DeploymentCreate; use graph::data::subgraph::LATEST_VERSION; use graph::entity; -use graph::prelude::{SubscriptionResult, Value}; +use graph::prelude::Value; use graph::schema::InputSchema; use std::iter::FromIterator; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; -use std::time::{Duration, Instant}; +use std::time::Instant; use std::{ collections::{BTreeSet, HashMap}, marker::PhantomData, @@ -18,7 +18,6 @@ use test_store::block_store::{ FakeBlock, BLOCK_FOUR, BLOCK_ONE, BLOCK_THREE, BLOCK_TWO, GENESIS_BLOCK, }; -use graph::futures03::stream::StreamExt; use graph::{ components::store::DeploymentLocator, data::graphql::{object, object_value}, @@ -28,17 +27,16 @@ use graph::{ subgraph::SubgraphFeature, }, prelude::{ - lazy_static, o, q, r, serde_json, slog, BlockPtr, DeploymentHash, Entity, EntityOperation, - FutureExtension, GraphQlRunner as _, Logger, NodeId, Query, QueryError, - QueryExecutionError, QueryResult, QueryStoreManager, QueryVariables, SubgraphManifest, - SubgraphName, SubgraphStore, SubgraphVersionSwitchingMode, Subscription, SubscriptionError, + lazy_static, q, r, serde_json, BlockPtr, DeploymentHash, Entity, EntityOperation, + GraphQlRunner as _, NodeId, Query, QueryError, QueryExecutionError, QueryResult, + QueryVariables, SubgraphManifest, SubgraphName, SubgraphStore, + SubgraphVersionSwitchingMode, }, }; -use graph_graphql::{prelude::*, subscription::execute_subscription}; +use graph_graphql::prelude::*; use test_store::{ - deployment_state, execute_subgraph_query, execute_subgraph_query_with_deadline, - graphql_metrics, revert_block, run_test_sequentially, transact_errors, Store, LOAD_MANAGER, - LOGGER, METRICS_REGISTRY, STORE, SUBSCRIPTION_MANAGER, + deployment_state, execute_subgraph_query, execute_subgraph_query_with_deadline, revert_block, + run_test_sequentially, transact_errors, Store, LOAD_MANAGER, LOGGER, METRICS_REGISTRY, STORE, }; /// Ids for the various entities that we create in `insert_entities` and @@ -615,7 +613,6 @@ async fn execute_query_document_with_variables( let runner = Arc::new(GraphQlRunner::new( &LOGGER, STORE.clone(), - SUBSCRIPTION_MANAGER.clone(), LOAD_MANAGER.clone(), METRICS_REGISTRY.clone(), )); @@ -726,7 +723,6 @@ where let runner = Arc::new(GraphQlRunner::new( &LOGGER, STORE.clone(), - SUBSCRIPTION_MANAGER.clone(), LOAD_MANAGER.clone(), METRICS_REGISTRY.clone(), )); @@ -745,43 +741,6 @@ where }) } -/// Helper to run a subscription -async fn run_subscription( - store: &Arc, - query: &str, - max_complexity: Option, -) -> Result { - let deployment = setup_readonly(store.as_ref()).await; - let logger = Logger::root(slog::Discard, o!()); - let query_store = store - .query_store( - QueryTarget::Deployment(deployment.hash.clone(), Default::default()), - true, - ) - .await - .unwrap(); - - let query = Query::new(q::parse_query(query).unwrap().into_static(), None, false); - let options = SubscriptionExecutionOptions { - logger: logger.clone(), - store: query_store.clone(), - subscription_manager: SUBSCRIPTION_MANAGER.clone(), - timeout: None, - max_complexity, - max_depth: 100, - max_first: std::u32::MAX, - max_skip: std::u32::MAX, - graphql_metrics: graphql_metrics(), - load_manager: LOAD_MANAGER.clone(), - }; - let schema = STORE - .subgraph_store() - .api_schema(&deployment.hash, &Default::default()) - .unwrap(); - - execute_subscription(Subscription { query }, schema, options) -} - #[test] fn can_query_one_to_one_relationship() { const QUERY: &str = " @@ -1858,58 +1817,6 @@ fn query_complexity() { }) } -#[test] -fn query_complexity_subscriptions() { - run_test_sequentially(|store| async move { - const QUERY1: &str = "subscription { - musicians(orderBy: id) { - name - bands(first: 100, orderBy: id) { - name - members(first: 100, orderBy: id) { - name - } - } - } - }"; - let max_complexity = Some(1_010_100); - - // This query is exactly at the maximum complexity. - // FIXME: Not collecting the stream because that will hang the test. - let _ignore_stream = run_subscription(&store, QUERY1, max_complexity) - .await - .unwrap(); - - const QUERY2: &str = "subscription { - musicians(orderBy: id) { - name - t1: bands(first: 100, orderBy: id) { - name - members(first: 100, orderBy: id) { - name - } - } - t2: bands(first: 200, orderBy: id) { - name - members(first: 100, orderBy: id) { - name - } - } - } - }"; - - let result = run_subscription(&store, QUERY2, max_complexity).await; - - match result { - Err(SubscriptionError::GraphQLError(e)) => match &e[0] { - QueryExecutionError::TooComplex(3_030_100, _) => (), // Expected - e => panic!("did not catch complexity: {:?}", e), - }, - _ => panic!("did not catch complexity"), - } - }) -} - #[test] fn instant_timeout() { run_test_sequentially(|store| async move { @@ -2138,38 +2045,6 @@ fn cannot_filter_by_derved_relationship_fields() { }) } -#[test] -fn subscription_gets_result_even_without_events() { - run_test_sequentially(|store| async move { - const QUERY: &str = "subscription { - musicians(orderBy: id, first: 2) { - name - } - }"; - - // Execute the subscription and expect at least one result to be - // available in the result stream - let stream = run_subscription(&store, QUERY, None).await.unwrap(); - let results: Vec<_> = stream - .take(1) - .collect() - .timeout(Duration::from_secs(3)) - .await - .unwrap(); - - assert_eq!(results.len(), 1); - let result = Arc::try_unwrap(results.into_iter().next().unwrap()).unwrap(); - let data = extract_data!(result).unwrap(); - let exp = object! { - musicians: vec![ - object! { name: "John" }, - object! { name: "Lisa" } - ] - }; - assert_eq!(data, exp); - }) -} - #[test] fn can_use_nested_filter() { const QUERY: &str = " diff --git a/store/test-store/tests/postgres/store.rs b/store/test-store/tests/postgres/store.rs index 5f2f1e80e6c..28fd05da18f 100644 --- a/store/test-store/tests/postgres/store.rs +++ b/store/test-store/tests/postgres/store.rs @@ -1,17 +1,12 @@ use graph::blockchain::block_stream::FirehoseCursor; use graph::blockchain::BlockTime; use graph::data::graphql::ext::TypeDefinitionExt; -use graph::data::query::QueryTarget; use graph::data::subgraph::schema::DeploymentCreate; use graph::data_source::common::MappingABI; -use graph::futures01::{future, Stream}; -use graph::futures03::compat::Future01CompatExt; use graph::schema::{EntityType, InputSchema}; use graph_chain_ethereum::Mapping; use hex_literal::hex; use lazy_static::lazy_static; -use std::time::Duration; -use std::{collections::HashSet, sync::Mutex}; use std::{marker::PhantomData, str::FromStr}; use test_store::*; @@ -19,10 +14,7 @@ use graph::components::store::{DeploymentLocator, ReadStore, WritableStore}; use graph::data::subgraph::*; use graph::{ blockchain::DataSource, - components::store::{ - BlockStore as _, EntityFilter, EntityOrder, EntityQuery, StatusStore, - SubscriptionManager as _, - }, + components::store::{BlockStore as _, EntityFilter, EntityOrder, EntityQuery, StatusStore}, prelude::ethabi::Contract, }; use graph::{data::store::scalar, semver::Version}; @@ -912,78 +904,7 @@ fn find() { }); } -fn make_entity_change(entity_type: &EntityType) -> EntityChange { - EntityChange::Data { - subgraph_id: TEST_SUBGRAPH_ID.clone(), - entity_type: entity_type.to_string(), - } -} - -// Get as events until we've seen all the expected events or we time out waiting -async fn check_events( - stream: StoreEventStream, Error = ()> + Send>, - expected: Vec, -) { - fn as_set(events: Vec>) -> HashSet { - events.into_iter().fold(HashSet::new(), |mut set, event| { - set.extend(event.changes.iter().cloned()); - set - }) - } - - let expected = Mutex::new(as_set(expected.into_iter().map(Arc::new).collect())); - // Capture extra changes here; this is only needed for debugging, really. - // It's permissible that we get more changes than we expected because of - // how store events group changes together - let extra: Mutex> = Mutex::new(HashSet::new()); - // Get events from the store until we've either seen all the changes we - // expected or we timed out waiting for them - stream - .take_while(|event| { - let mut expected = expected.lock().unwrap(); - for change in &event.changes { - if !expected.remove(change) { - extra.lock().unwrap().insert(change.clone()); - } - } - future::ok(!expected.is_empty()) - }) - .collect() - .compat() - .timeout(Duration::from_secs(3)) - .await - .unwrap_or_else(|_| { - panic!( - "timed out waiting for events\n still waiting for {:?}\n got extra events {:?}", - expected.lock().unwrap().clone(), - extra.lock().unwrap().clone() - ) - }) - .expect("something went wrong getting events"); - // Check again that we really got everything - assert_eq!(HashSet::new(), expected.lock().unwrap().clone()); -} - -// Subscribe to store events -fn subscribe( - subgraph: &DeploymentHash, - entity_type: &EntityType, -) -> StoreEventStream, Error = ()> + Send> { - let subscription = - SUBSCRIPTION_MANAGER.subscribe(FromIterator::from_iter([SubscriptionFilter::Entities( - subgraph.clone(), - entity_type.to_owned(), - )])); - - StoreEventStream::new(subscription) -} - -async fn check_basic_revert( - store: Arc, - expected: StoreEvent, - deployment: &DeploymentLocator, - entity_type: &EntityType, -) { +async fn check_basic_revert(store: Arc, deployment: &DeploymentLocator) { let this_query = user_query() .filter(EntityFilter::Equal( "name".to_owned(), @@ -991,7 +912,6 @@ async fn check_basic_revert( )) .desc("name"); - let subscription = subscribe(&deployment.hash, entity_type); let state = deployment_state(store.as_ref(), &deployment.hash).await; assert_eq!(&deployment.hash, &state.id); @@ -1014,17 +934,13 @@ async fn check_basic_revert( let state = deployment_state(store.as_ref(), &deployment.hash).await; assert_eq!(&deployment.hash, &state.id); - - check_events(subscription, vec![expected]).await } #[test] fn revert_block_basic_user() { run_test(|store, _, deployment| async move { - let expected = StoreEvent::new(vec![make_entity_change(&*USER_TYPE)]); - let count = get_entity_count(store.clone(), &deployment.hash); - check_basic_revert(store.clone(), expected, &deployment, &*USER_TYPE).await; + check_basic_revert(store.clone(), &deployment).await; assert_eq!(count, get_entity_count(store.clone(), &deployment.hash)); }) } @@ -1052,8 +968,6 @@ fn revert_block_with_delete() { .await .unwrap(); - let subscription = subscribe(&deployment.hash, &*USER_TYPE); - // Revert deletion let count = get_entity_count(store.clone(), &deployment.hash); revert_block(&store, &deployment, &TEST_BLOCK_2_PTR).await; @@ -1073,12 +987,6 @@ fn revert_block_with_delete() { let test_value = Value::String("dinici@email.com".to_owned()); assert!(returned_name.is_some()); assert_eq!(&test_value, returned_name.unwrap()); - - // Check that the subscription notified us of the changes - let expected = StoreEvent::new(vec![make_entity_change(&*USER_TYPE)]); - - // The last event is the one for the reversion - check_events(subscription, vec![expected]).await }) } @@ -1106,8 +1014,6 @@ fn revert_block_with_partial_update() { .await .unwrap(); - let subscription = subscribe(&deployment.hash, &*USER_TYPE); - // Perform revert operation, reversing the partial update let count = get_entity_count(store.clone(), &deployment.hash); revert_block(&store, &deployment, &TEST_BLOCK_2_PTR).await; @@ -1118,11 +1024,6 @@ fn revert_block_with_partial_update() { // Verify that the entity has been returned to its original state assert_eq!(reverted_entity, original_entity); - - // Check that the subscription notified us of the changes - let expected = StoreEvent::new(vec![make_entity_change(&*USER_TYPE)]); - - check_events(subscription, vec![expected]).await }) } @@ -1229,8 +1130,6 @@ fn revert_block_with_dynamic_data_source_operations() { **loaded_dds[0].param.as_ref().unwrap() ); - let subscription = subscribe(&deployment.hash, &*USER_TYPE); - // Revert block that added the user and the dynamic data source revert_block(&store, &deployment, &TEST_BLOCK_2_PTR).await; @@ -1246,233 +1145,6 @@ fn revert_block_with_dynamic_data_source_operations() { .await .unwrap(); assert_eq!(0, loaded_dds.len()); - - // Verify that the right change events were emitted for the reversion - let expected_events = vec![StoreEvent { - tag: 3, - changes: HashSet::from_iter( - vec![EntityChange::Data { - subgraph_id: DeploymentHash::new("testsubgraph").unwrap(), - entity_type: USER_TYPE.to_string(), - }] - .into_iter(), - ), - }]; - check_events(subscription, expected_events).await - }) -} - -#[test] -fn entity_changes_are_fired_and_forwarded_to_subscriptions() { - run_test(|store, _, _| async move { - let subgraph_id = DeploymentHash::new("EntityChangeTestSubgraph").unwrap(); - let schema = InputSchema::parse_latest(USER_GQL, subgraph_id.clone()) - .expect("Failed to parse user schema"); - let manifest = SubgraphManifest:: { - id: subgraph_id.clone(), - spec_version: Version::new(1, 3, 0), - features: Default::default(), - description: None, - repository: None, - schema: schema.clone(), - data_sources: vec![], - graft: None, - templates: vec![], - chain: PhantomData, - indexer_hints: None, - }; - - let deployment = - DeploymentCreate::new(String::new(), &manifest, Some(TEST_BLOCK_0_PTR.clone())); - let name = SubgraphName::new("test/entity-changes-are-fired").unwrap(); - let node_id = NodeId::new("test").unwrap(); - let deployment = store - .subgraph_store() - .create_subgraph_deployment( - name, - &schema, - deployment, - node_id, - NETWORK_NAME.to_string(), - SubgraphVersionSwitchingMode::Instant, - ) - .unwrap(); - - let subscription = subscribe(&subgraph_id, &*USER_TYPE); - - // Add two entities to the store - let added_entities = vec![ - ( - "1".to_owned(), - entity! { schema => id: "1", name: "Johnny Boy", vid: 5i64 }, - ), - ( - "2".to_owned(), - entity! { schema => id: "2", name: "Tessa", vid: 6i64 }, - ), - ]; - transact_and_wait( - &store.subgraph_store(), - &deployment, - TEST_BLOCK_1_PTR.clone(), - added_entities - .iter() - .map(|(id, data)| { - let mut data = data.clone(); - data.set_vid_if_empty(); - EntityOperation::Set { - key: USER_TYPE.parse_key(id.as_str()).unwrap(), - data, - } - }) - .collect(), - ) - .await - .unwrap(); - - // Update an entity in the store - let updated_entity = entity! { schema => id: "1", name: "Johnny", vid: 7i64 }; - let update_op = EntityOperation::Set { - key: USER_TYPE.parse_key("1").unwrap(), - data: updated_entity.clone(), - }; - - // Delete an entity in the store - let delete_op = EntityOperation::Remove { - key: USER_TYPE.parse_key("2").unwrap(), - }; - - // Commit update & delete ops - transact_and_wait( - &store.subgraph_store(), - &deployment, - TEST_BLOCK_2_PTR.clone(), - vec![update_op, delete_op], - ) - .await - .unwrap(); - - // We're expecting two events to be written to the subscription stream - let expected = vec![ - StoreEvent::new(vec![ - EntityChange::Data { - subgraph_id: subgraph_id.clone(), - entity_type: USER_TYPE.to_string(), - }, - EntityChange::Data { - subgraph_id: subgraph_id.clone(), - entity_type: USER_TYPE.to_string(), - }, - ]), - StoreEvent::new(vec![ - EntityChange::Data { - subgraph_id: subgraph_id.clone(), - entity_type: USER_TYPE.to_string(), - }, - EntityChange::Data { - subgraph_id: subgraph_id.clone(), - entity_type: USER_TYPE.to_string(), - }, - ]), - ]; - - check_events(subscription, expected).await - }) -} - -#[test] -fn throttle_subscription_delivers() { - run_test(|store, _, deployment| async move { - let subscription = subscribe(&deployment.hash, &*USER_TYPE) - .throttle_while_syncing( - &LOGGER, - store - .clone() - .query_store( - QueryTarget::Deployment(deployment.hash.clone(), Default::default()), - true, - ) - .await - .unwrap(), - Duration::from_millis(500), - ) - .await; - - let user4 = create_test_entity( - "4", - &*USER_TYPE, - "Steve", - "nieve@email.com", - 72_i32, - 120.7, - false, - None, - 7, - ); - - transact_entity_operations( - &store.subgraph_store(), - &deployment, - TEST_BLOCK_3_PTR.clone(), - vec![user4], - ) - .await - .unwrap(); - - let expected = StoreEvent::new(vec![make_entity_change(&*USER_TYPE)]); - - check_events(subscription, vec![expected]).await - }) -} - -#[test] -fn throttle_subscription_throttles() { - run_test(|store, _, deployment| async move { - // Throttle for a very long time (30s) - let subscription = subscribe(&deployment.hash, &*USER_TYPE) - .throttle_while_syncing( - &LOGGER, - store - .clone() - .query_store( - QueryTarget::Deployment(deployment.hash.clone(), Default::default()), - true, - ) - .await - .unwrap(), - Duration::from_secs(30), - ) - .await; - - let user4 = create_test_entity( - "4", - &*USER_TYPE, - "Steve", - "nieve@email.com", - 72_i32, - 120.7, - false, - None, - 8, - ); - - transact_entity_operations( - &store.subgraph_store(), - &deployment, - TEST_BLOCK_3_PTR.clone(), - vec![user4], - ) - .await - .unwrap(); - - // Make sure we time out waiting for the subscription - let res = subscription - .take(1) - .collect() - .compat() - .timeout(Duration::from_millis(500)) - .await; - assert!(res.is_err()); }) } diff --git a/tests/src/fixture/mod.rs b/tests/src/fixture/mod.rs index 1fc43e495b8..217c7f705b6 100644 --- a/tests/src/fixture/mod.rs +++ b/tests/src/fixture/mod.rs @@ -80,7 +80,7 @@ pub fn test_ptr_reorged(n: BlockNumber, reorg_n: u32) -> BlockPtr { } } -type GraphQlRunner = graph_graphql::prelude::GraphQlRunner; +type GraphQlRunner = graph_graphql::prelude::GraphQlRunner; struct CommonChainConfig { logger_factory: LoggerFactory, @@ -521,12 +521,10 @@ pub async fn setup( ); // Graphql runner - let subscription_manager = Arc::new(PanicSubscriptionManager {}); let load_manager = LoadManager::new(&logger, Vec::new(), Vec::new(), mock_registry.clone()); let graphql_runner = Arc::new(GraphQlRunner::new( &logger, stores.network_store.clone(), - subscription_manager.clone(), Arc::new(load_manager), mock_registry.clone(), )); From 082995455345141c888829433d6574ed00224c22 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Tue, 18 Feb 2025 12:00:27 -0800 Subject: [PATCH 004/150] graph, node, store: Remove unused SubscriptionManager.subscribe_no_payload --- graph/src/components/store/traits.rs | 3 - node/src/manager/mod.rs | 6 +- store/postgres/src/store_events.rs | 82 +--------------------------- 3 files changed, 3 insertions(+), 88 deletions(-) diff --git a/graph/src/components/store/traits.rs b/graph/src/components/store/traits.rs index 2292a1f61f5..f4e15e077c5 100644 --- a/graph/src/components/store/traits.rs +++ b/graph/src/components/store/traits.rs @@ -26,9 +26,6 @@ pub trait SubscriptionManager: Send + Sync + 'static { /// /// Returns a stream of store events that match the input arguments. fn subscribe(&self, entities: BTreeSet) -> StoreEventStreamBox; - - /// If the payload is not required, use for a more efficient subscription mechanism backed by a watcher. - fn subscribe_no_payload(&self, entities: BTreeSet) -> UnitStream; } /// Subgraph forking is the process of lazily fetching entities diff --git a/node/src/manager/mod.rs b/node/src/manager/mod.rs index b2eccaf6e9a..90e72d245a2 100644 --- a/node/src/manager/mod.rs +++ b/node/src/manager/mod.rs @@ -1,7 +1,7 @@ use std::collections::BTreeSet; use graph::{ - components::store::{SubscriptionManager, UnitStream}, + components::store::SubscriptionManager, prelude::{anyhow, StoreEventStreamBox, SubscriptionFilter}, }; @@ -19,10 +19,6 @@ impl SubscriptionManager for PanicSubscriptionManager { fn subscribe(&self, _: BTreeSet) -> StoreEventStreamBox { panic!("we were never meant to call `subscribe`"); } - - fn subscribe_no_payload(&self, _: BTreeSet) -> UnitStream { - panic!("we were never meant to call `subscribe_no_payload`"); - } } pub type CmdResult = Result<(), anyhow::Error>; diff --git a/store/postgres/src/store_events.rs b/store/postgres/src/store_events.rs index 6370cd3aa92..e56e7e56f44 100644 --- a/store/postgres/src/store_events.rs +++ b/store/postgres/src/store_events.rs @@ -2,19 +2,17 @@ use graph::futures01::Stream; use graph::futures03::compat::Stream01CompatExt; use graph::futures03::stream::StreamExt; use graph::futures03::TryStreamExt; -use graph::parking_lot::Mutex; use graph::tokio_stream::wrappers::ReceiverStream; use std::collections::BTreeSet; use std::sync::{atomic::Ordering, Arc, RwLock}; use std::{collections::HashMap, sync::atomic::AtomicUsize}; use tokio::sync::mpsc::{channel, Sender}; -use tokio::sync::watch; use uuid::Uuid; use crate::notification_listener::{NotificationListener, SafeChannelName}; -use graph::components::store::{SubscriptionManager as SubscriptionManagerTrait, UnitStream}; +use graph::components::store::SubscriptionManager as SubscriptionManagerTrait; use graph::prelude::serde_json; -use graph::{prelude::*, tokio_stream}; +use graph::prelude::*; pub struct StoreEventListener { notification_listener: NotificationListener, @@ -89,44 +87,9 @@ impl StoreEventListener { } } -struct Watcher { - sender: Arc>, - receiver: watch::Receiver, -} - -impl Watcher { - fn new(init: T) -> Self { - let (sender, receiver) = watch::channel(init); - Watcher { - sender: Arc::new(sender), - receiver, - } - } - - fn send(&self, v: T) { - // Unwrap: `self` holds a receiver. - self.sender.send(v).unwrap() - } - - fn stream(&self) -> Box + Unpin + Send + Sync> { - Box::new(tokio_stream::wrappers::WatchStream::new( - self.receiver.clone(), - )) - } - - /// Outstanding receivers returned from `Self::stream`. - fn receiver_count(&self) -> usize { - // Do not count the internal receiver. - self.sender.receiver_count() - 1 - } -} - /// Manage subscriptions to the `StoreEvent` stream. Keep a list of /// currently active subscribers and forward new events to each of them pub struct SubscriptionManager { - // These are more efficient since only one entry is stored per filter. - subscriptions_no_payload: Arc, Watcher<()>>>>, - subscriptions: Arc>, Sender>)>>>, @@ -139,7 +102,6 @@ impl SubscriptionManager { let (listener, store_events) = StoreEventListener::new(logger, postgres_url, registry); let mut manager = SubscriptionManager { - subscriptions_no_payload: Arc::new(Mutex::new(HashMap::new())), subscriptions: Arc::new(RwLock::new(HashMap::new())), listener, }; @@ -161,7 +123,6 @@ impl SubscriptionManager { store_events: Box + Send>, ) { let subscriptions = self.subscriptions.cheap_clone(); - let subscriptions_no_payload = self.subscriptions_no_payload.cheap_clone(); let mut store_events = store_events.compat(); // This channel is constantly receiving things and there are locks involved, @@ -186,24 +147,12 @@ impl SubscriptionManager { } } } - - // Send to `subscriptions_no_payload`. - { - let watchers = subscriptions_no_payload.lock(); - - // Write change to all matching subscription streams - for (_, watcher) in watchers.iter().filter(|(filter, _)| event.matches(filter)) - { - watcher.send(()); - } - } } }); } fn periodically_clean_up_stale_subscriptions(&self) { let subscriptions = self.subscriptions.cheap_clone(); - let subscriptions_no_payload = self.subscriptions_no_payload.cheap_clone(); // Clean up stale subscriptions every 5s graph::spawn(async move { @@ -229,25 +178,6 @@ impl SubscriptionManager { subscriptions.remove(&id); } } - - // Cleanup `subscriptions_no_payload`. - { - let mut subscriptions = subscriptions_no_payload.lock(); - - // Obtain IDs of subscriptions whose receiving end has gone - let stale_ids = subscriptions - .iter_mut() - .filter_map(|(id, watcher)| match watcher.receiver_count() == 0 { - true => Some(id.clone()), - false => None, - }) - .collect::>(); - - // Remove all stale subscriptions - for id in stale_ids { - subscriptions.remove(&id); - } - } } }); } @@ -270,12 +200,4 @@ impl SubscriptionManagerTrait for SubscriptionManager { StoreEventStream::new(Box::new(ReceiverStream::new(receiver).map(Ok).compat())) .filter_by_entities(entities) } - - fn subscribe_no_payload(&self, entities: BTreeSet) -> UnitStream { - self.subscriptions_no_payload - .lock() - .entry(entities) - .or_insert_with(|| Watcher::new(())) - .stream() - } } From c5764124a909f370b5004388820de78cbab98337 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Tue, 18 Feb 2025 12:11:35 -0800 Subject: [PATCH 005/150] core, graph, store: Do not send store events for data changes --- core/src/subgraph/registrar.rs | 1 - graph/src/components/store/mod.rs | 58 -------------------------- graph/src/components/store/write.rs | 14 +------ graph/src/data/store/mod.rs | 8 ---- store/postgres/src/deployment_store.rs | 41 ++++++------------ store/postgres/src/relational.rs | 22 ++-------- store/postgres/src/subgraph_store.rs | 6 +-- store/postgres/src/writable.rs | 24 ++--------- 8 files changed, 22 insertions(+), 152 deletions(-) diff --git a/core/src/subgraph/registrar.rs b/core/src/subgraph/registrar.rs index 258f4bbcd15..8a49478aed9 100644 --- a/core/src/subgraph/registrar.rs +++ b/core/src/subgraph/registrar.rs @@ -165,7 +165,6 @@ where .iter() .filter(|change| filter.matches(change)) .map(|change| match change { - EntityChange::Data { .. } => unreachable!(), EntityChange::Assignment { deployment, operation, diff --git a/graph/src/components/store/mod.rs b/graph/src/components/store/mod.rs index 31b0e62cfae..d6cfa7c71a7 100644 --- a/graph/src/components/store/mod.rs +++ b/graph/src/components/store/mod.rs @@ -552,11 +552,6 @@ pub enum EntityChangeOperation { /// Entity change events emitted by [Store](trait.Store.html) implementations. #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, Hash)] pub enum EntityChange { - Data { - subgraph_id: DeploymentHash, - /// Entity type name of the changed entity. - entity_type: String, - }, Assignment { deployment: DeploymentLocator, operation: EntityChangeOperation, @@ -564,34 +559,12 @@ pub enum EntityChange { } impl EntityChange { - pub fn for_data(subgraph_id: DeploymentHash, key: EntityKey) -> Self { - Self::Data { - subgraph_id, - entity_type: key.entity_type.to_string(), - } - } - pub fn for_assignment(deployment: DeploymentLocator, operation: EntityChangeOperation) -> Self { Self::Assignment { deployment, operation, } } - - pub fn as_filter(&self, schema: &InputSchema) -> SubscriptionFilter { - use EntityChange::*; - match self { - Data { - subgraph_id, - entity_type, - .. - } => SubscriptionFilter::Entities( - subgraph_id.clone(), - schema.entity_type(entity_type).unwrap(), - ), - Assignment { .. } => SubscriptionFilter::Assignment, - } - } } #[derive(Clone, Debug, Serialize, Deserialize)] @@ -623,37 +596,6 @@ impl StoreEvent { StoreEvent { tag, changes } } - pub fn from_mods<'a, I: IntoIterator>( - subgraph_id: &DeploymentHash, - mods: I, - ) -> Self { - let changes: Vec<_> = mods - .into_iter() - .map(|op| { - use EntityModification::*; - match op { - Insert { key, .. } | Overwrite { key, .. } | Remove { key, .. } => { - EntityChange::for_data(subgraph_id.clone(), key.clone()) - } - } - }) - .collect(); - StoreEvent::new(changes) - } - - pub fn from_types(deployment: &DeploymentHash, entity_types: HashSet) -> Self { - let changes = - HashSet::from_iter( - entity_types - .into_iter() - .map(|entity_type| EntityChange::Data { - subgraph_id: deployment.clone(), - entity_type: entity_type.to_string(), - }), - ); - Self::from_set(changes) - } - /// Extend `ev1` with `ev2`. If `ev1` is `None`, just set it to `ev2` fn accumulate(logger: &Logger, ev1: &mut Option, ev2: StoreEvent) { if let Some(e) = ev1 { diff --git a/graph/src/components/store/write.rs b/graph/src/components/store/write.rs index 721e3d80bc1..aa56fdcc910 100644 --- a/graph/src/components/store/write.rs +++ b/graph/src/components/store/write.rs @@ -9,11 +9,10 @@ use crate::{ data::{store::Id, subgraph::schema::SubgraphError}, data_source::CausalityRegion, derive::CacheWeight, - prelude::DeploymentHash, util::cache_weight::CacheWeight, }; -use super::{BlockNumber, EntityKey, EntityType, StoreError, StoreEvent, StoredDynamicDataSource}; +use super::{BlockNumber, EntityKey, EntityType, StoreError, StoredDynamicDataSource}; /// A data structure similar to `EntityModification`, but tagged with a /// block. We might eventually replace `EntityModification` with this, but @@ -779,17 +778,6 @@ impl Batch { }) } - /// Generate a store event for all the changes that this batch makes - pub fn store_event(&self, deployment: &DeploymentHash) -> StoreEvent { - let entity_types = HashSet::from_iter( - self.mods - .groups - .iter() - .map(|group| group.entity_type.clone()), - ); - StoreEvent::from_types(deployment, entity_types) - } - pub fn groups<'a>(&'a self) -> impl Iterator { self.mods.groups.iter() } diff --git a/graph/src/data/store/mod.rs b/graph/src/data/store/mod.rs index 25c0d3e0813..cf8e9164fb0 100644 --- a/graph/src/data/store/mod.rs +++ b/graph/src/data/store/mod.rs @@ -49,14 +49,6 @@ pub enum SubscriptionFilter { impl SubscriptionFilter { pub fn matches(&self, change: &EntityChange) -> bool { match (self, change) { - ( - Self::Entities(eid, etype), - EntityChange::Data { - subgraph_id, - entity_type, - .. - }, - ) => subgraph_id == eid && entity_type == etype.typename(), (Self::Assignment, EntityChange::Assignment { .. }) => true, _ => false, } diff --git a/store/postgres/src/deployment_store.rs b/store/postgres/src/deployment_store.rs index 99e4409d0ab..332d2806cbe 100644 --- a/store/postgres/src/deployment_store.rs +++ b/store/postgres/src/deployment_store.rs @@ -42,7 +42,7 @@ use graph::data::subgraph::schema::{DeploymentCreate, SubgraphError}; use graph::prelude::{ anyhow, debug, info, o, warn, web3, AttributeNames, BlockNumber, BlockPtr, CheapClone, DeploymentHash, DeploymentState, Entity, EntityQuery, Error, Logger, QueryExecutionError, - StopwatchMetrics, StoreError, StoreEvent, UnfailOutcome, Value, ENV_VARS, + StopwatchMetrics, StoreError, UnfailOutcome, Value, ENV_VARS, }; use graph::schema::{ApiSchema, EntityKey, EntityType, InputSchema}; use web3::types::Address; @@ -1119,18 +1119,12 @@ impl DeploymentStore { last_rollup: Option, stopwatch: &StopwatchMetrics, manifest_idx_and_name: &[(u32, String)], - ) -> Result { + ) -> Result<(), StoreError> { let mut conn = { let _section = stopwatch.start_section("transact_blocks_get_conn"); self.get_conn()? }; - // Emit a store event for the changes we are about to make. We - // wait with sending it until we have done all our other work - // so that we do not hold a lock on the notification queue - // for longer than we have to - let event: StoreEvent = batch.store_event(&site.deployment); - let (layout, earliest_block) = deployment::with_lock(&mut conn, &site, |conn| { conn.transaction(|conn| -> Result<_, StoreError> { // Make the changes @@ -1205,7 +1199,7 @@ impl DeploymentStore { )?; } - Ok(event) + Ok(()) } fn spawn_prune( @@ -1294,9 +1288,9 @@ impl DeploymentStore { block_ptr_to: BlockPtr, firehose_cursor: &FirehoseCursor, truncate: bool, - ) -> Result { + ) -> Result<(), StoreError> { let logger = self.logger.cheap_clone(); - let event = deployment::with_lock(conn, &site, |conn| { + deployment::with_lock(conn, &site, |conn| { conn.transaction(|conn| -> Result<_, StoreError> { // The revert functions want the number of the first block that we need to get rid of let block = block_ptr_to.number + 1; @@ -1311,15 +1305,12 @@ impl DeploymentStore { // Revert the data let layout = self.layout(conn, site.clone())?; - let event = if truncate { - let event = layout.truncate_tables(conn)?; + if truncate { deployment::set_entity_count(conn, site.as_ref(), layout.count_query.as_str())?; - event } else { - let (event, count) = layout.revert_block(conn, block)?; + let count = layout.revert_block(conn, block)?; deployment::update_entity_count(conn, site.as_ref(), count)?; - event - }; + } // Revert the meta data changes that correspond to this subgraph. // Only certain meta data changes need to be reverted, most @@ -1328,18 +1319,16 @@ impl DeploymentStore { // changes that might need to be reverted Layout::revert_metadata(&logger, conn, &site, block)?; - Ok(event) + Ok(()) }) - })?; - - Ok(event) + }) } pub(crate) fn truncate( &self, site: Arc, block_ptr_to: BlockPtr, - ) -> Result { + ) -> Result<(), StoreError> { let mut conn = self.get_conn()?; let block_ptr_from = Self::block_ptr_with_conn(&mut conn, site.cheap_clone())?; @@ -1365,11 +1354,7 @@ impl DeploymentStore { ) } - pub(crate) fn rewind( - &self, - site: Arc, - block_ptr_to: BlockPtr, - ) -> Result { + pub(crate) fn rewind(&self, site: Arc, block_ptr_to: BlockPtr) -> Result<(), StoreError> { let mut conn = self.get_conn()?; let block_ptr_from = Self::block_ptr_with_conn(&mut conn, site.cheap_clone())?; @@ -1400,7 +1385,7 @@ impl DeploymentStore { site: Arc, block_ptr_to: BlockPtr, firehose_cursor: &FirehoseCursor, - ) -> Result { + ) -> Result<(), StoreError> { let mut conn = self.get_conn()?; // Unwrap: If we are reverting then the block ptr is not `None`. let deployment_head = Self::block_ptr_with_conn(&mut conn, site.cheap_clone())?.unwrap(); diff --git a/store/postgres/src/relational.rs b/store/postgres/src/relational.rs index de7e6895083..b12cf790697 100644 --- a/store/postgres/src/relational.rs +++ b/store/postgres/src/relational.rs @@ -72,7 +72,7 @@ use graph::components::store::{AttributeNames, DerivedEntityQuery}; use graph::data::store::{Id, IdList, IdType, BYTES_SCALAR}; use graph::data::subgraph::schema::POI_TABLE; use graph::prelude::{ - anyhow, info, BlockNumber, DeploymentHash, Entity, EntityChange, EntityOperation, Logger, + anyhow, info, BlockNumber, DeploymentHash, Entity, EntityOperation, Logger, QueryExecutionError, StoreError, StoreEvent, ValueType, BLOCK_NUMBER_MAX, }; @@ -1041,8 +1041,7 @@ impl Layout { &self, conn: &mut PgConnection, block: BlockNumber, - ) -> Result<(StoreEvent, i32), StoreError> { - let mut changes: Vec = Vec::new(); + ) -> Result { let mut count: i32 = 0; for table in self.tables.values() { @@ -1071,23 +1070,8 @@ impl Layout { let deleted = removed.difference(&unclamped).count() as i32; let inserted = unclamped.difference(&removed).count() as i32; count += inserted - deleted; - // EntityChange for versions we just deleted - let deleted = removed - .into_iter() - .filter(|id| !unclamped.contains(id)) - .map(|_| EntityChange::Data { - subgraph_id: self.site.deployment.clone(), - entity_type: table.object.to_string(), - }); - changes.extend(deleted); - // EntityChange for versions that we just updated or inserted - let set = unclamped.into_iter().map(|_| EntityChange::Data { - subgraph_id: self.site.deployment.clone(), - entity_type: table.object.to_string(), - }); - changes.extend(set); } - Ok((StoreEvent::new(changes), count)) + Ok(count) } /// Revert the metadata (dynamic data sources and related entities) for diff --git a/store/postgres/src/subgraph_store.rs b/store/postgres/src/subgraph_store.rs index f6544a79e0d..a881aae27d1 100644 --- a/store/postgres/src/subgraph_store.rs +++ b/store/postgres/src/subgraph_store.rs @@ -1046,14 +1046,12 @@ impl SubgraphStoreInner { pub fn rewind(&self, id: DeploymentHash, block_ptr_to: BlockPtr) -> Result<(), StoreError> { let (store, site) = self.store(&id)?; - let event = store.rewind(site, block_ptr_to)?; - self.send_store_event(&event) + store.rewind(site, block_ptr_to) } pub fn truncate(&self, id: DeploymentHash, block_ptr_to: BlockPtr) -> Result<(), StoreError> { let (store, site) = self.store(&id)?; - let event = store.truncate(site, block_ptr_to)?; - self.send_store_event(&event) + store.truncate(site, block_ptr_to) } pub(crate) async fn get_proof_of_indexing( diff --git a/store/postgres/src/writable.rs b/store/postgres/src/writable.rs index a9525ba9eb5..33b5b40e3b7 100644 --- a/store/postgres/src/writable.rs +++ b/store/postgres/src/writable.rs @@ -190,19 +190,6 @@ impl SyncStore { last_rollup, }) } - - /// Try to send a `StoreEvent`; if sending fails, log the error but - /// return `Ok(())` - fn try_send_store_event(&self, event: StoreEvent) -> Result<(), StoreError> { - if !ENV_VARS.store.disable_subscription_notifications { - let _ = self.store.send_store_event(&event).map_err( - |e| error!(self.logger, "Could not send store event"; "error" => e.to_string()), - ); - Ok(()) - } else { - Ok(()) - } - } } // Methods that mirror `WritableStoreTrait` @@ -245,7 +232,7 @@ impl SyncStore { firehose_cursor: &FirehoseCursor, ) -> Result<(), StoreError> { retry::forever(&self.logger, "revert_block_operations", || { - let event = self.writable.revert_block_operations( + self.writable.revert_block_operations( self.site.clone(), block_ptr_to.clone(), firehose_cursor, @@ -254,9 +241,7 @@ impl SyncStore { let block_time = self .writable .block_time(self.site.cheap_clone(), block_ptr_to.number)?; - self.last_rollup.set(block_time)?; - - self.try_send_store_event(event) + self.last_rollup.set(block_time) }) } @@ -315,7 +300,7 @@ impl SyncStore { stopwatch: &StopwatchMetrics, ) -> Result<(), StoreError> { retry::forever(&self.logger, "transact_block_operations", move || { - let event = self.writable.transact_block_operations( + self.writable.transact_block_operations( &self.logger, self.site.clone(), batch, @@ -326,9 +311,6 @@ impl SyncStore { // unwrap: batch.block_times is never empty let last_block_time = batch.block_times.last().unwrap().1; self.last_rollup.set(Some(last_block_time))?; - - let _section = stopwatch.start_section("send_store_event"); - self.try_send_store_event(event)?; Ok(()) }) } From 8bcabf3557579709207c4859be4154bcd587b39b Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Tue, 18 Feb 2025 12:16:05 -0800 Subject: [PATCH 006/150] graph: Remove graph::data::subscription --- graph/src/components/graphql.rs | 6 ---- graph/src/data/mod.rs | 3 -- graph/src/data/subscription/error.rs | 34 --------------------- graph/src/data/subscription/mod.rs | 7 ----- graph/src/data/subscription/result.rs | 10 ------ graph/src/data/subscription/subscription.rs | 11 ------- graph/src/lib.rs | 5 +-- 7 files changed, 1 insertion(+), 75 deletions(-) delete mode 100644 graph/src/data/subscription/error.rs delete mode 100644 graph/src/data/subscription/mod.rs delete mode 100644 graph/src/data/subscription/result.rs delete mode 100644 graph/src/data/subscription/subscription.rs diff --git a/graph/src/components/graphql.rs b/graph/src/components/graphql.rs index 9ff8fc62261..b5fc4273860 100644 --- a/graph/src/components/graphql.rs +++ b/graph/src/components/graphql.rs @@ -1,17 +1,11 @@ use crate::data::query::QueryResults; use crate::data::query::{Query, QueryTarget}; -use crate::data::subscription::{SubscriptionError, SubscriptionResult}; use crate::prelude::DeploymentHash; use async_trait::async_trait; -use futures01::Future; use std::sync::Arc; use std::time::Duration; -/// Future for subscription results. -pub type SubscriptionResultFuture = - Box + Send>; - pub enum GraphQlTarget { SubgraphName(String), Deployment(DeploymentHash), diff --git a/graph/src/data/mod.rs b/graph/src/data/mod.rs index 45f085c96fa..246d4cdba12 100644 --- a/graph/src/data/mod.rs +++ b/graph/src/data/mod.rs @@ -7,9 +7,6 @@ pub mod query; /// Data types for dealing with storing entities. pub mod store; -/// Data types for dealing with GraphQL subscriptions. -pub mod subscription; - /// Data types for dealing with GraphQL values. pub mod graphql; diff --git a/graph/src/data/subscription/error.rs b/graph/src/data/subscription/error.rs deleted file mode 100644 index 20cf3f3af73..00000000000 --- a/graph/src/data/subscription/error.rs +++ /dev/null @@ -1,34 +0,0 @@ -use serde::ser::*; - -use crate::prelude::QueryExecutionError; -use thiserror::Error; - -/// Error caused while processing a [Subscription](struct.Subscription.html) request. -#[derive(Debug, Error)] -pub enum SubscriptionError { - #[error("GraphQL error: {0:?}")] - GraphQLError(Vec), -} - -impl From for SubscriptionError { - fn from(e: QueryExecutionError) -> Self { - SubscriptionError::GraphQLError(vec![e]) - } -} - -impl From> for SubscriptionError { - fn from(e: Vec) -> Self { - SubscriptionError::GraphQLError(e) - } -} -impl Serialize for SubscriptionError { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - let mut map = serializer.serialize_map(Some(1))?; - let msg = format!("{}", self); - map.serialize_entry("message", msg.as_str())?; - map.end() - } -} diff --git a/graph/src/data/subscription/mod.rs b/graph/src/data/subscription/mod.rs deleted file mode 100644 index 093c0008728..00000000000 --- a/graph/src/data/subscription/mod.rs +++ /dev/null @@ -1,7 +0,0 @@ -mod error; -mod result; -mod subscription; - -pub use self::error::SubscriptionError; -pub use self::result::{QueryResultStream, SubscriptionResult}; -pub use self::subscription::Subscription; diff --git a/graph/src/data/subscription/result.rs b/graph/src/data/subscription/result.rs deleted file mode 100644 index 648ce79ac52..00000000000 --- a/graph/src/data/subscription/result.rs +++ /dev/null @@ -1,10 +0,0 @@ -use crate::prelude::QueryResult; -use std::pin::Pin; -use std::sync::Arc; - -/// A stream of query results for a subscription. -pub type QueryResultStream = - Pin> + Send>>; - -/// The result of running a subscription, if successful. -pub type SubscriptionResult = QueryResultStream; diff --git a/graph/src/data/subscription/subscription.rs b/graph/src/data/subscription/subscription.rs deleted file mode 100644 index 8ae6b872fba..00000000000 --- a/graph/src/data/subscription/subscription.rs +++ /dev/null @@ -1,11 +0,0 @@ -use crate::prelude::Query; - -/// A GraphQL subscription made by a client. -/// -/// At the moment, this only contains the GraphQL query submitted as the -/// subscription payload. -#[derive(Clone, Debug)] -pub struct Subscription { - /// The GraphQL subscription query. - pub query: Query, -} diff --git a/graph/src/lib.rs b/graph/src/lib.rs index 04872aab196..34d9c685f92 100644 --- a/graph/src/lib.rs +++ b/graph/src/lib.rs @@ -117,7 +117,7 @@ pub mod prelude { EthereumBlock, EthereumBlockWithCalls, EthereumCall, LightEthereumBlock, LightEthereumBlockExt, }; - pub use crate::components::graphql::{GraphQLMetrics, GraphQlRunner, SubscriptionResultFuture}; + pub use crate::components::graphql::{GraphQLMetrics, GraphQlRunner}; pub use crate::components::link_resolver::{ IpfsResolver, JsonStreamValue, JsonValueStream, LinkResolver, }; @@ -163,9 +163,6 @@ pub mod prelude { SubgraphManifestValidationError, SubgraphName, SubgraphRegistrarError, UnvalidatedSubgraphManifest, }; - pub use crate::data::subscription::{ - QueryResultStream, Subscription, SubscriptionError, SubscriptionResult, - }; pub use crate::data_source::DataSourceTemplateInfo; pub use crate::ext::futures::{ CancelGuard, CancelHandle, CancelToken, CancelableError, FutureExtension, From 9f61d80e814c88de409bafbb315e925a991bde98 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Tue, 18 Feb 2025 12:33:04 -0800 Subject: [PATCH 007/150] graph, node: Remove SubscriptionFilter::Entities This will never match since we do not send such events any more. Also remove the `graphman listen entities` command --- graph/src/data/store/mod.rs | 10 +++---- node/src/bin/manager.rs | 22 +-------------- node/src/manager/commands/listen.rs | 44 ----------------------------- 3 files changed, 5 insertions(+), 71 deletions(-) diff --git a/graph/src/data/store/mod.rs b/graph/src/data/store/mod.rs index cf8e9164fb0..7405390d8e9 100644 --- a/graph/src/data/store/mod.rs +++ b/graph/src/data/store/mod.rs @@ -1,13 +1,13 @@ +use crate::prelude::EntityChange; use crate::{ components::store::DeploymentLocator, derive::CacheWeight, prelude::{lazy_static, q, r, s, CacheWeight, QueryExecutionError}, runtime::gas::{Gas, GasSizeOf}, - schema::{input::VID_FIELD, EntityKey, EntityType}, + schema::{input::VID_FIELD, EntityKey}, util::intern::{self, AtomPool}, util::intern::{Error as InternError, NullValue, Object}, }; -use crate::{data::subgraph::DeploymentHash, prelude::EntityChange}; use anyhow::{anyhow, Error}; use itertools::Itertools; use serde::de; @@ -39,9 +39,6 @@ pub mod sql; /// Filter subscriptions #[derive(Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] pub enum SubscriptionFilter { - /// Receive updates about all entities from the given deployment of the - /// given type - Entities(DeploymentHash, EntityType), /// Subscripe to changes in deployment assignments Assignment, } @@ -50,7 +47,6 @@ impl SubscriptionFilter { pub fn matches(&self, change: &EntityChange) -> bool { match (self, change) { (Self::Assignment, EntityChange::Assignment { .. }) => true, - _ => false, } } } @@ -1157,6 +1153,8 @@ fn value_bigint() { #[test] fn entity_validation() { + use crate::data::subgraph::DeploymentHash; + use crate::schema::EntityType; use crate::schema::InputSchema; const DOCUMENT: &str = " diff --git a/node/src/bin/manager.rs b/node/src/bin/manager.rs index 8024bacb183..20cf93d94df 100644 --- a/node/src/bin/manager.rs +++ b/node/src/bin/manager.rs @@ -467,14 +467,8 @@ pub enum ConfigCommand { pub enum ListenCommand { /// Listen only to assignment events Assignments, - /// Listen to events for entities in a specific deployment - Entities { - /// The deployment (see `help info`). - deployment: DeploymentSearch, - /// The entity types for which to print change notifications - entity_types: Vec, - }, } + #[derive(Clone, Debug, Subcommand)] pub enum CopyCommand { /// Create a copy of an existing subgraph @@ -930,13 +924,6 @@ impl Context { )) } - fn primary_and_subscription_manager(self) -> (ConnectionPool, Arc) { - let mgr = self.subscription_manager(); - let primary_pool = self.primary_pool(); - - (primary_pool, mgr) - } - fn store(&self) -> Arc { let (store, _) = self.store_and_pools(); store @@ -1310,13 +1297,6 @@ async fn main() -> anyhow::Result<()> { use ListenCommand::*; match cmd { Assignments => commands::listen::assignments(ctx.subscription_manager()).await, - Entities { - deployment, - entity_types, - } => { - let (primary, mgr) = ctx.primary_and_subscription_manager(); - commands::listen::entities(primary, mgr, &deployment, entity_types).await - } } } Copy(cmd) => { diff --git a/node/src/manager/commands/listen.rs b/node/src/manager/commands/listen.rs index feee8350797..1d793557b18 100644 --- a/node/src/manager/commands/listen.rs +++ b/node/src/manager/commands/listen.rs @@ -2,16 +2,12 @@ use std::iter::FromIterator; use std::sync::Arc; use std::{collections::BTreeSet, io::Write}; -use crate::manager::deployment::DeploymentSearch; use graph::futures01::Stream as _; use graph::futures03::compat::Future01CompatExt; -use graph::prelude::DeploymentHash; -use graph::schema::{EntityType, InputSchema}; use graph::{ components::store::SubscriptionManager as _, prelude::{serde_json, Error, SubscriptionFilter}, }; -use graph_store_postgres::connection_pool::ConnectionPool; use graph_store_postgres::SubscriptionManager; async fn listen( @@ -52,43 +48,3 @@ pub async fn assignments(mgr: Arc) -> Result<(), Error> { Ok(()) } - -pub async fn entities( - primary_pool: ConnectionPool, - mgr: Arc, - search: &DeploymentSearch, - entity_types: Vec, -) -> Result<(), Error> { - // We convert the entity type names into entity types in this very - // awkward way to avoid needing to have a SubgraphStore from which we - // load the input schema - fn as_entity_types( - entity_types: Vec, - id: &DeploymentHash, - ) -> Result, Error> { - use std::fmt::Write; - - let schema = entity_types - .iter() - .fold(String::new(), |mut buf, entity_type| { - writeln!(buf, "type {entity_type} @entity {{ id: ID! }}").unwrap(); - buf - }); - let schema = InputSchema::parse_latest(&schema, id.clone()).unwrap(); - entity_types - .iter() - .map(|et| schema.entity_type(et)) - .collect::>() - } - - let locator = search.locate_unique(&primary_pool)?; - let filter = as_entity_types(entity_types, &locator.hash)? - .into_iter() - .map(|et| SubscriptionFilter::Entities(locator.hash.clone(), et)) - .collect(); - - println!("waiting for store events from {}", locator); - listen(mgr, filter).await?; - - Ok(()) -} From d0ac9238c9b1e9ff83f86304d0b10ec4e212ae46 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Tue, 18 Feb 2025 12:40:37 -0800 Subject: [PATCH 008/150] graph: Remove unused StoreEventStream.throttle_while_syncing --- graph/src/components/store/mod.rs | 103 +----------------------------- 1 file changed, 2 insertions(+), 101 deletions(-) diff --git a/graph/src/components/store/mod.rs b/graph/src/components/store/mod.rs index d6cfa7c71a7..a9da59566a8 100644 --- a/graph/src/components/store/mod.rs +++ b/graph/src/components/store/mod.rs @@ -4,8 +4,7 @@ mod traits; pub mod write; pub use entity_cache::{EntityCache, EntityLfuCache, GetScope, ModificationsAndCache}; -use futures03::future::{FutureExt, TryFutureExt}; -use slog::{trace, Logger}; +use slog::Logger; pub use super::subgraph::Entity; pub use err::StoreError; @@ -14,8 +13,7 @@ use strum_macros::Display; pub use traits::*; pub use write::Batch; -use futures01::stream::poll_fn; -use futures01::{Async, Poll, Stream}; +use futures01::{Async, Stream}; use serde::{Deserialize, Serialize}; use std::collections::btree_map::Entry; use std::collections::{BTreeMap, BTreeSet, HashSet}; @@ -596,17 +594,6 @@ impl StoreEvent { StoreEvent { tag, changes } } - /// Extend `ev1` with `ev2`. If `ev1` is `None`, just set it to `ev2` - fn accumulate(logger: &Logger, ev1: &mut Option, ev2: StoreEvent) { - if let Some(e) = ev1 { - trace!(logger, "Adding changes to event"; - "from" => ev2.tag, "to" => e.tag); - e.changes.extend(ev2.changes); - } else { - *ev1 = Some(ev2); - } - } - pub fn extend(mut self, other: StoreEvent) -> Self { self.changes.extend(other.changes); self @@ -678,92 +665,6 @@ where StoreEventStream::new(Box::new(source)) } - - /// Reduce the frequency with which events are generated while a - /// subgraph deployment is syncing. While the given `deployment` is not - /// synced yet, events from `source` are reported at most every - /// `interval`. At the same time, no event is held for longer than - /// `interval`. The `StoreEvents` that arrive during an interval appear - /// on the returned stream as a single `StoreEvent`; the events are - /// combined by using the maximum of all sources and the concatenation - /// of the changes of the `StoreEvents` received during the interval. - // - // Currently unused, needs to be made compatible with `subscribe_no_payload`. - pub async fn throttle_while_syncing( - self, - logger: &Logger, - store: Arc, - interval: Duration, - ) -> StoreEventStreamBox { - // Check whether a deployment is marked as synced in the store. Note that in the moment a - // subgraph becomes synced any existing subscriptions will continue to be throttled since - // this is not re-checked. - let synced = store.is_deployment_synced().await.unwrap_or(false); - - let mut pending_event: Option = None; - let mut source = self.source.fuse(); - let mut had_err = false; - let mut delay = tokio::time::sleep(interval).unit_error().boxed().compat(); - let logger = logger.clone(); - - let source = Box::new(poll_fn(move || -> Poll>, ()> { - if had_err { - // We had an error the last time through, but returned the pending - // event first. Indicate the error now - had_err = false; - return Err(()); - } - - if synced { - return source.poll(); - } - - // Check if interval has passed since the last time we sent something. - // If it has, start a new delay timer - let should_send = match futures01::future::Future::poll(&mut delay) { - Ok(Async::NotReady) => false, - // Timer errors are harmless. Treat them as if the timer had - // become ready. - Ok(Async::Ready(())) | Err(_) => { - delay = tokio::time::sleep(interval).unit_error().boxed().compat(); - true - } - }; - - // Get as many events as we can off of the source stream - loop { - match source.poll() { - Ok(Async::NotReady) => { - if should_send && pending_event.is_some() { - let event = pending_event.take().map(Arc::new); - return Ok(Async::Ready(event)); - } else { - return Ok(Async::NotReady); - } - } - Ok(Async::Ready(None)) => { - let event = pending_event.take().map(Arc::new); - return Ok(Async::Ready(event)); - } - Ok(Async::Ready(Some(event))) => { - StoreEvent::accumulate(&logger, &mut pending_event, (*event).clone()); - } - Err(()) => { - // Before we report the error, deliver what we have accumulated so far. - // We will report the error the next time poll() is called - if pending_event.is_some() { - had_err = true; - let event = pending_event.take().map(Arc::new); - return Ok(Async::Ready(event)); - } else { - return Err(()); - } - } - }; - } - })); - StoreEventStream::new(source) - } } /// An entity operation that can be transacted into the store. From c1f12148e28ccdcdda35337de72f40babad59f51 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Tue, 18 Feb 2025 12:48:56 -0800 Subject: [PATCH 009/150] all: Remove SubscriptionFilter All events are now for assignments, so there's no need to filter anything --- core/src/subgraph/registrar.rs | 6 +----- graph/src/components/store/mod.rs | 17 +---------------- graph/src/components/store/traits.rs | 2 +- graph/src/data/store/mod.rs | 16 ---------------- graph/src/lib.rs | 4 +--- node/src/manager/commands/listen.rs | 18 +++++------------- node/src/manager/mod.rs | 6 ++---- store/postgres/src/store_events.rs | 21 ++++++--------------- 8 files changed, 17 insertions(+), 73 deletions(-) diff --git a/core/src/subgraph/registrar.rs b/core/src/subgraph/registrar.rs index 8a49478aed9..67327d566a2 100644 --- a/core/src/subgraph/registrar.rs +++ b/core/src/subgraph/registrar.rs @@ -154,16 +154,12 @@ where let logger = self.logger.clone(); self.subscription_manager - .subscribe(FromIterator::from_iter([SubscriptionFilter::Assignment])) + .subscribe() .map_err(|()| anyhow!("Entity change stream failed")) .map(|event| { - // We're only interested in the SubgraphDeploymentAssignment change; we - // know that there is at least one, as that is what we subscribed to - let filter = SubscriptionFilter::Assignment; let assignments = event .changes .iter() - .filter(|change| filter.matches(change)) .map(|change| match change { EntityChange::Assignment { deployment, diff --git a/graph/src/components/store/mod.rs b/graph/src/components/store/mod.rs index a9da59566a8..5d168b64a86 100644 --- a/graph/src/components/store/mod.rs +++ b/graph/src/components/store/mod.rs @@ -33,7 +33,7 @@ use crate::data::value::Word; use crate::data_source::CausalityRegion; use crate::derive::CheapClone; use crate::env::ENV_VARS; -use crate::prelude::{s, Attribute, DeploymentHash, SubscriptionFilter, ValueType}; +use crate::prelude::{s, Attribute, DeploymentHash, ValueType}; use crate::schema::{ast as sast, EntityKey, EntityType, InputSchema}; use crate::util::stats::MovingStats; @@ -598,12 +598,6 @@ impl StoreEvent { self.changes.extend(other.changes); self } - - pub fn matches(&self, filters: &BTreeSet) -> bool { - self.changes - .iter() - .any(|change| filters.iter().any(|filter| filter.matches(change))) - } } impl fmt::Display for StoreEvent { @@ -656,15 +650,6 @@ where pub fn new(source: S) -> Self { StoreEventStream { source } } - - /// Filter a `StoreEventStream` by subgraph and entity. Only events that have - /// at least one change to one of the given (subgraph, entity) combinations - /// will be delivered by the filtered stream. - pub fn filter_by_entities(self, filters: BTreeSet) -> StoreEventStreamBox { - let source = self.source.filter(move |event| event.matches(&filters)); - - StoreEventStream::new(Box::new(source)) - } } /// An entity operation that can be transacted into the store. diff --git a/graph/src/components/store/traits.rs b/graph/src/components/store/traits.rs index f4e15e077c5..9357663cddf 100644 --- a/graph/src/components/store/traits.rs +++ b/graph/src/components/store/traits.rs @@ -25,7 +25,7 @@ pub trait SubscriptionManager: Send + Sync + 'static { /// Subscribe to changes for specific subgraphs and entities. /// /// Returns a stream of store events that match the input arguments. - fn subscribe(&self, entities: BTreeSet) -> StoreEventStreamBox; + fn subscribe(&self) -> StoreEventStreamBox; } /// Subgraph forking is the process of lazily fetching entities diff --git a/graph/src/data/store/mod.rs b/graph/src/data/store/mod.rs index 7405390d8e9..c8786e9b473 100644 --- a/graph/src/data/store/mod.rs +++ b/graph/src/data/store/mod.rs @@ -1,4 +1,3 @@ -use crate::prelude::EntityChange; use crate::{ components::store::DeploymentLocator, derive::CacheWeight, @@ -36,21 +35,6 @@ pub mod ethereum; /// Conversion of values to/from SQL pub mod sql; -/// Filter subscriptions -#[derive(Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub enum SubscriptionFilter { - /// Subscripe to changes in deployment assignments - Assignment, -} - -impl SubscriptionFilter { - pub fn matches(&self, change: &EntityChange) -> bool { - match (self, change) { - (Self::Assignment, EntityChange::Assignment { .. }) => true, - } - } -} - #[derive(Clone, Debug, PartialEq, Eq, Hash)] pub struct NodeId(String); diff --git a/graph/src/lib.rs b/graph/src/lib.rs index 34d9c685f92..e93de34df9d 100644 --- a/graph/src/lib.rs +++ b/graph/src/lib.rs @@ -153,9 +153,7 @@ pub mod prelude { Query, QueryError, QueryExecutionError, QueryResult, QueryTarget, QueryVariables, }; pub use crate::data::store::scalar::{BigDecimal, BigInt, BigIntSign}; - pub use crate::data::store::{ - AssignmentEvent, Attribute, Entity, NodeId, SubscriptionFilter, Value, ValueType, - }; + pub use crate::data::store::{AssignmentEvent, Attribute, Entity, NodeId, Value, ValueType}; pub use crate::data::subgraph::schema::SubgraphDeploymentEntity; pub use crate::data::subgraph::{ CreateSubgraphResult, DataSourceContext, DeploymentHash, DeploymentState, Link, diff --git a/node/src/manager/commands/listen.rs b/node/src/manager/commands/listen.rs index 1d793557b18..69c3ff93cbf 100644 --- a/node/src/manager/commands/listen.rs +++ b/node/src/manager/commands/listen.rs @@ -1,20 +1,16 @@ -use std::iter::FromIterator; +use std::io::Write; use std::sync::Arc; -use std::{collections::BTreeSet, io::Write}; use graph::futures01::Stream as _; use graph::futures03::compat::Future01CompatExt; use graph::{ components::store::SubscriptionManager as _, - prelude::{serde_json, Error, SubscriptionFilter}, + prelude::{serde_json, Error}, }; use graph_store_postgres::SubscriptionManager; -async fn listen( - mgr: Arc, - filter: BTreeSet, -) -> Result<(), Error> { - let events = mgr.subscribe(filter); +async fn listen(mgr: Arc) -> Result<(), Error> { + let events = mgr.subscribe(); println!("press ctrl-c to stop"); let res = events .inspect(move |event| { @@ -40,11 +36,7 @@ async fn listen( pub async fn assignments(mgr: Arc) -> Result<(), Error> { println!("waiting for assignment events"); - listen( - mgr, - FromIterator::from_iter([SubscriptionFilter::Assignment]), - ) - .await?; + listen(mgr).await?; Ok(()) } diff --git a/node/src/manager/mod.rs b/node/src/manager/mod.rs index 90e72d245a2..6a332653ca8 100644 --- a/node/src/manager/mod.rs +++ b/node/src/manager/mod.rs @@ -1,8 +1,6 @@ -use std::collections::BTreeSet; - use graph::{ components::store::SubscriptionManager, - prelude::{anyhow, StoreEventStreamBox, SubscriptionFilter}, + prelude::{anyhow, StoreEventStreamBox}, }; pub mod catalog; @@ -16,7 +14,7 @@ pub mod prompt; pub struct PanicSubscriptionManager; impl SubscriptionManager for PanicSubscriptionManager { - fn subscribe(&self, _: BTreeSet) -> StoreEventStreamBox { + fn subscribe(&self) -> StoreEventStreamBox { panic!("we were never meant to call `subscribe`"); } } diff --git a/store/postgres/src/store_events.rs b/store/postgres/src/store_events.rs index e56e7e56f44..503b5cae08a 100644 --- a/store/postgres/src/store_events.rs +++ b/store/postgres/src/store_events.rs @@ -3,7 +3,6 @@ use graph::futures03::compat::Stream01CompatExt; use graph::futures03::stream::StreamExt; use graph::futures03::TryStreamExt; use graph::tokio_stream::wrappers::ReceiverStream; -use std::collections::BTreeSet; use std::sync::{atomic::Ordering, Arc, RwLock}; use std::{collections::HashMap, sync::atomic::AtomicUsize}; use tokio::sync::mpsc::{channel, Sender}; @@ -90,8 +89,7 @@ impl StoreEventListener { /// Manage subscriptions to the `StoreEvent` stream. Keep a list of /// currently active subscribers and forward new events to each of them pub struct SubscriptionManager { - subscriptions: - Arc>, Sender>)>>>, + subscriptions: Arc>>>>, /// Keep the notification listener alive listener: StoreEventListener, @@ -137,13 +135,10 @@ impl SubscriptionManager { // Write change to all matching subscription streams; remove subscriptions // whose receiving end has been dropped - for (id, (_, sender)) in senders - .iter() - .filter(|(_, (filter, _))| event.matches(filter)) - { + for (id, sender) in senders { if sender.send(event.cheap_clone()).await.is_err() { // Receiver was dropped - subscriptions.write().unwrap().remove(id); + subscriptions.write().unwrap().remove(&id); } } } @@ -167,7 +162,7 @@ impl SubscriptionManager { // Obtain IDs of subscriptions whose receiving end has gone let stale_ids = subscriptions .iter_mut() - .filter_map(|(id, (_, sender))| match sender.is_closed() { + .filter_map(|(id, sender)| match sender.is_closed() { true => Some(id.clone()), false => None, }) @@ -184,20 +179,16 @@ impl SubscriptionManager { } impl SubscriptionManagerTrait for SubscriptionManager { - fn subscribe(&self, entities: BTreeSet) -> StoreEventStreamBox { + fn subscribe(&self) -> StoreEventStreamBox { let id = Uuid::new_v4().to_string(); // Prepare the new subscription by creating a channel and a subscription object let (sender, receiver) = channel(100); // Add the new subscription - self.subscriptions - .write() - .unwrap() - .insert(id, (Arc::new(entities.clone()), sender)); + self.subscriptions.write().unwrap().insert(id, sender); // Return the subscription ID and entity change stream StoreEventStream::new(Box::new(ReceiverStream::new(receiver).map(Ok).compat())) - .filter_by_entities(entities) } } From 9f075353d419d2def2c52c637066a70d8323303f Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Tue, 18 Feb 2025 12:53:54 -0800 Subject: [PATCH 010/150] graph, graphql: Remove Resolver.resolve_field_stream and UnitStream --- graph/src/components/store/mod.rs | 2 -- graphql/src/execution/resolver.rs | 14 +------------- 2 files changed, 1 insertion(+), 15 deletions(-) diff --git a/graph/src/components/store/mod.rs b/graph/src/components/store/mod.rs index 5d168b64a86..08098093fb0 100644 --- a/graph/src/components/store/mod.rs +++ b/graph/src/components/store/mod.rs @@ -628,8 +628,6 @@ pub struct StoreEventStream { pub type StoreEventStreamBox = StoreEventStream, Error = ()> + Send>>; -pub type UnitStream = Box + Unpin + Send + Sync>; - impl Stream for StoreEventStream where S: Stream, Error = ()> + Send, diff --git a/graphql/src/execution/resolver.rs b/graphql/src/execution/resolver.rs index 1b139c65828..ca59e401dfc 100644 --- a/graphql/src/execution/resolver.rs +++ b/graphql/src/execution/resolver.rs @@ -1,6 +1,6 @@ use std::time::Duration; -use graph::components::store::{QueryPermit, UnitStream}; +use graph::components::store::QueryPermit; use graph::data::query::{CacheStatus, Trace}; use graph::prelude::{async_trait, s, Error, QueryExecutionError}; use graph::schema::ApiSchema; @@ -111,18 +111,6 @@ pub trait Resolver: Sized + Send + Sync + 'static { } } - // Resolves a change stream for a given field. - fn resolve_field_stream( - &self, - _schema: &ApiSchema, - _object_type: &s::ObjectType, - _field: &a::Field, - ) -> Result { - Err(QueryExecutionError::NotSupported(String::from( - "Resolving field streams is not supported by this resolver", - ))) - } - fn post_process(&self, _result: &mut QueryResult) -> Result<(), Error> { Ok(()) } From b9aa9e43222b9a7faf0392ca2af05290a9dff05d Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Tue, 18 Feb 2025 15:59:15 -0800 Subject: [PATCH 011/150] all: Rename EntityChange to AssignmentChange and clean up its API --- .../src/commands/deployment/reassign.rs | 4 +- core/src/subgraph/registrar.rs | 17 ++----- graph/src/components/store/mod.rs | 44 +++++++++++------- graph/src/lib.rs | 4 +- store/postgres/src/primary.rs | 46 +++++++++---------- store/test-store/tests/postgres/subgraph.rs | 21 +++------ 6 files changed, 63 insertions(+), 73 deletions(-) diff --git a/core/graphman/src/commands/deployment/reassign.rs b/core/graphman/src/commands/deployment/reassign.rs index 2e5916a7aae..5d3d633e082 100644 --- a/core/graphman/src/commands/deployment/reassign.rs +++ b/core/graphman/src/commands/deployment/reassign.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use anyhow::anyhow; use graph::components::store::DeploymentLocator; use graph::components::store::StoreEvent; -use graph::prelude::EntityChange; +use graph::prelude::AssignmentChange; use graph::prelude::NodeId; use graph_store_postgres::command_support::catalog; use graph_store_postgres::command_support::catalog::Site; @@ -74,7 +74,7 @@ pub fn reassign_deployment( let primary_conn = primary_pool.get().map_err(GraphmanError::from)?; let mut catalog_conn = catalog::Connection::new(primary_conn); - let changes: Vec = match catalog_conn + let changes: Vec = match catalog_conn .assigned_node(&deployment.site) .map_err(GraphmanError::from)? { diff --git a/core/src/subgraph/registrar.rs b/core/src/subgraph/registrar.rs index 67327d566a2..41f7697c529 100644 --- a/core/src/subgraph/registrar.rs +++ b/core/src/subgraph/registrar.rs @@ -157,17 +157,8 @@ where .subscribe() .map_err(|()| anyhow!("Entity change stream failed")) .map(|event| { - let assignments = event - .changes - .iter() - .map(|change| match change { - EntityChange::Assignment { - deployment, - operation, - } => (deployment.clone(), operation.clone()), - }) - .collect::>(); - stream::iter_ok(assignments) + let changes: Vec<_> = event.changes.iter().cloned().map(AssignmentChange::into_parts).collect(); + stream::iter_ok(changes) }) .flatten() .and_then( @@ -178,7 +169,7 @@ where ); match operation { - EntityChangeOperation::Set => { + AssignmentOperation::Set => { store .assignment_status(&deployment) .map_err(|e| { @@ -215,7 +206,7 @@ where } }) } - EntityChangeOperation::Removed => { + AssignmentOperation::Removed => { // Send remove event without checking node ID. // If node ID does not match, then this is a no-op when handled in // assignment provider. diff --git a/graph/src/components/store/mod.rs b/graph/src/components/store/mod.rs index 08098093fb0..b64f8b35964 100644 --- a/graph/src/components/store/mod.rs +++ b/graph/src/components/store/mod.rs @@ -537,32 +537,42 @@ impl EntityQuery { } } -/// Operation types that lead to entity changes. -#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, Hash)] +/// Operation types that lead to changes in assignments +#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq, Hash)] #[serde(rename_all = "lowercase")] -pub enum EntityChangeOperation { - /// An entity was added or updated +pub enum AssignmentOperation { + /// An assignment was added or updated Set, - /// An existing entity was removed. + /// An assignment was removed. Removed, } -/// Entity change events emitted by [Store](trait.Store.html) implementations. +/// Assignment change events emitted by [Store](trait.Store.html) implementations. #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq, Hash)] -pub enum EntityChange { - Assignment { - deployment: DeploymentLocator, - operation: EntityChangeOperation, - }, +pub struct AssignmentChange { + deployment: DeploymentLocator, + operation: AssignmentOperation, } -impl EntityChange { - pub fn for_assignment(deployment: DeploymentLocator, operation: EntityChangeOperation) -> Self { - Self::Assignment { +impl AssignmentChange { + fn new(deployment: DeploymentLocator, operation: AssignmentOperation) -> Self { + Self { deployment, operation, } } + + pub fn set(deployment: DeploymentLocator) -> Self { + Self::new(deployment, AssignmentOperation::Set) + } + + pub fn removed(deployment: DeploymentLocator) -> Self { + Self::new(deployment, AssignmentOperation::Removed) + } + + pub fn into_parts(self) -> (DeploymentLocator, AssignmentOperation) { + (self.deployment, self.operation) + } } #[derive(Clone, Debug, Serialize, Deserialize)] @@ -578,16 +588,16 @@ pub struct StoreEvent { // The tag is only there to make it easier to track StoreEvents in the // logs as they flow through the system pub tag: usize, - pub changes: HashSet, + pub changes: HashSet, } impl StoreEvent { - pub fn new(changes: Vec) -> StoreEvent { + pub fn new(changes: Vec) -> StoreEvent { let changes = changes.into_iter().collect(); StoreEvent::from_set(changes) } - fn from_set(changes: HashSet) -> StoreEvent { + fn from_set(changes: HashSet) -> StoreEvent { static NEXT_TAG: AtomicUsize = AtomicUsize::new(0); let tag = NEXT_TAG.fetch_add(1, Ordering::Relaxed); diff --git a/graph/src/lib.rs b/graph/src/lib.rs index e93de34df9d..19139cd2804 100644 --- a/graph/src/lib.rs +++ b/graph/src/lib.rs @@ -127,8 +127,8 @@ pub mod prelude { }; pub use crate::components::server::subscription::SubscriptionServer; pub use crate::components::store::{ - write::EntityModification, AttributeNames, BlockNumber, CachedEthereumCall, ChainStore, - Child, ChildMultiplicity, EntityCache, EntityChange, EntityChangeOperation, + write::EntityModification, AssignmentChange, AssignmentOperation, AttributeNames, + BlockNumber, CachedEthereumCall, ChainStore, Child, ChildMultiplicity, EntityCache, EntityCollection, EntityFilter, EntityLink, EntityOperation, EntityOrder, EntityOrderByChild, EntityOrderByChildInfo, EntityQuery, EntityRange, EntityWindow, EthereumCallCache, ParentLink, PartialBlockPtr, PoolWaitStats, QueryStore, diff --git a/store/postgres/src/primary.rs b/store/postgres/src/primary.rs index d62ab74da06..ab6be9ee0ba 100644 --- a/store/postgres/src/primary.rs +++ b/store/postgres/src/primary.rs @@ -39,8 +39,8 @@ use graph::{ prelude::{ anyhow, chrono::{DateTime, Utc}, - serde_json, DeploymentHash, EntityChange, EntityChangeOperation, NodeId, StoreError, - SubgraphName, SubgraphVersionSwitchingMode, + serde_json, AssignmentChange, DeploymentHash, NodeId, StoreError, SubgraphName, + SubgraphVersionSwitchingMode, }, }; use graph::{ @@ -790,7 +790,7 @@ impl<'a> Connection<'a> { /// Delete all assignments for deployments that are neither the current nor the /// pending version of a subgraph and return the deployment id's - fn remove_unused_assignments(&mut self) -> Result, StoreError> { + fn remove_unused_assignments(&mut self) -> Result, StoreError> { use deployment_schemas as ds; use subgraph as s; use subgraph_deployment_assignment as a; @@ -827,12 +827,7 @@ impl<'a> Connection<'a> { .into_iter() .map(|(id, hash)| { DeploymentHash::new(hash) - .map(|hash| { - EntityChange::for_assignment( - DeploymentLocator::new(id.into(), hash), - EntityChangeOperation::Removed, - ) - }) + .map(|hash| AssignmentChange::removed(DeploymentLocator::new(id.into(), hash))) .map_err(|id| { StoreError::ConstraintViolation(format!( "invalid id `{}` for deployment assignment", @@ -851,7 +846,7 @@ impl<'a> Connection<'a> { pub fn promote_deployment( &mut self, id: &DeploymentHash, - ) -> Result, StoreError> { + ) -> Result, StoreError> { use subgraph as s; use subgraph_version as v; @@ -926,7 +921,7 @@ impl<'a> Connection<'a> { node_id: NodeId, mode: SubgraphVersionSwitchingMode, exists_and_synced: F, - ) -> Result, StoreError> + ) -> Result, StoreError> where F: Fn(&DeploymentHash) -> Result, { @@ -1034,13 +1029,16 @@ impl<'a> Connection<'a> { // Clean up any assignments we might have displaced let mut changes = self.remove_unused_assignments()?; if new_assignment { - let change = EntityChange::for_assignment(site.into(), EntityChangeOperation::Set); + let change = AssignmentChange::set(site.into()); changes.push(change); } Ok(changes) } - pub fn remove_subgraph(&mut self, name: SubgraphName) -> Result, StoreError> { + pub fn remove_subgraph( + &mut self, + name: SubgraphName, + ) -> Result, StoreError> { use subgraph as s; use subgraph_version as v; @@ -1062,7 +1060,7 @@ impl<'a> Connection<'a> { } } - pub fn pause_subgraph(&mut self, site: &Site) -> Result, StoreError> { + pub fn pause_subgraph(&mut self, site: &Site) -> Result, StoreError> { use subgraph_deployment_assignment as a; let conn = self.conn.as_mut(); @@ -1073,8 +1071,7 @@ impl<'a> Connection<'a> { match updates { 0 => Err(StoreError::DeploymentNotFound(site.deployment.to_string())), 1 => { - let change = - EntityChange::for_assignment(site.into(), EntityChangeOperation::Removed); + let change = AssignmentChange::removed(site.into()); Ok(vec![change]) } _ => { @@ -1085,7 +1082,7 @@ impl<'a> Connection<'a> { } } - pub fn resume_subgraph(&mut self, site: &Site) -> Result, StoreError> { + pub fn resume_subgraph(&mut self, site: &Site) -> Result, StoreError> { use subgraph_deployment_assignment as a; let conn = self.conn.as_mut(); @@ -1096,7 +1093,7 @@ impl<'a> Connection<'a> { match updates { 0 => Err(StoreError::DeploymentNotFound(site.deployment.to_string())), 1 => { - let change = EntityChange::for_assignment(site.into(), EntityChangeOperation::Set); + let change = AssignmentChange::set(site.into()); Ok(vec![change]) } _ => { @@ -1111,7 +1108,7 @@ impl<'a> Connection<'a> { &mut self, site: &Site, node: &NodeId, - ) -> Result, StoreError> { + ) -> Result, StoreError> { use subgraph_deployment_assignment as a; let conn = self.conn.as_mut(); @@ -1121,7 +1118,7 @@ impl<'a> Connection<'a> { match updates { 0 => Err(StoreError::DeploymentNotFound(site.deployment.to_string())), 1 => { - let change = EntityChange::for_assignment(site.into(), EntityChangeOperation::Set); + let change = AssignmentChange::set(site.into()); Ok(vec![change]) } _ => { @@ -1248,7 +1245,7 @@ impl<'a> Connection<'a> { &mut self, site: &Site, node: &NodeId, - ) -> Result, StoreError> { + ) -> Result, StoreError> { use subgraph_deployment_assignment as a; let conn = self.conn.as_mut(); @@ -1256,11 +1253,11 @@ impl<'a> Connection<'a> { .values((a::id.eq(site.id), a::node_id.eq(node.as_str()))) .execute(conn)?; - let change = EntityChange::for_assignment(site.into(), EntityChangeOperation::Set); + let change = AssignmentChange::set(site.into()); Ok(vec![change]) } - pub fn unassign_subgraph(&mut self, site: &Site) -> Result, StoreError> { + pub fn unassign_subgraph(&mut self, site: &Site) -> Result, StoreError> { use subgraph_deployment_assignment as a; let conn = self.conn.as_mut(); @@ -1271,8 +1268,7 @@ impl<'a> Connection<'a> { match delete_count { 0 => Ok(vec![]), 1 => { - let change = - EntityChange::for_assignment(site.into(), EntityChangeOperation::Removed); + let change = AssignmentChange::removed(site.into()); Ok(vec![change]) } _ => { diff --git a/store/test-store/tests/postgres/subgraph.rs b/store/test-store/tests/postgres/subgraph.rs index 3065c8800ef..70925cf70f6 100644 --- a/store/test-store/tests/postgres/subgraph.rs +++ b/store/test-store/tests/postgres/subgraph.rs @@ -10,9 +10,8 @@ use graph::{ schema::{DeploymentCreate, SubgraphError}, DeploymentFeatures, }, + prelude::AssignmentChange, prelude::BlockPtr, - prelude::EntityChange, - prelude::EntityChangeOperation, prelude::QueryStoreManager, prelude::StoreEvent, prelude::SubgraphManifest, @@ -59,18 +58,12 @@ const SUBGRAPH_FEATURES_GQL: &str = " } "; -fn assigned(deployment: &DeploymentLocator) -> EntityChange { - EntityChange::Assignment { - deployment: deployment.clone(), - operation: EntityChangeOperation::Set, - } +fn assigned(deployment: &DeploymentLocator) -> AssignmentChange { + AssignmentChange::set(deployment.clone()) } -fn unassigned(deployment: &DeploymentLocator) -> EntityChange { - EntityChange::Assignment { - deployment: deployment.clone(), - operation: EntityChangeOperation::Removed, - } +fn unassigned(deployment: &DeploymentLocator) -> AssignmentChange { + AssignmentChange::removed(deployment.clone()) } fn get_version_info(store: &Store, subgraph_name: &str) -> VersionInfo { @@ -163,7 +156,7 @@ fn create_subgraph() { store: &SubgraphStore, id: &str, mode: SubgraphVersionSwitchingMode, - ) -> (DeploymentLocator, HashSet) { + ) -> (DeploymentLocator, HashSet) { let name = SubgraphName::new(SUBGRAPH_NAME.to_string()).unwrap(); let id = DeploymentHash::new(id.to_string()).unwrap(); let schema = InputSchema::parse_latest(SUBGRAPH_GQL, id.clone()).unwrap(); @@ -203,7 +196,7 @@ fn create_subgraph() { (deployment, events) } - fn deploy_event(deployment: &DeploymentLocator) -> HashSet { + fn deploy_event(deployment: &DeploymentLocator) -> HashSet { let mut changes = HashSet::new(); changes.insert(assigned(deployment)); changes From 73c42e46502c5b7bdcd8e8e0d3799d9dbf1f7116 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Tue, 18 Feb 2025 16:15:30 -0800 Subject: [PATCH 012/150] graph: Remove unused SubscriptionServer trait --- graph/src/components/server/mod.rs | 3 --- graph/src/components/server/subscription.rs | 8 -------- graph/src/lib.rs | 1 - 3 files changed, 12 deletions(-) delete mode 100644 graph/src/components/server/subscription.rs diff --git a/graph/src/components/server/mod.rs b/graph/src/components/server/mod.rs index 404a9bfcf4f..89323b9c8b1 100644 --- a/graph/src/components/server/mod.rs +++ b/graph/src/components/server/mod.rs @@ -1,9 +1,6 @@ /// Component for running GraphQL queries over HTTP. pub mod query; -/// Component for running GraphQL subscriptions over WebSockets. -pub mod subscription; - /// Component for the index node server. pub mod index_node; diff --git a/graph/src/components/server/subscription.rs b/graph/src/components/server/subscription.rs deleted file mode 100644 index dae619356b6..00000000000 --- a/graph/src/components/server/subscription.rs +++ /dev/null @@ -1,8 +0,0 @@ -use async_trait::async_trait; - -/// Common trait for GraphQL subscription servers. -#[async_trait] -pub trait SubscriptionServer { - /// Returns a Future that, when spawned, brings up the GraphQL subscription server. - async fn serve(self, port: u16); -} diff --git a/graph/src/lib.rs b/graph/src/lib.rs index 19139cd2804..7d1536929bd 100644 --- a/graph/src/lib.rs +++ b/graph/src/lib.rs @@ -125,7 +125,6 @@ pub mod prelude { stopwatch::StopwatchMetrics, subgraph::*, Collector, Counter, CounterVec, Gauge, GaugeVec, Histogram, HistogramOpts, HistogramVec, MetricsRegistry, Opts, PrometheusError, Registry, }; - pub use crate::components::server::subscription::SubscriptionServer; pub use crate::components::store::{ write::EntityModification, AssignmentChange, AssignmentOperation, AttributeNames, BlockNumber, CachedEthereumCall, ChainStore, Child, ChildMultiplicity, EntityCache, From a3ca3196a862a471fac5621ef52af898acb1333c Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Tue, 18 Feb 2025 16:17:51 -0800 Subject: [PATCH 013/150] docs, graph: Remove unused env var GRAPH_DISABLE_SUBSCRIPTION_NOTIFICATIONS --- docs/environment-variables.md | 4 ---- graph/src/env/store.rs | 10 ---------- 2 files changed, 14 deletions(-) diff --git a/docs/environment-variables.md b/docs/environment-variables.md index 5d2b501f2a6..04a73896a87 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -120,10 +120,6 @@ those. - `GRAPH_SQL_STATEMENT_TIMEOUT`: the maximum number of seconds an individual SQL query is allowed to take during GraphQL execution. Default: unlimited -- `GRAPH_DISABLE_SUBSCRIPTION_NOTIFICATIONS`: disables the internal - mechanism that is used to trigger updates on GraphQL subscriptions. When - this variable is set to any value, `graph-node` will still accept GraphQL - subscriptions, but they won't receive any updates. - `ENABLE_GRAPHQL_VALIDATIONS`: enables GraphQL validations, based on the GraphQL specification. This will validate and ensure every query executes follows the execution rules. Default: `false` diff --git a/graph/src/env/store.rs b/graph/src/env/store.rs index af0f076978f..148e5641088 100644 --- a/graph/src/env/store.rs +++ b/graph/src/env/store.rs @@ -49,13 +49,6 @@ pub struct EnvVarsStore { /// only as an emergency setting for the hosted service. Remove after /// 2022-07-01 if hosted service had no issues with it being `true` pub order_by_block_range: bool, - /// Whether to disable the notifications that feed GraphQL - /// subscriptions. When the flag is set, no updates - /// about entity changes will be sent to query nodes. - /// - /// Set by the flag `GRAPH_DISABLE_SUBSCRIPTION_NOTIFICATIONS`. Not set - /// by default. - pub disable_subscription_notifications: bool, /// Set by the environment variable `GRAPH_REMOVE_UNUSED_INTERVAL` /// (expressed in minutes). The default value is 360 minutes. pub remove_unused_interval: chrono::Duration, @@ -163,7 +156,6 @@ impl From for EnvVarsStore { typea_batch_size: x.typea_batch_size, typed_children_set_size: x.typed_children_set_size, order_by_block_range: x.order_by_block_range.0, - disable_subscription_notifications: x.disable_subscription_notifications.0, remove_unused_interval: chrono::Duration::minutes( x.remove_unused_interval_in_minutes as i64, ), @@ -207,8 +199,6 @@ pub struct InnerStore { typed_children_set_size: usize, #[envconfig(from = "ORDER_BY_BLOCK_RANGE", default = "true")] order_by_block_range: EnvVarBoolean, - #[envconfig(from = "GRAPH_DISABLE_SUBSCRIPTION_NOTIFICATIONS", default = "false")] - disable_subscription_notifications: EnvVarBoolean, #[envconfig(from = "GRAPH_REMOVE_UNUSED_INTERVAL", default = "360")] remove_unused_interval_in_minutes: u64, #[envconfig(from = "GRAPH_STORE_RECENT_BLOCKS_CACHE_CAPACITY", default = "10")] From 0f0940a2b73906e8ce96ec2e8f615e31eb01727c Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Tue, 18 Feb 2025 16:36:26 -0800 Subject: [PATCH 014/150] graph: Remove unused file (has been unused for a long time) --- graph/src/data/graphql/visitor.rs | 62 ------------------------------- 1 file changed, 62 deletions(-) delete mode 100644 graph/src/data/graphql/visitor.rs diff --git a/graph/src/data/graphql/visitor.rs b/graph/src/data/graphql/visitor.rs deleted file mode 100644 index 94d26c08644..00000000000 --- a/graph/src/data/graphql/visitor.rs +++ /dev/null @@ -1,62 +0,0 @@ -use crate::prelude::q; - -pub trait Visitor { - fn enter_field(&mut self, _: &q::Field) -> Result<(), E> { - Ok(()) - } - fn leave_field(&mut self, _: &mut q::Field) -> Result<(), E> { - Ok(()) - } - - fn enter_query(&mut self, _: &q::Query) -> Result<(), E> { - Ok(()) - } - fn leave_query(&mut self, _: &mut q::Query) -> Result<(), E> { - Ok(()) - } - - fn visit_fragment_spread(&mut self, _: &q::FragmentSpread) -> Result<(), E> { - Ok(()) - } -} - -pub fn visit(visitor: &mut dyn Visitor, doc: &mut q::Document) -> Result<(), E> { - for def in &mut doc.definitions { - match def { - q::Definition::Operation(op) => match op { - q::OperationDefinition::SelectionSet(set) => { - visit_selection_set(visitor, set)?; - } - q::OperationDefinition::Query(query) => { - visitor.enter_query(query)?; - visit_selection_set(visitor, &mut query.selection_set)?; - visitor.leave_query(query)?; - } - q::OperationDefinition::Mutation(_) => todo!(), - q::OperationDefinition::Subscription(_) => todo!(), - }, - q::Definition::Fragment(frag) => {} - } - } - Ok(()) -} - -fn visit_selection_set( - visitor: &mut dyn Visitor, - set: &mut q::SelectionSet, -) -> Result<(), E> { - for sel in &mut set.items { - match sel { - q::Selection::Field(field) => { - visitor.enter_field(field)?; - visit_selection_set(visitor, &mut field.selection_set)?; - visitor.leave_field(field)?; - } - q::Selection::FragmentSpread(frag) => { - visitor.visit_fragment_spread(frag)?; - } - q::Selection::InlineFragment(frag) => {} - } - } - Ok(()) -} From 8d35904c3a093ac6a2a3073f2e539e0ee1dcaebe Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Tue, 18 Feb 2025 16:26:54 -0800 Subject: [PATCH 015/150] graph, graphql: Remove Subscription type from API schema --- graph/src/data/graphql/ext.rs | 17 - graph/src/data/query/error.rs | 14 - graph/src/schema/api.rs | 50 +-- graphql/src/execution/query.rs | 47 +-- graphql/src/introspection/resolver.rs | 5 +- graphql/src/query/mod.rs | 10 +- .../tests/graphql/mock_introspection.json | 338 +----------------- 7 files changed, 14 insertions(+), 467 deletions(-) diff --git a/graph/src/data/graphql/ext.rs b/graph/src/data/graphql/ext.rs index 7e873353984..271ace79237 100644 --- a/graph/src/data/graphql/ext.rs +++ b/graph/src/data/graphql/ext.rs @@ -52,8 +52,6 @@ pub trait DocumentExt { fn get_root_query_type(&self) -> Option<&ObjectType>; - fn get_root_subscription_type(&self) -> Option<&ObjectType>; - fn object_or_interface(&self, name: &str) -> Option>; fn get_named_type(&self, name: &str) -> Option<&TypeDefinition>; @@ -159,21 +157,6 @@ impl DocumentExt for Document { .next() } - fn get_root_subscription_type(&self) -> Option<&ObjectType> { - self.definitions - .iter() - .filter_map(|d| match d { - Definition::TypeDefinition(TypeDefinition::Object(t)) - if t.name == "Subscription" => - { - Some(t) - } - _ => None, - }) - .peekable() - .next() - } - fn object_or_interface(&self, name: &str) -> Option> { match self.get_named_type(name) { Some(TypeDefinition::Object(t)) => Some(t.into()), diff --git a/graph/src/data/query/error.rs b/graph/src/data/query/error.rs index e8b63422cdf..65fc1bcd259 100644 --- a/graph/src/data/query/error.rs +++ b/graph/src/data/query/error.rs @@ -26,7 +26,6 @@ pub enum QueryExecutionError { OperationNameRequired, OperationNotFound(String), NotSupported(String), - NoRootSubscriptionObjectType, NonNullError(Pos, String), ListValueError(Pos, String), NamedTypeError(String), @@ -42,7 +41,6 @@ pub enum QueryExecutionError { FilterNotSupportedError(String, String), UnknownField(Pos, String, String), EmptyQuery, - MultipleSubscriptionFields, SubgraphDeploymentIdError(String), RangeArgumentsError(&'static str, u32, i64), InvalidFilterError, @@ -67,7 +65,6 @@ pub enum QueryExecutionError { Throttled, UndefinedFragment(String), Panic(String), - EventStreamError, FulltextQueryRequiresFilter, FulltextQueryInvalidSyntax(String), DeploymentReverted, @@ -87,7 +84,6 @@ impl QueryExecutionError { OperationNameRequired | OperationNotFound(_) | NotSupported(_) - | NoRootSubscriptionObjectType | NonNullError(_, _) | NamedTypeError(_) | AbstractTypeError(_) @@ -101,7 +97,6 @@ impl QueryExecutionError { | ChildFilterNestingNotSupportedError(_, _) | UnknownField(_, _, _) | EmptyQuery - | MultipleSubscriptionFields | SubgraphDeploymentIdError(_) | InvalidFilterError | EntityFieldError(_, _) @@ -127,7 +122,6 @@ impl QueryExecutionError { | TooComplex(_, _) | TooDeep(_) | Panic(_) - | EventStreamError | TooExpensive | Throttled | DeploymentReverted @@ -166,9 +160,6 @@ impl fmt::Display for QueryExecutionError { write!(f, "{}", message) } NotSupported(s) => write!(f, "Not supported: {}", s), - NoRootSubscriptionObjectType => { - write!(f, "No root Subscription type defined in the schema") - } NonNullError(_, s) => { write!(f, "Null value resolved for non-null field `{}`", s) } @@ -212,10 +203,6 @@ impl fmt::Display for QueryExecutionError { write!(f, "Type `{}` has no field `{}`", t, s) } EmptyQuery => write!(f, "The query is empty"), - MultipleSubscriptionFields => write!( - f, - "Only a single top-level field is allowed in subscriptions" - ), SubgraphDeploymentIdError(s) => { write!(f, "Failed to get subgraph ID from type: `{}`", s) } @@ -276,7 +263,6 @@ impl fmt::Display for QueryExecutionError { CyclicalFragment(name) =>write!(f, "query has fragment cycle including `{}`", name), UndefinedFragment(frag_name) => write!(f, "fragment `{}` is not defined", frag_name), Panic(msg) => write!(f, "panic processing query: {}", msg), - EventStreamError => write!(f, "error in the subscription event stream"), FulltextQueryRequiresFilter => write!(f, "fulltext search queries can only use EntityFilter::Equal"), FulltextQueryInvalidSyntax(msg) => write!(f, "Invalid fulltext search query syntax. Error: {}. Hint: Search terms with spaces need to be enclosed in single quotes", msg), TooExpensive => write!(f, "query is too expensive"), diff --git a/graph/src/schema/api.rs b/graph/src/schema/api.rs index 6d936177b67..db179b049d2 100644 --- a/graph/src/schema/api.rs +++ b/graph/src/schema/api.rs @@ -90,8 +90,8 @@ impl TryFrom<&r::Value> for ErrorPolicy { /// /// (2) By parsing an appropriate GraphQL schema from text and calling /// `from_graphql_schema`. In that case, it's the caller's responsibility to -/// make sure that the schema has all the types needed for querying, like -/// `Query` and `Subscription` +/// make sure that the schema has all the types needed for querying, in +/// particular `Query` /// /// Because of the second point, once constructed, it can not be assumed /// that an `ApiSchema` is based on an `InputSchema` and it can only be used @@ -102,7 +102,6 @@ pub struct ApiSchema { // Root types for the api schema. pub query_type: Arc, - pub subscription_type: Option>, object_types: HashMap>, } @@ -121,11 +120,6 @@ impl ApiSchema { .get_root_query_type() .context("no root `Query` in the schema")? .clone(); - let subscription_type = schema - .document - .get_root_subscription_type() - .cloned() - .map(Arc::new); let object_types = HashMap::from_iter( schema @@ -138,7 +132,6 @@ impl ApiSchema { Ok(Self { schema, query_type: Arc::new(query_type), - subscription_type, object_types, }) } @@ -360,7 +353,6 @@ pub(in crate::schema) fn api_schema( add_types_for_interface_types(&mut api, input_schema)?; add_types_for_aggregation_types(&mut api, input_schema)?; add_query_type(&mut api.document, input_schema)?; - add_subscription_type(&mut api.document, input_schema)?; Ok(api.document) } @@ -1135,44 +1127,6 @@ fn query_field_for_fulltext(fulltext: &s::Directive) -> Option { }) } -/// Adds a root `Subscription` object type to the schema. -fn add_subscription_type( - api: &mut s::Document, - input_schema: &InputSchema, -) -> Result<(), APISchemaError> { - let type_name = String::from("Subscription"); - - if api.get_named_type(&type_name).is_some() { - return Err(APISchemaError::TypeExists(type_name)); - } - - let mut fields: Vec = input_schema - .object_types() - .map(|(name, _)| name) - .chain(input_schema.interface_types().map(|(name, _)| name)) - .flat_map(|name| query_fields_for_type(name, FilterOps::Object)) - .collect(); - let mut agg_fields = input_schema - .aggregation_types() - .map(|(name, _)| name) - .flat_map(query_fields_for_agg_type) - .collect::>(); - fields.append(&mut agg_fields); - fields.push(meta_field()); - - let typedef = s::TypeDefinition::Object(s::ObjectType { - position: Pos::default(), - description: None, - name: type_name, - implements_interfaces: vec![], - directives: vec![], - fields, - }); - let def = s::Definition::TypeDefinition(typedef); - api.definitions.push(def); - Ok(()) -} - fn block_argument() -> s::InputValue { s::InputValue { position: Pos::default(), diff --git a/graphql/src/execution/query.rs b/graphql/src/execution/query.rs index 4dfdd6c25b0..e8593f27fba 100644 --- a/graphql/src/execution/query.rs +++ b/graphql/src/execution/query.rs @@ -35,7 +35,6 @@ lazy_static! { vec![ Box::new(UniqueOperationNames::new()), Box::new(LoneAnonymousOperation::new()), - Box::new(SingleFieldSubscriptions::new()), Box::new(KnownTypeNames::new()), Box::new(FragmentsOnCompositeTypes::new()), Box::new(VariablesAreInputTypes::new()), @@ -69,12 +68,6 @@ pub enum ComplexityError { CyclicalFragment(String), } -#[derive(Copy, Clone)] -enum Kind { - Query, - Subscription, -} - /// Helper to log the fields in a `SelectionSet` without cloning. Writes /// a list of field names from the selection set separated by ';'. Using /// ';' as a separator makes parsing the log a little easier since slog @@ -130,8 +123,6 @@ pub struct Query { start: Instant, - kind: Kind, - /// Used only for logging; if logging is configured off, these will /// have dummy values pub query_text: Arc, @@ -226,14 +217,14 @@ impl Query { let operation = operation.ok_or(QueryExecutionError::OperationNameRequired)?; let variables = coerce_variables(schema.as_ref(), &operation, query.variables)?; - let (kind, selection_set) = match operation { - q::OperationDefinition::Query(q::Query { selection_set, .. }) => { - (Kind::Query, selection_set) - } + let selection_set = match operation { + q::OperationDefinition::Query(q::Query { selection_set, .. }) => selection_set, // Queries can be run by just sending a selection set - q::OperationDefinition::SelectionSet(selection_set) => (Kind::Query, selection_set), - q::OperationDefinition::Subscription(q::Subscription { selection_set, .. }) => { - (Kind::Subscription, selection_set) + q::OperationDefinition::SelectionSet(selection_set) => selection_set, + q::OperationDefinition::Subscription(_) => { + return Err(vec![QueryExecutionError::NotSupported( + "Subscriptions are not supported".to_owned(), + )]) } q::OperationDefinition::Mutation(_) => { return Err(vec![QueryExecutionError::NotSupported( @@ -243,10 +234,8 @@ impl Query { }; let start = Instant::now(); - let root_type = match kind { - Kind::Query => schema.query_type.as_ref(), - Kind::Subscription => schema.subscription_type.as_ref().unwrap(), - }; + let root_type = schema.query_type.as_ref(); + // Use an intermediate struct so we can modify the query before // enclosing it in an Arc let raw_query = RawQuery { @@ -269,7 +258,6 @@ impl Query { schema, selection_set: Arc::new(selection_set), shape_hash: query.shape_hash, - kind, network, logger, start, @@ -345,23 +333,6 @@ impl Query { Ok(bcs) } - /// Return `true` if this is a query, and not a subscription or - /// mutation - pub fn is_query(&self) -> bool { - match self.kind { - Kind::Query => true, - Kind::Subscription => false, - } - } - - /// Return `true` if this is a subscription, not a query or a mutation - pub fn is_subscription(&self) -> bool { - match self.kind { - Kind::Subscription => true, - Kind::Query => false, - } - } - /// Log details about the overall execution of the query pub fn log_execution(&self, block: BlockNumber) { if ENV_VARS.log_gql_timing() { diff --git a/graphql/src/introspection/resolver.rs b/graphql/src/introspection/resolver.rs index 534cb6aa729..0f67b717c5a 100644 --- a/graphql/src/introspection/resolver.rs +++ b/graphql/src/introspection/resolver.rs @@ -332,10 +332,7 @@ impl IntrospectionResolver { self.type_objects .get(&String::from("Query")) .cloned(), - subscriptionType: - self.type_objects - .get(&String::from("Subscription")) - .cloned(), + subscriptionType: r::Value::Null, mutationType: r::Value::Null, types: self.type_objects.values().cloned().collect::>(), directives: self.directives.clone(), diff --git a/graphql/src/query/mod.rs b/graphql/src/query/mod.rs index a5a01f39ca3..641eb4581bb 100644 --- a/graphql/src/query/mod.rs +++ b/graphql/src/query/mod.rs @@ -1,6 +1,6 @@ use graph::{ data::query::CacheStatus, - prelude::{BlockPtr, CheapClone, QueryExecutionError, QueryResult}, + prelude::{BlockPtr, CheapClone, QueryResult}, }; use std::sync::Arc; use std::time::Instant; @@ -54,14 +54,6 @@ where trace: options.trace, }); - if !query.is_query() { - return ( - Arc::new( - QueryExecutionError::NotSupported("Only queries are supported".to_string()).into(), - ), - CacheStatus::default(), - ); - } let selection_set = selection_set .map(Arc::new) .unwrap_or_else(|| query.selection_set.cheap_clone()); diff --git a/store/test-store/tests/graphql/mock_introspection.json b/store/test-store/tests/graphql/mock_introspection.json index 11a3ed6fdec..9ebb47acb5d 100644 --- a/store/test-store/tests/graphql/mock_introspection.json +++ b/store/test-store/tests/graphql/mock_introspection.json @@ -4,9 +4,7 @@ "name": "Query" }, "mutationType": null, - "subscriptionType": { - "name": "Subscription" - }, + "subscriptionType": null, "types": [ { "kind": "ENUM", @@ -762,340 +760,6 @@ "enumValues": null, "possibleTypes": null }, - { - "kind": "OBJECT", - "name": "Subscription", - "description": null, - "fields": [ - { - "name": "user", - "description": null, - "args": [ - { - "name": "id", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "ID", - "ofType": null - } - }, - "defaultValue": null - }, - { - "name": "block", - "description": "The block at which the query should be executed. Can either be a `{ hash: Bytes }` value containing a block hash, a `{ number: Int }` containing the block number, or a `{ number_gte: Int }` containing the minimum block number. In the case of `number_gte`, the query will be executed on the latest block only if the subgraph has progressed to or past the minimum block number. Defaults to the latest block when omitted.", - "type": { - "kind": "INPUT_OBJECT", - "name": "Block_height", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "subgraphError", - "description": "Set to `allow` to receive data even if the subgraph has skipped over errors while syncing.", - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "ENUM", - "name": "_SubgraphErrorPolicy_", - "ofType": null - } - }, - "defaultValue": "deny" - } - ], - "type": { - "kind": "OBJECT", - "name": "User", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "users", - "description": null, - "args": [ - { - "name": "skip", - "description": null, - "type": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - }, - "defaultValue": "0" - }, - { - "name": "first", - "description": null, - "type": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - }, - "defaultValue": "100" - }, - { - "name": "orderBy", - "description": null, - "type": { - "kind": "ENUM", - "name": "User_orderBy", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "orderDirection", - "description": null, - "type": { - "kind": "ENUM", - "name": "OrderDirection", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "where", - "description": null, - "type": { - "kind": "INPUT_OBJECT", - "name": "User_filter", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "block", - "description": "The block at which the query should be executed. Can either be a `{ hash: Bytes }` value containing a block hash, a `{ number: Int }` containing the block number, or a `{ number_gte: Int }` containing the minimum block number. In the case of `number_gte`, the query will be executed on the latest block only if the subgraph has progressed to or past the minimum block number. Defaults to the latest block when omitted.", - "type": { - "kind": "INPUT_OBJECT", - "name": "Block_height", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "subgraphError", - "description": "Set to `allow` to receive data even if the subgraph has skipped over errors while syncing.", - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "ENUM", - "name": "_SubgraphErrorPolicy_", - "ofType": null - } - }, - "defaultValue": "deny" - } - ], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "OBJECT", - "name": "User", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "node", - "description": null, - "args": [ - { - "name": "id", - "description": null, - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "SCALAR", - "name": "ID", - "ofType": null - } - }, - "defaultValue": null - }, - { - "name": "block", - "description": "The block at which the query should be executed. Can either be a `{ hash: Bytes }` value containing a block hash, a `{ number: Int }` containing the block number, or a `{ number_gte: Int }` containing the minimum block number. In the case of `number_gte`, the query will be executed on the latest block only if the subgraph has progressed to or past the minimum block number. Defaults to the latest block when omitted.", - "type": { - "kind": "INPUT_OBJECT", - "name": "Block_height", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "subgraphError", - "description": "Set to `allow` to receive data even if the subgraph has skipped over errors while syncing.", - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "ENUM", - "name": "_SubgraphErrorPolicy_", - "ofType": null - } - }, - "defaultValue": "deny" - } - ], - "type": { - "kind": "INTERFACE", - "name": "Node", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "nodes", - "description": null, - "args": [ - { - "name": "skip", - "description": null, - "type": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - }, - "defaultValue": "0" - }, - { - "name": "first", - "description": null, - "type": { - "kind": "SCALAR", - "name": "Int", - "ofType": null - }, - "defaultValue": "100" - }, - { - "name": "orderBy", - "description": null, - "type": { - "kind": "ENUM", - "name": "Node_orderBy", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "orderDirection", - "description": null, - "type": { - "kind": "ENUM", - "name": "OrderDirection", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "where", - "description": null, - "type": { - "kind": "INPUT_OBJECT", - "name": "Node_filter", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "block", - "description": "The block at which the query should be executed. Can either be a `{ hash: Bytes }` value containing a block hash, a `{ number: Int }` containing the block number, or a `{ number_gte: Int }` containing the minimum block number. In the case of `number_gte`, the query will be executed on the latest block only if the subgraph has progressed to or past the minimum block number. Defaults to the latest block when omitted.", - "type": { - "kind": "INPUT_OBJECT", - "name": "Block_height", - "ofType": null - }, - "defaultValue": null - }, - { - "name": "subgraphError", - "description": "Set to `allow` to receive data even if the subgraph has skipped over errors while syncing.", - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "ENUM", - "name": "_SubgraphErrorPolicy_", - "ofType": null - } - }, - "defaultValue": "deny" - } - ], - "type": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "LIST", - "name": null, - "ofType": { - "kind": "NON_NULL", - "name": null, - "ofType": { - "kind": "INTERFACE", - "name": "Node", - "ofType": null - } - } - } - }, - "isDeprecated": false, - "deprecationReason": null - }, - { - "name": "_meta", - "description": "Access to subgraph metadata", - "args": [ - { - "name": "block", - "description": null, - "type": { - "kind": "INPUT_OBJECT", - "name": "Block_height", - "ofType": null - }, - "defaultValue": null - } - ], - "type": { - "kind": "OBJECT", - "name": "_Meta_", - "ofType": null - }, - "isDeprecated": false, - "deprecationReason": null - } - ], - "inputFields": null, - "interfaces": [], - "enumValues": null, - "possibleTypes": null - }, { "kind": "SCALAR", "name": "Timestamp", From a04a08e075aef0b23338a72de453b59f55a22d8e Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Tue, 18 Feb 2025 17:00:47 -0800 Subject: [PATCH 016/150] all: Remove ws routes and related settings --- docs/environment-variables.md | 13 ++++--------- graph/src/env/graphql.rs | 6 ------ node/src/main.rs | 4 +--- node/src/opt.rs | 8 -------- server/http/src/server.rs | 4 ++-- server/http/src/service.rs | 13 +++++-------- server/http/tests/server.rs | 8 ++++---- server/json-rpc/src/lib.rs | 15 ++------------- tests/src/config.rs | 4 ---- 9 files changed, 18 insertions(+), 57 deletions(-) diff --git a/docs/environment-variables.md b/docs/environment-variables.md index 04a73896a87..8b395680e6a 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -112,11 +112,7 @@ those. result is checked while the response is being constructed, so that execution does not take more memory than what is configured. The default value for both is unlimited. -- `GRAPH_GRAPHQL_MAX_OPERATIONS_PER_CONNECTION`: maximum number of GraphQL - operations per WebSocket connection. Any operation created after the limit - will return an error to the client. Default: 1000. - `GRAPH_GRAPHQL_HTTP_PORT` : Port for the GraphQL HTTP server -- `GRAPH_GRAPHQL_WS_PORT` : Port for the GraphQL WebSocket server - `GRAPH_SQL_STATEMENT_TIMEOUT`: the maximum number of seconds an individual SQL query is allowed to take during GraphQL execution. Default: unlimited @@ -181,11 +177,10 @@ those. query, and the `query_id` of the GraphQL query that caused the SQL query. These SQL queries are marked with `component: GraphQlRunner` There are additional SQL queries that get logged when `sql` is given. These are - queries caused by mappings when processing blocks for a subgraph, and - queries caused by subscriptions. If `cache` is present in addition to - `gql`, also logs information for each toplevel GraphQL query field - whether that could be retrieved from cache or not. Defaults to no - logging. + queries caused by mappings when processing blocks for a subgraph. If + `cache` is present in addition to `gql`, also logs information for each + toplevel GraphQL query field whether that could be retrieved from cache + or not. Defaults to no logging. - `GRAPH_LOG_TIME_FORMAT`: Custom log time format.Default value is `%b %d %H:%M:%S%.3f`. More information [here](https://docs.rs/chrono/latest/chrono/#formatting-and-parsing). - `STORE_CONNECTION_POOL_SIZE`: How many simultaneous connections to allow to the store. Due to implementation details, this value may not be strictly adhered to. Defaults to 10. diff --git a/graph/src/env/graphql.rs b/graph/src/env/graphql.rs index 23fab23cd49..4f1f9896488 100644 --- a/graph/src/env/graphql.rs +++ b/graph/src/env/graphql.rs @@ -86,9 +86,6 @@ pub struct EnvVarsGraphQl { /// Set by the environment variable `GRAPH_GRAPHQL_ERROR_RESULT_SIZE`. The /// default value is [`usize::MAX`]. pub error_result_size: usize, - /// Set by the flag `GRAPH_GRAPHQL_MAX_OPERATIONS_PER_CONNECTION`. - /// Defaults to 1000. - pub max_operations_per_connection: usize, /// Set by the flag `GRAPH_GRAPHQL_DISABLE_BOOL_FILTERS`. Off by default. /// Disables AND/OR filters pub disable_bool_filters: bool, @@ -144,7 +141,6 @@ impl From for EnvVarsGraphQl { allow_deployment_change: x.allow_deployment_change.0, warn_result_size: x.warn_result_size.0 .0, error_result_size: x.error_result_size.0 .0, - max_operations_per_connection: x.max_operations_per_connection, disable_bool_filters: x.disable_bool_filters.0, disable_child_sorting: x.disable_child_sorting.0, query_trace_token: x.query_trace_token, @@ -192,8 +188,6 @@ pub struct InnerGraphQl { warn_result_size: WithDefaultUsize, { usize::MAX }>, #[envconfig(from = "GRAPH_GRAPHQL_ERROR_RESULT_SIZE", default = "")] error_result_size: WithDefaultUsize, { usize::MAX }>, - #[envconfig(from = "GRAPH_GRAPHQL_MAX_OPERATIONS_PER_CONNECTION", default = "1000")] - max_operations_per_connection: usize, #[envconfig(from = "GRAPH_GRAPHQL_DISABLE_BOOL_FILTERS", default = "false")] pub disable_bool_filters: EnvVarBoolean, #[envconfig(from = "GRAPH_GRAPHQL_DISABLE_CHILD_SORTING", default = "false")] diff --git a/node/src/main.rs b/node/src/main.rs index 72ff4f9ee27..9b0e94250dc 100644 --- a/node/src/main.rs +++ b/node/src/main.rs @@ -141,7 +141,6 @@ async fn main() { // Obtain ports to use for the GraphQL server(s) let http_port = opt.http_port; - let ws_port = opt.ws_port; // Obtain JSON-RPC server port let json_rpc_port = opt.admin_port; @@ -442,7 +441,6 @@ async fn main() { let json_rpc_server = JsonRpcServer::serve( json_rpc_port, http_port, - ws_port, subgraph_registrar.clone(), node_id.clone(), logger.clone(), @@ -504,7 +502,7 @@ async fn main() { } // Serve GraphQL queries over HTTP - graph::spawn(async move { graphql_server.start(http_port, ws_port).await }); + graph::spawn(async move { graphql_server.start(http_port).await }); // Run the index node server graph::spawn(async move { index_node_server.start(index_node_port).await }); diff --git a/node/src/opt.rs b/node/src/opt.rs index e4dc44ba92a..9928144396a 100644 --- a/node/src/opt.rs +++ b/node/src/opt.rs @@ -132,14 +132,6 @@ pub struct Opt { help = "Port for the index node server" )] pub index_node_port: u16, - #[clap( - long, - default_value = "8001", - value_name = "PORT", - help = "Port for the GraphQL WebSocket server", - env = "GRAPH_GRAPHQL_WS_PORT" - )] - pub ws_port: u16, #[clap( long, default_value = "8020", diff --git a/server/http/src/server.rs b/server/http/src/server.rs index e02fb54fade..f5868cff5b8 100644 --- a/server/http/src/server.rs +++ b/server/http/src/server.rs @@ -32,7 +32,7 @@ impl GraphQLServer { } } - pub async fn start(&self, port: u16, ws_port: u16) -> Result { + pub async fn start(&self, port: u16) -> Result { let logger = self.logger.clone(); info!( @@ -42,7 +42,7 @@ impl GraphQLServer { let graphql_runner = self.graphql_runner.clone(); - let service = Arc::new(GraphQLService::new(logger.clone(), graphql_runner, ws_port)); + let service = Arc::new(GraphQLService::new(logger.clone(), graphql_runner)); start(logger, port, move |req| { let service = service.cheap_clone(); diff --git a/server/http/src/service.rs b/server/http/src/service.rs index ca03893b534..8e2237b86ff 100644 --- a/server/http/src/service.rs +++ b/server/http/src/service.rs @@ -48,7 +48,6 @@ fn client_error(msg: impl Into) -> ServerResponse { pub struct GraphQLService { logger: Logger, graphql_runner: Arc, - ws_port: u16, } impl GraphQLService @@ -56,17 +55,15 @@ where Q: GraphQlRunner, { /// Creates a new GraphQL service. - pub fn new(logger: Logger, graphql_runner: Arc, ws_port: u16) -> Self { + pub fn new(logger: Logger, graphql_runner: Arc) -> Self { GraphQLService { logger, graphql_runner, - ws_port, } } fn graphiql_html(&self) -> String { - include_str!("../assets/index.html") - .replace("__WS_PORT__", format!("{}", self.ws_port).as_str()) + include_str!("../assets/index.html").to_string() } async fn index(&self) -> ServerResult { @@ -459,7 +456,7 @@ mod tests { let logger = Logger::root(slog::Discard, o!()); let graphql_runner = Arc::new(TestGraphQlRunner); - let service = GraphQLService::new(logger, graphql_runner, 8001); + let service = GraphQLService::new(logger, graphql_runner); let request: Request> = Request::builder() .method(Method::GET) @@ -491,7 +488,7 @@ mod tests { let subgraph_id = USERS.clone(); let graphql_runner = Arc::new(TestGraphQlRunner); - let service = GraphQLService::new(logger, graphql_runner, 8001); + let service = GraphQLService::new(logger, graphql_runner); let request: Request> = Request::builder() .method(Method::POST) @@ -523,7 +520,7 @@ mod tests { let subgraph_id = USERS.clone(); let graphql_runner = Arc::new(TestGraphQlRunner); - let service = GraphQLService::new(logger, graphql_runner, 8001); + let service = GraphQLService::new(logger, graphql_runner); let request: Request> = Request::builder() .method(Method::POST) diff --git a/server/http/tests/server.rs b/server/http/tests/server.rs index 9970420ed2d..3ad78138437 100644 --- a/server/http/tests/server.rs +++ b/server/http/tests/server.rs @@ -165,7 +165,7 @@ mod test { let query_runner = Arc::new(TestGraphQlRunner); let server = HyperGraphQLServer::new(&logger_factory, query_runner); let server_handle = server - .start(8007, 8008) + .start(8007) .await .expect("Failed to start GraphQL server"); while !server_handle.accepting.load(Ordering::SeqCst) { @@ -197,7 +197,7 @@ mod test { let query_runner = Arc::new(TestGraphQlRunner); let server = HyperGraphQLServer::new(&logger_factory, query_runner); let server_handle = server - .start(8002, 8003) + .start(8002) .await .expect("Failed to start GraphQL server"); while !server_handle.accepting.load(Ordering::SeqCst) { @@ -269,7 +269,7 @@ mod test { let query_runner = Arc::new(TestGraphQlRunner); let server = HyperGraphQLServer::new(&logger_factory, query_runner); let server_handle = server - .start(8003, 8004) + .start(8003) .await .expect("Failed to start GraphQL server"); while !server_handle.accepting.load(Ordering::SeqCst) { @@ -306,7 +306,7 @@ mod test { let query_runner = Arc::new(TestGraphQlRunner); let server = HyperGraphQLServer::new(&logger_factory, query_runner); let server_handle = server - .start(8005, 8006) + .start(8005) .await .expect("Failed to start GraphQL server"); while !server_handle.accepting.load(Ordering::SeqCst) { diff --git a/server/json-rpc/src/lib.rs b/server/json-rpc/src/lib.rs index b8e5b0330b7..103d36f806c 100644 --- a/server/json-rpc/src/lib.rs +++ b/server/json-rpc/src/lib.rs @@ -21,7 +21,6 @@ impl JsonRpcServer { pub async fn serve( port: u16, http_port: u16, - ws_port: u16, registrar: Arc, node_id: NodeId, logger: Logger, @@ -39,7 +38,6 @@ impl JsonRpcServer { let state = ServerState { registrar, http_port, - ws_port, node_id, logger, }; @@ -87,7 +85,6 @@ impl JsonRpcServer { struct ServerState { registrar: Arc, http_port: u16, - ws_port: u16, node_id: NodeId, logger: Logger, } @@ -123,7 +120,7 @@ impl ServerState { info!(&self.logger, "Received subgraph_deploy request"; "params" => format!("{:?}", params)); let node_id = params.node_id.clone().unwrap_or(self.node_id.clone()); - let routes = subgraph_routes(¶ms.name, self.http_port, self.ws_port); + let routes = subgraph_routes(¶ms.name, self.http_port); match self .registrar .create_subgraph_version( @@ -243,15 +240,11 @@ fn json_rpc_error( ))) } -fn subgraph_routes(name: &SubgraphName, http_port: u16, ws_port: u16) -> JsonValue { +fn subgraph_routes(name: &SubgraphName, http_port: u16) -> JsonValue { let http_base_url = ENV_VARS .external_http_base_url .clone() .unwrap_or_else(|| format!(":{}", http_port)); - let ws_base_url = ENV_VARS - .external_ws_base_url - .clone() - .unwrap_or_else(|| format!(":{}", ws_port)); let mut map = BTreeMap::new(); map.insert( @@ -262,10 +255,6 @@ fn subgraph_routes(name: &SubgraphName, http_port: u16, ws_port: u16) -> JsonVal "queries", format!("{}/subgraphs/name/{}", http_base_url, name), ); - map.insert( - "subscriptions", - format!("{}/subgraphs/name/{}", ws_base_url, name), - ); serde_json::to_value(map).expect("invalid subgraph routes") } diff --git a/tests/src/config.rs b/tests/src/config.rs index 6762e542168..09e3b55fa47 100644 --- a/tests/src/config.rs +++ b/tests/src/config.rs @@ -61,7 +61,6 @@ impl EthConfig { pub struct GraphNodePorts { pub http: u16, pub index: u16, - pub ws: u16, pub admin: u16, pub metrics: u16, } @@ -70,7 +69,6 @@ impl Default for GraphNodePorts { fn default() -> Self { Self { http: 3030, - ws: 3031, admin: 3032, index: 3033, metrics: 3034, @@ -160,8 +158,6 @@ impl Config { &ports.http.to_string(), "--index-node-port", &ports.index.to_string(), - "--ws-port", - &ports.ws.to_string(), "--admin-port", &ports.admin.to_string(), "--metrics-port", From 67044013ad218f9f8a4aa8941e5718efb055f552 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Tue, 18 Feb 2025 17:56:59 -0800 Subject: [PATCH 017/150] all: Remove special handling of subscriptions from QueryStore --- graph/src/components/store/traits.rs | 3 --- graphql/src/runner.rs | 2 +- node/src/manager/commands/copy.rs | 8 +++---- server/graphman/tests/deployment_query.rs | 8 +++---- store/postgres/src/deployment_store.rs | 23 +++++++------------ store/postgres/src/store.rs | 3 +-- store/postgres/src/subgraph_store.rs | 3 +-- store/test-store/src/store.rs | 10 ++++---- store/test-store/tests/postgres/chain_head.rs | 8 +++---- store/test-store/tests/postgres/subgraph.rs | 16 ++++++------- 10 files changed, 36 insertions(+), 48 deletions(-) diff --git a/graph/src/components/store/traits.rs b/graph/src/components/store/traits.rs index 9357663cddf..e2cd0640ee0 100644 --- a/graph/src/components/store/traits.rs +++ b/graph/src/components/store/traits.rs @@ -439,12 +439,9 @@ pub trait QueryStoreManager: Send + Sync + 'static { /// which deployment will be queried. It is not possible to use the id of the /// metadata subgraph, though the resulting store can be used to query /// metadata about the deployment `id` (but not metadata about other deployments). - /// - /// If `for_subscription` is true, the main replica will always be used. async fn query_store( &self, target: QueryTarget, - for_subscription: bool, ) -> Result, QueryExecutionError>; } diff --git a/graphql/src/runner.rs b/graphql/src/runner.rs index a70f1e752ac..96f30e8bc9d 100644 --- a/graphql/src/runner.rs +++ b/graphql/src/runner.rs @@ -103,7 +103,7 @@ where // point, and everything needs to go through the `store` we are // setting up here - let store = self.store.query_store(target.clone(), false).await?; + let store = self.store.query_store(target.clone()).await?; let state = store.deployment_state().await?; let network = Some(store.network_name().to_string()); let schema = store.api_schema()?; diff --git a/node/src/manager/commands/copy.rs b/node/src/manager/commands/copy.rs index 620e27eef6c..ab007ea319d 100644 --- a/node/src/manager/commands/copy.rs +++ b/node/src/manager/commands/copy.rs @@ -106,10 +106,10 @@ pub async fn create( let subgraph_store = store.subgraph_store(); let src = src.locate_unique(&primary)?; let query_store = store - .query_store( - QueryTarget::Deployment(src.hash.clone(), Default::default()), - true, - ) + .query_store(QueryTarget::Deployment( + src.hash.clone(), + Default::default(), + )) .await?; let network = query_store.network_name(); diff --git a/server/graphman/tests/deployment_query.rs b/server/graphman/tests/deployment_query.rs index 87ac04c3ca3..ee66323716c 100644 --- a/server/graphman/tests/deployment_query.rs +++ b/server/graphman/tests/deployment_query.rs @@ -60,10 +60,10 @@ fn graphql_returns_deployment_info() { let namespace = format!("sgd{}", locator.id); let node = SUBGRAPH_STORE.assigned_node(&locator).unwrap().unwrap(); let qs = STORE - .query_store( - QueryTarget::Deployment(locator.hash.clone(), Default::default()), - false, - ) + .query_store(QueryTarget::Deployment( + locator.hash.clone(), + Default::default(), + )) .await .expect("could get a query store"); let shard = qs.shard(); diff --git a/store/postgres/src/deployment_store.rs b/store/postgres/src/deployment_store.rs index 332d2806cbe..92b1a27a7e5 100644 --- a/store/postgres/src/deployment_store.rs +++ b/store/postgres/src/deployment_store.rs @@ -1434,23 +1434,16 @@ impl DeploymentStore { Ok(()) } - pub(crate) fn replica_for_query( - &self, - for_subscription: bool, - ) -> Result { + pub(crate) fn replica_for_query(&self) -> Result { use std::sync::atomic::Ordering; - let replica_id = match for_subscription { - // Pick a weighted ReplicaId. `replica_order` contains a list of - // replicas with repetitions according to their weight - false => { - let weights_count = self.replica_order.len(); - let index = - self.conn_round_robin_counter.fetch_add(1, Ordering::SeqCst) % weights_count; - *self.replica_order.get(index).unwrap() - } - // Subscriptions always go to the main replica. - true => ReplicaId::Main, + // Pick a weighted ReplicaId. `replica_order` contains a list of + // replicas with repetitions according to their weight + let replica_id = { + let weights_count = self.replica_order.len(); + let index = + self.conn_round_robin_counter.fetch_add(1, Ordering::SeqCst) % weights_count; + *self.replica_order.get(index).unwrap() }; Ok(replica_id) diff --git a/store/postgres/src/store.rs b/store/postgres/src/store.rs index b663053c3da..50a5e4b21e0 100644 --- a/store/postgres/src/store.rs +++ b/store/postgres/src/store.rs @@ -70,7 +70,6 @@ impl QueryStoreManager for Store { async fn query_store( &self, target: graph::data::query::QueryTarget, - for_subscription: bool, ) -> Result< Arc, graph::prelude::QueryExecutionError, @@ -80,7 +79,7 @@ impl QueryStoreManager for Store { let target = target.clone(); let (store, site, replica) = graph::spawn_blocking_allow_panic(move || { store - .replica_for_query(target.clone(), for_subscription) + .replica_for_query(target.clone()) .map_err(|e| e.into()) }) .await diff --git a/store/postgres/src/subgraph_store.rs b/store/postgres/src/subgraph_store.rs index a881aae27d1..e9f5f2cce34 100644 --- a/store/postgres/src/subgraph_store.rs +++ b/store/postgres/src/subgraph_store.rs @@ -811,7 +811,6 @@ impl SubgraphStoreInner { pub(crate) fn replica_for_query( &self, target: QueryTarget, - for_subscription: bool, ) -> Result<(Arc, Arc, ReplicaId), StoreError> { let id = match target { QueryTarget::Name(name, _) => self.mirror.current_deployment_for_subgraph(&name)?, @@ -819,7 +818,7 @@ impl SubgraphStoreInner { }; let (store, site) = self.store(&id)?; - let replica = store.replica_for_query(for_subscription)?; + let replica = store.replica_for_query()?; Ok((store.clone(), site, replica)) } diff --git a/store/test-store/src/store.rs b/store/test-store/src/store.rs index ca0e97f7c2f..2fa96148ba9 100644 --- a/store/test-store/src/store.rs +++ b/store/test-store/src/store.rs @@ -530,7 +530,7 @@ async fn execute_subgraph_query_internal( let deployment = query.schema.id().clone(); let store = STORE .clone() - .query_store(QueryTarget::Deployment(deployment, version.clone()), false) + .query_store(QueryTarget::Deployment(deployment, version.clone())) .await .unwrap(); let state = store.deployment_state().await.unwrap(); @@ -571,10 +571,10 @@ async fn execute_subgraph_query_internal( pub async fn deployment_state(store: &Store, subgraph_id: &DeploymentHash) -> DeploymentState { store - .query_store( - QueryTarget::Deployment(subgraph_id.clone(), Default::default()), - false, - ) + .query_store(QueryTarget::Deployment( + subgraph_id.clone(), + Default::default(), + )) .await .expect("could get a query store") .deployment_state() diff --git a/store/test-store/tests/postgres/chain_head.rs b/store/test-store/tests/postgres/chain_head.rs index 3b840ba3566..c93eb624e59 100644 --- a/store/test-store/tests/postgres/chain_head.rs +++ b/store/test-store/tests/postgres/chain_head.rs @@ -219,10 +219,10 @@ fn test_get_block_number() { create_test_subgraph(&subgraph, "type Dummy @entity { id: ID! }").await; let query_store = subgraph_store - .query_store( - QueryTarget::Deployment(subgraph.cheap_clone(), Default::default()), - false, - ) + .query_store(QueryTarget::Deployment( + subgraph.cheap_clone(), + Default::default(), + )) .await .unwrap(); diff --git a/store/test-store/tests/postgres/subgraph.rs b/store/test-store/tests/postgres/subgraph.rs index 70925cf70f6..c66d34e27c7 100644 --- a/store/test-store/tests/postgres/subgraph.rs +++ b/store/test-store/tests/postgres/subgraph.rs @@ -696,10 +696,10 @@ fn fatal_vs_non_fatal() { run_test_sequentially(|store| async move { let deployment = setup().await; let query_store = store - .query_store( - QueryTarget::Deployment(deployment.hash.clone(), Default::default()), - false, - ) + .query_store(QueryTarget::Deployment( + deployment.hash.clone(), + Default::default(), + )) .await .unwrap(); @@ -757,10 +757,10 @@ fn fail_unfail_deterministic_error() { let deployment = setup().await; let query_store = store - .query_store( - QueryTarget::Deployment(deployment.hash.clone(), Default::default()), - false, - ) + .query_store(QueryTarget::Deployment( + deployment.hash.clone(), + Default::default(), + )) .await .unwrap(); From d4952114c21bdbb9f537d010c3f78aa568468cf3 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Thu, 13 Feb 2025 13:17:50 -0800 Subject: [PATCH 018/150] store: Defer remapping until the source shard has finished migrations Without this, starting `graph-node` on an empty database can cause an error that the fdw namespace does not exist --- store/postgres/src/connection_pool.rs | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/store/postgres/src/connection_pool.rs b/store/postgres/src/connection_pool.rs index c2795fca5db..ab44d956026 100644 --- a/store/postgres/src/connection_pool.rs +++ b/store/postgres/src/connection_pool.rs @@ -1015,12 +1015,12 @@ impl PoolInner { // in the database instead of just in memory let result = pool .configure_fdw(coord.servers.as_ref()) - .and_then(|()| migrate_schema(&pool.logger, &mut conn)) - .and_then(|count| coord.propagate(&pool, count)); + .and_then(|()| migrate_schema(&pool.logger, &mut conn)); debug!(&pool.logger, "Release migration lock"); advisory_lock::unlock_migration(&mut conn).unwrap_or_else(|err| { die(&pool.logger, "failed to release migration lock", &err); }); + let result = result.and_then(|count| coord.propagate(&pool, count)); result.unwrap_or_else(|err| die(&pool.logger, "migrations failed", &err)); // Locale check @@ -1233,6 +1233,10 @@ impl PoolCoordinator { /// other pools will then recreate any tables that they imported from /// `shard`. If `pool` is a new shard, we also map all other shards into /// it. + /// + /// This tries to take the migration lock and must therefore be run from + /// code that does _not_ hold the migration lock as it will otherwise + /// deadlock fn propagate(&self, pool: &PoolInner, count: MigrationCount) -> Result<(), StoreError> { // pool is a new shard, map all other shards into it if count.is_new() { @@ -1244,7 +1248,14 @@ impl PoolCoordinator { if count.had_migrations() { let server = self.server(&pool.shard)?; for pool in self.pools.lock().unwrap().values() { - if let Err(e) = pool.remap(server) { + let mut conn = pool.get()?; + let remap_res = { + advisory_lock::lock_migration(&mut conn)?; + let res = pool.remap(server); + advisory_lock::unlock_migration(&mut conn)?; + res + }; + if let Err(e) = remap_res { error!(pool.logger, "Failed to map imports from {}", server.shard; "error" => e.to_string()); return Err(e); } From 8b828f78d14df33f796f8b7d709437fed5e59b7a Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Thu, 13 Feb 2025 14:06:09 -0800 Subject: [PATCH 019/150] store: Map more tables across shards This now includes ethereum_networks, copy_state, copy_table_state, and subgraph_features. We no longer map subgraph, subgraph_version, and subgraph_deployment_assignment into the shard_* namespace since these tables are only maintained in the primary, and are mapped in the primary_public namespace. --- store/postgres/src/connection_pool.rs | 35 +++++++++++++++++---------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/store/postgres/src/connection_pool.rs b/store/postgres/src/connection_pool.rs index ab44d956026..00158cb29d7 100644 --- a/store/postgres/src/connection_pool.rs +++ b/store/postgres/src/connection_pool.rs @@ -206,22 +206,31 @@ impl ForeignServer { /// Map the `subgraphs` schema from the foreign server `self` into the /// database accessible through `conn` fn map_metadata(&self, conn: &mut PgConnection) -> Result<(), StoreError> { + const MAP_TABLES: [(&str, &[&str]); 2] = [ + ("public", &["ethereum_networks"]), + ( + "subgraphs", + &[ + "copy_state", + "copy_table_state", + "dynamic_ethereum_contract_data_source", + "subgraph_deployment", + "subgraph_error", + "subgraph_features", + "subgraph_manifest", + "table_stats", + ], + ), + ]; let nsp = Self::metadata_schema(&self.shard); catalog::recreate_schema(conn, &nsp)?; let mut query = String::new(); - for table_name in [ - "subgraph_error", - "dynamic_ethereum_contract_data_source", - "table_stats", - "subgraph_deployment_assignment", - "subgraph", - "subgraph_version", - "subgraph_deployment", - "subgraph_manifest", - ] { - let create_stmt = - catalog::create_foreign_table(conn, "subgraphs", table_name, &nsp, &self.name)?; - write!(query, "{}", create_stmt)?; + for (src_nsp, src_tables) in MAP_TABLES { + for src_table in src_tables { + let create_stmt = + catalog::create_foreign_table(conn, src_nsp, src_table, &nsp, &self.name)?; + write!(query, "{}", create_stmt)?; + } } Ok(conn.batch_execute(&query)?) } From 0e7b3bc3584fda39e3977395cf7bbe3cb8f235e2 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Thu, 13 Feb 2025 16:56:00 -0800 Subject: [PATCH 020/150] store: Create views in the primary that union the tables from shards Fixes https://github.com/graphprotocol/graph-node/issues/4824 Fixes https://github.com/graphprotocol/graph-node/issues/4825 --- store/postgres/src/catalog.rs | 52 ++++++++++++ store/postgres/src/connection_pool.rs | 113 ++++++++++++++++++++++---- 2 files changed, 147 insertions(+), 18 deletions(-) diff --git a/store/postgres/src/catalog.rs b/store/postgres/src/catalog.rs index 8e988e31522..1524a768acc 100644 --- a/store/postgres/src/catalog.rs +++ b/store/postgres/src/catalog.rs @@ -626,6 +626,58 @@ pub fn create_foreign_table( Ok(query) } +/// Create a SQL statement unioning imported tables from all shards, +/// something like +/// +/// ```sql +/// create view "dst_nsp"."src_table" as +/// select 'shard1' as shard, "col1", "col2" from "shard_shard1_subgraphs"."table_name" +/// union all +/// ... +/// ```` +/// +/// The list `shard_nsps` consists of pairs `(name, namespace)` where `name` +/// is the name of the shard and `namespace` is the namespace where the +/// `src_table` is mapped +pub fn create_cross_shard_view( + conn: &mut PgConnection, + src_nsp: &str, + src_table: &str, + dst_nsp: &str, + shard_nsps: &[(&str, String)], +) -> Result { + fn build_query( + columns: &[table_schema::Column], + table_name: &str, + dst_nsp: &str, + shard_nsps: &[(&str, String)], + ) -> Result { + let mut query = String::new(); + write!(query, "create view \"{}\".\"{}\" as ", dst_nsp, table_name)?; + for (idx, (name, nsp)) in shard_nsps.into_iter().enumerate() { + if idx > 0 { + write!(query, " union all ")?; + } + write!(query, "select '{name}' as shard")?; + for column in columns { + write!(query, ", \"{}\"", column.column_name)?; + } + writeln!(query, " from \"{}\".\"{}\"", nsp, table_name)?; + } + Ok(query) + } + + let columns = table_schema::columns(conn, src_nsp, src_table)?; + let query = build_query(&columns, src_table, dst_nsp, shard_nsps).map_err(|_| { + anyhow!( + "failed to generate 'create foreign table' query for {}.{}", + dst_nsp, + src_table + ) + })?; + Ok(query) +} + /// Checks in the database if a given index is valid. pub(crate) fn check_index_is_valid( conn: &mut PgConnection, diff --git a/store/postgres/src/connection_pool.rs b/store/postgres/src/connection_pool.rs index 00158cb29d7..ae8ab3e71b6 100644 --- a/store/postgres/src/connection_pool.rs +++ b/store/postgres/src/connection_pool.rs @@ -37,6 +37,23 @@ use crate::primary::{self, NAMESPACE_PUBLIC}; use crate::{advisory_lock, catalog}; use crate::{Shard, PRIMARY_SHARD}; +const SHARDED_TABLES: [(&str, &[&str]); 2] = [ + ("public", &["ethereum_networks"]), + ( + "subgraphs", + &[ + "copy_state", + "copy_table_state", + "dynamic_ethereum_contract_data_source", + "subgraph_deployment", + "subgraph_error", + "subgraph_features", + "subgraph_manifest", + "table_stats", + ], + ), +]; + pub struct ForeignServer { pub name: String, pub shard: Shard, @@ -49,6 +66,7 @@ pub struct ForeignServer { impl ForeignServer { pub(crate) const PRIMARY_PUBLIC: &'static str = "primary_public"; + pub(crate) const CROSS_SHARD_NSP: &'static str = "sharded"; /// The name of the foreign server under which data for `shard` is /// accessible @@ -206,26 +224,10 @@ impl ForeignServer { /// Map the `subgraphs` schema from the foreign server `self` into the /// database accessible through `conn` fn map_metadata(&self, conn: &mut PgConnection) -> Result<(), StoreError> { - const MAP_TABLES: [(&str, &[&str]); 2] = [ - ("public", &["ethereum_networks"]), - ( - "subgraphs", - &[ - "copy_state", - "copy_table_state", - "dynamic_ethereum_contract_data_source", - "subgraph_deployment", - "subgraph_error", - "subgraph_features", - "subgraph_manifest", - "table_stats", - ], - ), - ]; let nsp = Self::metadata_schema(&self.shard); catalog::recreate_schema(conn, &nsp)?; let mut query = String::new(); - for (src_nsp, src_tables) in MAP_TABLES { + for (src_nsp, src_tables) in SHARDED_TABLES { for src_table in src_tables { let create_stmt = catalog::create_foreign_table(conn, src_nsp, src_table, &nsp, &self.name)?; @@ -1024,7 +1026,12 @@ impl PoolInner { // in the database instead of just in memory let result = pool .configure_fdw(coord.servers.as_ref()) - .and_then(|()| migrate_schema(&pool.logger, &mut conn)); + .and_then(|()| pool.drop_cross_shard_views()) + .and_then(|()| migrate_schema(&pool.logger, &mut conn)) + .and_then(|count| { + pool.create_cross_shard_views(coord.servers.as_ref()) + .map(|()| count) + }); debug!(&pool.logger, "Release migration lock"); advisory_lock::unlock_migration(&mut conn).unwrap_or_else(|err| { die(&pool.logger, "failed to release migration lock", &err); @@ -1077,6 +1084,76 @@ impl PoolInner { }) } + /// If this is the primary shard, drop the namespace `CROSS_SHARD_NSP` + fn drop_cross_shard_views(&self) -> Result<(), StoreError> { + if self.shard != *PRIMARY_SHARD { + return Ok(()); + } + + info!(&self.logger, "Dropping cross-shard views"); + let mut conn = self.get()?; + conn.transaction(|conn| { + let query = format!( + "drop schema if exists {} cascade", + ForeignServer::CROSS_SHARD_NSP + ); + conn.batch_execute(&query)?; + Ok(()) + }) + } + + /// If this is the primary shard, create the namespace `CROSS_SHARD_NSP` + /// and populate it with tables that union various imported tables + fn create_cross_shard_views(&self, servers: &[ForeignServer]) -> Result<(), StoreError> { + fn shard_nsp_pairs<'a>( + current: &Shard, + local_nsp: &str, + servers: &'a [ForeignServer], + ) -> Vec<(&'a str, String)> { + servers + .into_iter() + .map(|server| { + let nsp = if &server.shard == current { + local_nsp.to_string() + } else { + ForeignServer::metadata_schema(&server.shard) + }; + (server.shard.as_str(), nsp) + }) + .collect::>() + } + + if self.shard != *PRIMARY_SHARD { + return Ok(()); + } + + info!(&self.logger, "Creating cross-shard views"); + let mut conn = self.get()?; + + conn.transaction(|conn| { + let query = format!( + "create schema if not exists {}", + ForeignServer::CROSS_SHARD_NSP + ); + conn.batch_execute(&query)?; + for (src_nsp, src_tables) in SHARDED_TABLES { + // Pairs of (shard, nsp) for all servers + let nsps = shard_nsp_pairs(&self.shard, src_nsp, servers); + for src_table in src_tables { + let create_view = catalog::create_cross_shard_view( + conn, + src_nsp, + src_table, + ForeignServer::CROSS_SHARD_NSP, + &nsps, + )?; + conn.batch_execute(&create_view)?; + } + } + Ok(()) + }) + } + /// Copy the data from key tables in the primary into our local schema /// so it can be used as a fallback when the primary goes down pub async fn mirror_primary_tables(&self) -> Result<(), StoreError> { From 36e3ad5af490db571e3beaac7e3555a9430dd0f5 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Wed, 12 Feb 2025 10:16:06 -0800 Subject: [PATCH 021/150] store: Fold BatchCopy into TableState With recent changes, BatchCopy has become unnecessary --- store/postgres/src/copy.rs | 117 +++++++++++++------------------------ 1 file changed, 39 insertions(+), 78 deletions(-) diff --git a/store/postgres/src/copy.rs b/store/postgres/src/copy.rs index 9b64390581b..45d0240de7f 100644 --- a/store/postgres/src/copy.rs +++ b/store/postgres/src/copy.rs @@ -208,17 +208,17 @@ impl CopyState { }) }) .collect::>()?; - tables.sort_by_key(|table| table.batch.dst.object.to_string()); + tables.sort_by_key(|table| table.dst.object.to_string()); let values = tables .iter() .map(|table| { ( - cts::entity_type.eq(table.batch.dst.object.as_str()), + cts::entity_type.eq(table.dst.object.as_str()), cts::dst.eq(dst.site.id), - cts::next_vid.eq(table.batch.next_vid()), - cts::target_vid.eq(table.batch.target_vid()), - cts::batch_size.eq(table.batch.batch_size()), + cts::next_vid.eq(table.batcher.next_vid()), + cts::target_vid.eq(table.batcher.target_vid()), + cts::batch_size.eq(table.batcher.batch_size() as i64), ) }) .collect::>(); @@ -294,51 +294,11 @@ pub(crate) fn source( /// so that we can copy rows from one to the other with very little /// transformation. See `CopyEntityBatchQuery` for the details of what /// exactly that means -pub(crate) struct BatchCopy { +struct TableState { src: Arc, dst: Arc
, - batcher: VidBatcher, -} - -impl BatchCopy { - pub fn new(batcher: VidBatcher, src: Arc
, dst: Arc
) -> Self { - Self { src, dst, batcher } - } - - /// Copy one batch of entities and update internal state so that the - /// next call to `run` will copy the next batch - pub fn run(&mut self, conn: &mut PgConnection) -> Result { - let (duration, _) = self.batcher.step(|start, end| { - rq::CopyEntityBatchQuery::new(self.dst.as_ref(), &self.src, start, end)? - .execute(conn)?; - Ok(()) - })?; - - Ok(duration) - } - - pub fn finished(&self) -> bool { - self.batcher.finished() - } - - /// The first `vid` that has not been copied yet - pub fn next_vid(&self) -> i64 { - self.batcher.next_vid() - } - - /// The last `vid` that should be copied - pub fn target_vid(&self) -> i64 { - self.batcher.target_vid() - } - - pub fn batch_size(&self) -> i64 { - self.batcher.batch_size() as i64 - } -} - -struct TableState { - batch: BatchCopy, dst_site: Arc, + batcher: VidBatcher, duration_ms: i64, } @@ -354,14 +314,16 @@ impl TableState { let vid_range = VidRange::for_copy(conn, &src, target_block)?; let batcher = VidBatcher::load(conn, &src_layout.site.namespace, src.as_ref(), vid_range)?; Ok(Self { - batch: BatchCopy::new(batcher, src, dst), + src, + dst, dst_site, + batcher, duration_ms: 0, }) } fn finished(&self) -> bool { - self.batch.finished() + self.batcher.finished() } fn load( @@ -427,11 +389,12 @@ impl TableState { VidRange::new(current_vid, target_vid), )? .with_batch_size(size as usize); - let batch = BatchCopy::new(batcher, src, dst); Ok(TableState { - batch, + src, + dst, dst_site: dst_layout.site.clone(), + batcher, duration_ms, }) } @@ -460,20 +423,20 @@ impl TableState { update( cts::table .filter(cts::dst.eq(self.dst_site.id)) - .filter(cts::entity_type.eq(self.batch.dst.object.as_str())) + .filter(cts::entity_type.eq(self.dst.object.as_str())) .filter(cts::duration_ms.eq(0)), ) .set(cts::started_at.eq(sql("now()"))) .execute(conn)?; let values = ( - cts::next_vid.eq(self.batch.next_vid()), - cts::batch_size.eq(self.batch.batch_size()), + cts::next_vid.eq(self.batcher.next_vid()), + cts::batch_size.eq(self.batcher.batch_size() as i64), cts::duration_ms.eq(self.duration_ms), ); update( cts::table .filter(cts::dst.eq(self.dst_site.id)) - .filter(cts::entity_type.eq(self.batch.dst.object.as_str())), + .filter(cts::entity_type.eq(self.dst.object.as_str())), ) .set(values) .execute(conn)?; @@ -486,7 +449,7 @@ impl TableState { update( cts::table .filter(cts::dst.eq(self.dst_site.id)) - .filter(cts::entity_type.eq(self.batch.dst.object.as_str())), + .filter(cts::entity_type.eq(self.dst.object.as_str())), ) .set(cts::finished_at.eq(sql("now()"))) .execute(conn)?; @@ -512,7 +475,11 @@ impl TableState { } fn copy_batch(&mut self, conn: &mut PgConnection) -> Result { - let duration = self.batch.run(conn)?; + let (duration, _) = self.batcher.step(|start, end| { + rq::CopyEntityBatchQuery::new(self.dst.as_ref(), &self.src, start, end)? + .execute(conn)?; + Ok(()) + })?; self.record_progress(conn, duration)?; @@ -539,12 +506,12 @@ impl<'a> CopyProgress<'a> { let target_vid: i64 = state .tables .iter() - .map(|table| table.batch.target_vid()) + .map(|table| table.batcher.target_vid()) .sum(); let current_vid = state .tables .iter() - .map(|table| table.batch.next_vid()) + .map(|table| table.batcher.next_vid()) .sum(); Self { logger, @@ -577,23 +544,23 @@ impl<'a> CopyProgress<'a> { } } - fn update(&mut self, batch: &BatchCopy) { + fn update(&mut self, entity_type: &EntityType, batcher: &VidBatcher) { if self.last_log.elapsed() > LOG_INTERVAL { info!( self.logger, "Copied {:.2}% of `{}` entities ({}/{} entity versions), {:.2}% of overall data", - Self::progress_pct(batch.next_vid(), batch.target_vid()), - batch.dst.object, - batch.next_vid(), - batch.target_vid(), - Self::progress_pct(self.current_vid + batch.next_vid(), self.target_vid) + Self::progress_pct(batcher.next_vid(), batcher.target_vid()), + entity_type, + batcher.next_vid(), + batcher.target_vid(), + Self::progress_pct(self.current_vid + batcher.next_vid(), self.target_vid) ); self.last_log = Instant::now(); } } - fn table_finished(&mut self, batch: &BatchCopy) { - self.current_vid += batch.next_vid(); + fn table_finished(&mut self, batcher: &VidBatcher) { + self.current_vid += batcher.next_vid(); } fn finished(&self) { @@ -728,9 +695,9 @@ impl Connection { if status == Status::Cancelled { return Ok(status); } - progress.update(&table.batch); + progress.update(&table.dst.object, &table.batcher); } - progress.table_finished(&table.batch); + progress.table_finished(&table.batcher); } // Create indexes for all the attributes that were postponed at the start of @@ -740,8 +707,8 @@ impl Connection { for table in state.tables.iter() { let arr = index_list.indexes_for_table( &self.dst.site.namespace, - &table.batch.src.name.to_string(), - &table.batch.dst, + &table.src.name.to_string(), + &table.dst, true, true, )?; @@ -756,18 +723,12 @@ impl Connection { // Here we need to skip those created in the first step for the old fields. for table in state.tables.iter() { let orig_colums = table - .batch .src .columns .iter() .map(|c| c.name.to_string()) .collect_vec(); - for sql in table - .batch - .dst - .create_postponed_indexes(orig_colums) - .into_iter() - { + for sql in table.dst.create_postponed_indexes(orig_colums).into_iter() { let query = sql_query(sql); query.execute(conn)?; } From b089776a55528fd746cb9c7b965092253adf3f26 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Wed, 12 Feb 2025 11:25:49 -0800 Subject: [PATCH 022/150] store: Count entities during copying We used to count entities after all the data had been copied, but that is a very slow operation for large subgraphs; that in turn can lead to bad side-effects like connection timeouts and the copy ultimately failing. We now keep track of the number of current entity versions that we copy with each batch and calculate the entity count incrementally --- store/postgres/src/copy.rs | 16 ++++++--- store/postgres/src/deployment_store.rs | 9 ++--- store/postgres/src/relational.rs | 3 ++ store/postgres/src/relational_queries.rs | 46 ++++++++++++++++++++++++ 4 files changed, 63 insertions(+), 11 deletions(-) diff --git a/store/postgres/src/copy.rs b/store/postgres/src/copy.rs index 45d0240de7f..f2f7e9f1d66 100644 --- a/store/postgres/src/copy.rs +++ b/store/postgres/src/copy.rs @@ -33,7 +33,7 @@ use graph::{ use itertools::Itertools; use crate::{ - advisory_lock, catalog, + advisory_lock, catalog, deployment, dynds::DataSourcesTable, primary::{DeploymentId, Site}, relational::index::IndexList, @@ -475,12 +475,18 @@ impl TableState { } fn copy_batch(&mut self, conn: &mut PgConnection) -> Result { - let (duration, _) = self.batcher.step(|start, end| { - rq::CopyEntityBatchQuery::new(self.dst.as_ref(), &self.src, start, end)? - .execute(conn)?; - Ok(()) + let (duration, count) = self.batcher.step(|start, end| { + let count = rq::CopyEntityBatchQuery::new(self.dst.as_ref(), &self.src, start, end)? + .count_current() + .get_result::(conn) + .optional()?; + Ok(count.unwrap_or(0) as i32) })?; + let count = count.unwrap_or(0); + + deployment::update_entity_count(conn, &self.dst_site, count)?; + self.record_progress(conn, duration)?; if self.finished() { diff --git a/store/postgres/src/deployment_store.rs b/store/postgres/src/deployment_store.rs index 92b1a27a7e5..e4878b82c9c 100644 --- a/store/postgres/src/deployment_store.rs +++ b/store/postgres/src/deployment_store.rs @@ -1570,13 +1570,10 @@ impl DeploymentStore { .number .checked_add(1) .expect("block numbers fit into an i32"); - dst.revert_block(conn, block_to_revert)?; - info!(logger, "Rewound subgraph to block {}", block.number; - "time_ms" => start.elapsed().as_millis()); + let (_, count) = dst.revert_block(conn, block_to_revert)?; + deployment::update_entity_count(conn, &dst.site, count)?; - let start = Instant::now(); - deployment::set_entity_count(conn, &dst.site, &dst.count_query)?; - info!(logger, "Counted the entities"; + info!(logger, "Rewound subgraph to block {}", block.number; "time_ms" => start.elapsed().as_millis()); deployment::set_history_blocks( diff --git a/store/postgres/src/relational.rs b/store/postgres/src/relational.rs index b12cf790697..74cb31f5b4d 100644 --- a/store/postgres/src/relational.rs +++ b/store/postgres/src/relational.rs @@ -1037,6 +1037,9 @@ impl Layout { /// numbers. After this operation, only entity versions inserted or /// updated at blocks with numbers strictly lower than `block` will /// remain + /// + /// The `i32` that is returned is the amount by which the entity count + /// for the subgraph needs to be adjusted pub fn revert_block( &self, conn: &mut PgConnection, diff --git a/store/postgres/src/relational_queries.rs b/store/postgres/src/relational_queries.rs index f4b55e89150..19f9400c470 100644 --- a/store/postgres/src/relational_queries.rs +++ b/store/postgres/src/relational_queries.rs @@ -4799,6 +4799,10 @@ impl<'a> CopyEntityBatchQuery<'a> { last_vid, }) } + + pub fn count_current(self) -> CountCurrentVersionsQuery<'a> { + CountCurrentVersionsQuery::new(self) + } } impl<'a> QueryFragment for CopyEntityBatchQuery<'a> { @@ -4810,6 +4814,8 @@ impl<'a> QueryFragment for CopyEntityBatchQuery<'a> { // Construct a query // insert into {dst}({columns}) // select {columns} from {src} + // where vid >= {first_vid} and vid <= {last_vid} + // returning {upper_inf(block_range)|true} out.push_sql("insert into "); out.push_sql(self.dst.qualified_name.as_str()); out.push_sql("("); @@ -4905,6 +4911,12 @@ impl<'a> QueryFragment for CopyEntityBatchQuery<'a> { out.push_bind_param::(&self.first_vid)?; out.push_sql(" and vid <= "); out.push_bind_param::(&self.last_vid)?; + out.push_sql("\n returning "); + if self.dst.immutable { + out.push_sql("true"); + } else { + out.push_sql(BLOCK_RANGE_CURRENT); + } Ok(()) } } @@ -4917,6 +4929,40 @@ impl<'a> QueryId for CopyEntityBatchQuery<'a> { impl<'a, Conn> RunQueryDsl for CopyEntityBatchQuery<'a> {} +#[derive(Debug, Clone)] +pub struct CountCurrentVersionsQuery<'a> { + copy: CopyEntityBatchQuery<'a>, +} + +impl<'a> CountCurrentVersionsQuery<'a> { + pub fn new(copy: CopyEntityBatchQuery<'a>) -> Self { + Self { copy } + } +} +impl<'a> QueryFragment for CountCurrentVersionsQuery<'a> { + fn walk_ast<'b>(&'b self, mut out: AstPass<'_, 'b, Pg>) -> QueryResult<()> { + // Generate a query + // with copy_cte as ( {copy} ) + // select count(*) from copy_cte where {block_range_current} + out.push_sql("with copy_cte(current) as ("); + self.copy.walk_ast(out.reborrow())?; + out.push_sql(")\nselect count(*) from copy_cte where current"); + Ok(()) + } +} + +impl<'a> QueryId for CountCurrentVersionsQuery<'a> { + type QueryId = (); + + const HAS_STATIC_QUERY_ID: bool = false; +} + +impl<'a> Query for CountCurrentVersionsQuery<'a> { + type SqlType = BigInt; +} + +impl<'a, Conn> RunQueryDsl for CountCurrentVersionsQuery<'a> {} + /// Helper struct for returning the id's touched by the RevertRemove and /// RevertExtend queries #[derive(QueryableByName, PartialEq, Eq, Hash)] From d36df68ff79a1ddad3317a43da3f36b6b19ceb60 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Wed, 12 Feb 2025 11:31:22 -0800 Subject: [PATCH 023/150] store: Clear the entity count when truncating a deployment We used to count entities, but we know the result is 0 because all the tables we are counting are empty --- store/postgres/src/deployment.rs | 11 +++-------- store/postgres/src/deployment_store.rs | 5 +++-- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/store/postgres/src/deployment.rs b/store/postgres/src/deployment.rs index 836048912b1..92181ac5a6c 100644 --- a/store/postgres/src/deployment.rs +++ b/store/postgres/src/deployment.rs @@ -1249,17 +1249,12 @@ pub fn update_entity_count( Ok(()) } -/// Set the deployment's entity count to whatever `full_count_query` produces -pub fn set_entity_count( - conn: &mut PgConnection, - site: &Site, - full_count_query: &str, -) -> Result<(), StoreError> { +/// Set the deployment's entity count back to `0` +pub fn clear_entity_count(conn: &mut PgConnection, site: &Site) -> Result<(), StoreError> { use subgraph_deployment as d; - let full_count_query = format!("({})", full_count_query); update(d::table.filter(d::id.eq(site.id))) - .set(d::entity_count.eq(sql(&full_count_query))) + .set(d::entity_count.eq(BigDecimal::from(0))) .execute(conn)?; Ok(()) } diff --git a/store/postgres/src/deployment_store.rs b/store/postgres/src/deployment_store.rs index e4878b82c9c..05f2b44a33e 100644 --- a/store/postgres/src/deployment_store.rs +++ b/store/postgres/src/deployment_store.rs @@ -1306,7 +1306,8 @@ impl DeploymentStore { let layout = self.layout(conn, site.clone())?; if truncate { - deployment::set_entity_count(conn, site.as_ref(), layout.count_query.as_str())?; + layout.truncate_tables(conn)?; + deployment::clear_entity_count(conn, site.as_ref())?; } else { let count = layout.revert_block(conn, block)?; deployment::update_entity_count(conn, site.as_ref(), count)?; @@ -1570,7 +1571,7 @@ impl DeploymentStore { .number .checked_add(1) .expect("block numbers fit into an i32"); - let (_, count) = dst.revert_block(conn, block_to_revert)?; + let count = dst.revert_block(conn, block_to_revert)?; deployment::update_entity_count(conn, &dst.site, count)?; info!(logger, "Rewound subgraph to block {}", block.number; From 9886aa181bc0a63fb457a4a44c7e0a6c885a5a1d Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Wed, 12 Feb 2025 11:33:18 -0800 Subject: [PATCH 024/150] store: Remove unused Layout.count_query --- store/postgres/src/relational.rs | 24 +----------------------- 1 file changed, 1 insertion(+), 23 deletions(-) diff --git a/store/postgres/src/relational.rs b/store/postgres/src/relational.rs index 74cb31f5b4d..8a8f47b6560 100644 --- a/store/postgres/src/relational.rs +++ b/store/postgres/src/relational.rs @@ -73,7 +73,7 @@ use graph::data::store::{Id, IdList, IdType, BYTES_SCALAR}; use graph::data::subgraph::schema::POI_TABLE; use graph::prelude::{ anyhow, info, BlockNumber, DeploymentHash, Entity, EntityOperation, Logger, - QueryExecutionError, StoreError, StoreEvent, ValueType, BLOCK_NUMBER_MAX, + QueryExecutionError, StoreError, StoreEvent, ValueType, }; use crate::block_range::{BoundSide, BLOCK_COLUMN, BLOCK_RANGE_COLUMN}; @@ -231,8 +231,6 @@ pub struct Layout { pub tables: HashMap>, /// The database schema for this subgraph pub catalog: Catalog, - /// The query to count all entities - pub count_query: String, /// How many blocks of history the subgraph should keep pub history_blocks: BlockNumber, @@ -290,25 +288,6 @@ impl Layout { )) } - let count_query = tables - .iter() - .map(|table| { - if table.immutable { - format!( - "select count(*) from \"{}\".\"{}\"", - &catalog.site.namespace, table.name - ) - } else { - format!( - "select count(*) from \"{}\".\"{}\" where block_range @> {}", - &catalog.site.namespace, table.name, BLOCK_NUMBER_MAX - ) - } - }) - .collect::>() - .join("\nunion all\n"); - let count_query = format!("select sum(e.count) from ({}) e", count_query); - let tables: HashMap<_, _> = tables .into_iter() .fold(HashMap::new(), |mut tables, table| { @@ -322,7 +301,6 @@ impl Layout { site, catalog, tables, - count_query, history_blocks: i32::MAX, input_schema: schema.cheap_clone(), rollups, From 5a1bb5e2c6d04a816e890243781e345a6971e50a Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Tue, 4 Mar 2025 18:38:47 +0100 Subject: [PATCH 025/150] store: Do not return a StoreEvent from Layout.truncate_tables --- store/postgres/src/relational.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/store/postgres/src/relational.rs b/store/postgres/src/relational.rs index 8a8f47b6560..c1e7bceef60 100644 --- a/store/postgres/src/relational.rs +++ b/store/postgres/src/relational.rs @@ -73,7 +73,7 @@ use graph::data::store::{Id, IdList, IdType, BYTES_SCALAR}; use graph::data::subgraph::schema::POI_TABLE; use graph::prelude::{ anyhow, info, BlockNumber, DeploymentHash, Entity, EntityOperation, Logger, - QueryExecutionError, StoreError, StoreEvent, ValueType, + QueryExecutionError, StoreError, ValueType, }; use crate::block_range::{BoundSide, BLOCK_COLUMN, BLOCK_RANGE_COLUMN}; @@ -1004,11 +1004,11 @@ impl Layout { Ok(count) } - pub fn truncate_tables(&self, conn: &mut PgConnection) -> Result { + pub fn truncate_tables(&self, conn: &mut PgConnection) -> Result<(), StoreError> { for table in self.tables.values() { sql_query(&format!("TRUNCATE TABLE {}", table.qualified_name)).execute(conn)?; } - Ok(StoreEvent::new(vec![])) + Ok(()) } /// Revert the block with number `block` and all blocks with higher From 59f889b77652d2dd4eaee7a39505de75c945b79b Mon Sep 17 00:00:00 2001 From: incrypto32 Date: Tue, 18 Feb 2025 18:03:42 +0400 Subject: [PATCH 026/150] graph: Add ability to order by fields other than timestamp for aggregate entities --- graph/src/schema/api.rs | 32 +++++++++++++++++++++++++------- graph/src/schema/input/mod.rs | 2 +- graphql/src/store/prefetch.rs | 4 ++-- 3 files changed, 28 insertions(+), 10 deletions(-) diff --git a/graph/src/schema/api.rs b/graph/src/schema/api.rs index db179b049d2..7fe29806a3f 100644 --- a/graph/src/schema/api.rs +++ b/graph/src/schema/api.rs @@ -485,6 +485,12 @@ fn add_types_for_aggregation_types( input_schema: &InputSchema, ) -> Result<(), APISchemaError> { for (name, agg_type) in input_schema.aggregation_types() { + // Combine regular fields and aggregate fields for ordering + let mut all_fields = agg_type.fields.to_vec(); + for agg in agg_type.aggregates.iter() { + all_fields.push(agg.as_agg_field()); + } + add_order_by_type(&mut api.document, name, &all_fields)?; add_aggregation_filter_type(api, name, agg_type)?; } Ok(()) @@ -678,13 +684,25 @@ impl FilterOps { s::Type::NamedType("OrderDirection".to_string()), ), ], - FilterOps::Aggregation => vec![input_value( - "interval", - "", - s::Type::NonNullType(Box::new(s::Type::NamedType( - "Aggregation_interval".to_string(), - ))), - )], + FilterOps::Aggregation => vec![ + input_value( + "interval", + "", + s::Type::NonNullType(Box::new(s::Type::NamedType( + "Aggregation_interval".to_string(), + ))), + ), + input_value( + "orderBy", + "", + s::Type::NamedType(format!("{}_orderBy", type_name)), + ), + input_value( + "orderDirection", + "", + s::Type::NamedType("OrderDirection".to_string()), + ), + ], }; let mut args = vec![skip, first]; diff --git a/graph/src/schema/input/mod.rs b/graph/src/schema/input/mod.rs index 5ff8ddcda2f..634930c5731 100644 --- a/graph/src/schema/input/mod.rs +++ b/graph/src/schema/input/mod.rs @@ -824,7 +824,7 @@ impl Aggregate { /// The field needed for the finalised aggregation for hourly/daily /// values - fn as_agg_field(&self) -> Field { + pub fn as_agg_field(&self) -> Field { Field { name: self.name.clone(), field_type: self.field_type.clone(), diff --git a/graphql/src/store/prefetch.rs b/graphql/src/store/prefetch.rs index de97b243227..33f0b67452b 100644 --- a/graphql/src/store/prefetch.rs +++ b/graphql/src/store/prefetch.rs @@ -713,8 +713,8 @@ impl<'a> Loader<'a> { // that causes unnecessary work in the database query.order = EntityOrder::Unordered; } - // Aggregations are always ordered by (timestamp, id) - if child_type.is_aggregation() { + // Apply default timestamp ordering for aggregations if no custom order is specified + if child_type.is_aggregation() && matches!(query.order, EntityOrder::Default) { let ts = child_type.field(kw::TIMESTAMP).unwrap(); query.order = EntityOrder::Descending(ts.name.to_string(), ts.value_type); } From dfdfba8677e2e71084a71183dbfc3f0a752f2cbe Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 12 Dec 2024 21:32:13 +0000 Subject: [PATCH 027/150] build(deps): bump nanoid from 3.3.6 to 3.3.8 in /tests/runner-tests Bumps [nanoid](https://github.com/ai/nanoid) from 3.3.6 to 3.3.8. - [Release notes](https://github.com/ai/nanoid/releases) - [Changelog](https://github.com/ai/nanoid/blob/main/CHANGELOG.md) - [Commits](https://github.com/ai/nanoid/compare/3.3.6...3.3.8) --- updated-dependencies: - dependency-name: nanoid dependency-type: indirect ... Signed-off-by: dependabot[bot] --- tests/runner-tests/yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/runner-tests/yarn.lock b/tests/runner-tests/yarn.lock index 9f3bdae834d..def5047f0b2 100644 --- a/tests/runner-tests/yarn.lock +++ b/tests/runner-tests/yarn.lock @@ -2909,9 +2909,9 @@ mustache@^4.2.0: integrity sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ== nanoid@^3.0.2, nanoid@^3.1.20, nanoid@^3.1.23: - version "3.3.6" - resolved "https://registry.npmjs.org/nanoid/-/nanoid-3.3.6.tgz" - integrity sha512-BGcqMMJuToF7i1rt+2PWSNVnWIkGCU78jBG3RxO/bZlnZPK2Cmi2QaffxGO/2RvWi9sL+FAiRiXMgsyxQ1DIDA== + version "3.3.8" + resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.3.8.tgz#b1be3030bee36aaff18bacb375e5cce521684baf" + integrity sha512-WNLf5Sd8oZxOm+TzppcYk8gVOgP+l58xNy58D0nbUnOxOWRWvlcCV4kUF7ltmI6PsrLl/BgKEyS4mqsGChFN0w== native-abort-controller@^1.0.3, native-abort-controller@^1.0.4: version "1.0.4" From 9ed8f6e0b1d78d5de9284f44c5fca0b6c694861f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Oct 2024 22:51:16 +0000 Subject: [PATCH 028/150] build(deps): bump async-stream from 0.3.5 to 0.3.6 Bumps [async-stream](https://github.com/tokio-rs/async-stream) from 0.3.5 to 0.3.6. - [Release notes](https://github.com/tokio-rs/async-stream/releases) - [Commits](https://github.com/tokio-rs/async-stream/compare/v0.3.5...v0.3.6) --- updated-dependencies: - dependency-name: async-stream dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Cargo.lock | 8 ++++---- tests/Cargo.toml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0d20d8900d3..e65d980ed28 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -287,9 +287,9 @@ dependencies = [ [[package]] name = "async-stream" -version = "0.3.5" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" dependencies = [ "async-stream-impl", "futures-core", @@ -298,9 +298,9 @@ dependencies = [ [[package]] name = "async-stream-impl" -version = "0.3.5" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", diff --git a/tests/Cargo.toml b/tests/Cargo.toml index b79702ceb7f..ad4a4a9c785 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -6,7 +6,7 @@ edition.workspace = true [dependencies] anyhow = "1.0" assert-json-diff = "2.0.2" -async-stream = "0.3.5" +async-stream = "0.3.6" graph = { path = "../graph" } graph-chain-ethereum = { path = "../chain/ethereum" } graph-chain-substreams= {path = "../chain/substreams"} From f0380340f5b1d57d33882a618908fd557f9c3466 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Oct 2024 18:17:58 +0000 Subject: [PATCH 029/150] build(deps): bump secp256k1 from 4.0.3 to 4.0.4 in /tests/runner-tests Bumps [secp256k1](https://github.com/cryptocoinjs/secp256k1-node) from 4.0.3 to 4.0.4. - [Release notes](https://github.com/cryptocoinjs/secp256k1-node/releases) - [Commits](https://github.com/cryptocoinjs/secp256k1-node/compare/v4.0.3...v4.0.4) --- updated-dependencies: - dependency-name: secp256k1 dependency-type: indirect ... Signed-off-by: dependabot[bot] --- tests/runner-tests/yarn.lock | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/tests/runner-tests/yarn.lock b/tests/runner-tests/yarn.lock index def5047f0b2..2f9f1287bec 100644 --- a/tests/runner-tests/yarn.lock +++ b/tests/runner-tests/yarn.lock @@ -1605,7 +1605,7 @@ electron-fetch@^1.7.2: dependencies: encoding "^0.1.13" -elliptic@6.5.4, elliptic@^6.5.4: +elliptic@6.5.4: version "6.5.4" resolved "https://registry.npmjs.org/elliptic/-/elliptic-6.5.4.tgz" integrity sha512-iLhC6ULemrljPZb+QutR5TQGB+pdW6KGD5RSegS+8sorOZT+rdQFbsQFJgvN3eRqNALqJer4oQ16YvJHlU8hzQ== @@ -1618,6 +1618,19 @@ elliptic@6.5.4, elliptic@^6.5.4: minimalistic-assert "^1.0.1" minimalistic-crypto-utils "^1.0.1" +elliptic@^6.5.7: + version "6.5.7" + resolved "https://registry.yarnpkg.com/elliptic/-/elliptic-6.5.7.tgz#8ec4da2cb2939926a1b9a73619d768207e647c8b" + integrity sha512-ESVCtTwiA+XhY3wyh24QqRGBoP3rEdDUl3EDUUo9tft074fi19IrdpH7hLCMMP3CIj7jb3W96rn8lt/BqIlt5Q== + dependencies: + bn.js "^4.11.9" + brorand "^1.1.0" + hash.js "^1.0.0" + hmac-drbg "^1.0.1" + inherits "^2.0.4" + minimalistic-assert "^1.0.1" + minimalistic-crypto-utils "^1.0.1" + emoji-regex@^8.0.0: version "8.0.0" resolved "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz" @@ -2938,6 +2951,11 @@ node-addon-api@^2.0.0: resolved "https://registry.npmjs.org/node-addon-api/-/node-addon-api-2.0.2.tgz" integrity sha512-Ntyt4AIXyaLIuMHF6IOoTakB3K+RWxwtsHNRxllEoA6vPwP9o4866g6YWDLUdnucilZhmkxiHwHr11gAENw+QA== +node-addon-api@^5.0.0: + version "5.1.0" + resolved "https://registry.yarnpkg.com/node-addon-api/-/node-addon-api-5.1.0.tgz#49da1ca055e109a23d537e9de43c09cca21eb762" + integrity sha512-eh0GgfEkpnoWDq+VY8OyvYhFEzBk6jIYbRKdIlyTiAXIVJ8PyBaKb0rp7oDtoddbdoHWhq8wwr+XZ81F1rpNdA== + node-fetch@^2.6.8: version "2.6.9" resolved "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.9.tgz" @@ -3394,12 +3412,12 @@ scrypt-js@^3.0.0: integrity sha512-cdwTTnqPu0Hyvf5in5asVdZocVDTNRmR7XEcJuIzMjJeSHybHl7vpB66AzwTaIg6CLSbtjcxc8fqcySfnTkccA== secp256k1@^4.0.1: - version "4.0.3" - resolved "https://registry.npmjs.org/secp256k1/-/secp256k1-4.0.3.tgz" - integrity sha512-NLZVf+ROMxwtEj3Xa562qgv2BK5e2WNmXPiOdVIPLgs6lyTzMvBq0aWTYMI5XCP9jZMVKOcqZLw/Wc4vDkuxhA== + version "4.0.4" + resolved "https://registry.yarnpkg.com/secp256k1/-/secp256k1-4.0.4.tgz#58f0bfe1830fe777d9ca1ffc7574962a8189f8ab" + integrity sha512-6JfvwvjUOn8F/jUoBY2Q1v5WY5XS+rj8qSe0v8Y4ezH4InLgTEeOOPQsRll9OV429Pvo6BCHGavIyJfr3TAhsw== dependencies: - elliptic "^6.5.4" - node-addon-api "^2.0.0" + elliptic "^6.5.7" + node-addon-api "^5.0.0" node-gyp-build "^4.2.0" semver@7.3.5: From 39312237285ef72fc841afedab315a5f55e6a137 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Oct 2024 21:20:06 +0000 Subject: [PATCH 030/150] build(deps): bump protobuf-parse from 3.5.0 to 3.7.1 Bumps [protobuf-parse](https://github.com/stepancheg/rust-protobuf) from 3.5.0 to 3.7.1. - [Changelog](https://github.com/stepancheg/rust-protobuf/blob/master/CHANGELOG.md) - [Commits](https://github.com/stepancheg/rust-protobuf/compare/v3.5.0...v3.7.1) --- updated-dependencies: - dependency-name: protobuf-parse dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Cargo.lock | 16 ++++++++-------- chain/common/Cargo.toml | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e65d980ed28..dd943c479b4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1886,7 +1886,7 @@ version = "0.36.0" dependencies = [ "anyhow", "heck 0.5.0", - "protobuf 3.5.0", + "protobuf 3.7.1", "protobuf-parse", ] @@ -3845,9 +3845,9 @@ checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94" [[package]] name = "protobuf" -version = "3.5.0" +version = "3.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df67496db1a89596beaced1579212e9b7c53c22dca1d9745de00ead76573d514" +checksum = "a3a7c64d9bf75b1b8d981124c14c179074e8caa7dfe7b6a12e6222ddcd0c8f72" dependencies = [ "once_cell", "protobuf-support", @@ -3856,14 +3856,14 @@ dependencies = [ [[package]] name = "protobuf-parse" -version = "3.5.0" +version = "3.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a16027030d4ec33e423385f73bb559821827e9ec18c50e7874e4d6de5a4e96f" +checksum = "322330e133eab455718444b4e033ebfac7c6528972c784fcde28d2cc783c6257" dependencies = [ "anyhow", "indexmap 2.2.6", "log", - "protobuf 3.5.0", + "protobuf 3.7.1", "protobuf-support", "tempfile", "thiserror", @@ -3872,9 +3872,9 @@ dependencies = [ [[package]] name = "protobuf-support" -version = "3.5.0" +version = "3.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70e2d30ab1878b2e72d1e2fc23ff5517799c9929e2cf81a8516f9f4dcf2b9cf3" +checksum = "b088fd20b938a875ea00843b6faf48579462630015c3788d397ad6a786663252" dependencies = [ "thiserror", ] diff --git a/chain/common/Cargo.toml b/chain/common/Cargo.toml index 2715c18a845..6c1cfd9dc03 100644 --- a/chain/common/Cargo.toml +++ b/chain/common/Cargo.toml @@ -7,6 +7,6 @@ edition.workspace = true [dependencies] protobuf = "3.0.2" -protobuf-parse = "3.5.0" +protobuf-parse = "3.7.1" anyhow = "1" heck = "0.5" From 1aa4398f043a19e0e9716179db480a2414c265a3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 13 Nov 2024 09:32:30 +0000 Subject: [PATCH 031/150] build(deps): bump syn from 2.0.69 to 2.0.87 Bumps [syn](https://github.com/dtolnay/syn) from 2.0.69 to 2.0.87. - [Release notes](https://github.com/dtolnay/syn/releases) - [Commits](https://github.com/dtolnay/syn/compare/2.0.69...2.0.87) --- updated-dependencies: - dependency-name: syn dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Cargo.lock | 74 +++++++++++++++++++++++++++--------------------------- Cargo.toml | 2 +- 2 files changed, 38 insertions(+), 38 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index dd943c479b4..d5d1bed6ac8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -246,7 +246,7 @@ dependencies = [ "proc-macro2", "quote", "strum", - "syn 2.0.69", + "syn 2.0.87", "thiserror", ] @@ -282,7 +282,7 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.87", ] [[package]] @@ -304,7 +304,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.87", ] [[package]] @@ -315,7 +315,7 @@ checksum = "6e0c28dcc82d7c8ead5cb13beb15405b57b8546e93215673ff8ca0349a028107" dependencies = [ "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.87", ] [[package]] @@ -731,7 +731,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.87", ] [[package]] @@ -1078,7 +1078,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.69", + "syn 2.0.87", ] [[package]] @@ -1089,7 +1089,7 @@ checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ "darling_core", "quote", - "syn 2.0.69", + "syn 2.0.87", ] [[package]] @@ -1182,7 +1182,7 @@ dependencies = [ "proc-macro2", "quote", "rustc_version", - "syn 2.0.69", + "syn 2.0.87", ] [[package]] @@ -1215,7 +1215,7 @@ dependencies = [ "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.87", ] [[package]] @@ -1237,7 +1237,7 @@ dependencies = [ "dsl_auto_type", "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.87", ] [[package]] @@ -1257,7 +1257,7 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "209c735641a413bc68c4923a9d6ad4bcb3ca306b794edaa7eb0b3228a99ffb25" dependencies = [ - "syn 2.0.69", + "syn 2.0.87", ] [[package]] @@ -1355,7 +1355,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.87", ] [[package]] @@ -1647,7 +1647,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.87", ] [[package]] @@ -1765,7 +1765,7 @@ dependencies = [ "log", "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.87", "time", ] @@ -2041,7 +2041,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.87", ] [[package]] @@ -2188,7 +2188,7 @@ dependencies = [ "proc-macro-utils", "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.87", ] [[package]] @@ -3339,7 +3339,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.87", ] [[package]] @@ -3473,7 +3473,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.87", ] [[package]] @@ -3532,7 +3532,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.87", ] [[package]] @@ -3657,7 +3657,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f12335488a2f3b0a83b14edad48dca9879ce89b2edd10e80237e4e852dd645e" dependencies = [ "proc-macro2", - "syn 2.0.69", + "syn 2.0.87", ] [[package]] @@ -3789,7 +3789,7 @@ dependencies = [ "prost 0.12.6", "prost-types 0.12.6", "regex", - "syn 2.0.69", + "syn 2.0.87", "tempfile", ] @@ -3816,7 +3816,7 @@ dependencies = [ "itertools 0.10.5", "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.87", ] [[package]] @@ -4401,7 +4401,7 @@ checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222" dependencies = [ "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.87", ] [[package]] @@ -4490,7 +4490,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.87", ] [[package]] @@ -4682,7 +4682,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.87", ] [[package]] @@ -4813,7 +4813,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.69", + "syn 2.0.87", ] [[package]] @@ -4911,9 +4911,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.69" +version = "2.0.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "201fcda3845c23e8212cd466bfebf0bd20694490fc0356ae8e428e0824a915a6" +checksum = "25aa4ce346d03a6dcd68dd8b4010bcb74e54e62c90c573f394c46eae99aba32d" dependencies = [ "proc-macro2", "quote", @@ -5047,7 +5047,7 @@ checksum = "46c3384250002a6d5af4d114f2845d37b57521033f30d5c3f46c4d70e1197533" dependencies = [ "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.87", ] [[package]] @@ -5161,7 +5161,7 @@ checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.87", ] [[package]] @@ -5407,7 +5407,7 @@ dependencies = [ "proc-macro2", "prost-build 0.12.6", "quote", - "syn 2.0.69", + "syn 2.0.87", ] [[package]] @@ -5520,7 +5520,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.87", ] [[package]] @@ -5794,7 +5794,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.87", "wasm-bindgen-shared", ] @@ -5828,7 +5828,7 @@ checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.87", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -5974,7 +5974,7 @@ dependencies = [ "anyhow", "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.87", "wasmtime-component-util", "wasmtime-wit-bindgen", "wit-parser", @@ -6162,7 +6162,7 @@ checksum = "f50f51f8d79bfd2aa8e9d9a0ae7c2d02b45fe412e62ff1b87c0c81b07c738231" dependencies = [ "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.87", ] [[package]] @@ -6571,7 +6571,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.69", + "syn 2.0.87", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 4cf78a3eea0..c054e146d5d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -68,7 +68,7 @@ serde_yaml = "0.9.21" slog = { version = "2.7.0", features = ["release_max_level_trace", "max_level_trace"] } sqlparser = "0.46.0" strum = { version = "0.26", features = ["derive"] } -syn = { version = "2.0.66", features = ["full"] } +syn = { version = "2.0.87", features = ["full"] } test-store = { path = "./store/test-store" } thiserror = "1.0.25" tokio = { version = "1.38.0", features = ["full"] } From a20f2950817f6f39c0056e1c8cf42030a37e034e Mon Sep 17 00:00:00 2001 From: shiyasmohd Date: Mon, 3 Feb 2025 07:06:56 +0530 Subject: [PATCH 032/150] store: pass list of indexes when pruning a subgraph --- store/postgres/src/relational/index.rs | 16 ++++++++++++---- store/postgres/src/relational/prune.rs | 4 ++-- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/store/postgres/src/relational/index.rs b/store/postgres/src/relational/index.rs index 6013a5d9e68..4cc354b8e89 100644 --- a/store/postgres/src/relational/index.rs +++ b/store/postgres/src/relational/index.rs @@ -734,6 +734,16 @@ pub struct IndexList { pub(crate) indexes: HashMap>, } +pub fn load_indexes_from_table( + conn: &mut PgConnection, + table: &Arc
, + schema_name: &str, +) -> Result, StoreError> { + let table_name = table.name.as_str(); + let indexes = catalog::indexes_for_table(conn, schema_name, table_name)?; + Ok(indexes.into_iter().map(CreateIndex::parse).collect()) +} + impl IndexList { pub fn load( conn: &mut PgConnection, @@ -746,10 +756,8 @@ impl IndexList { let schema_name = site.namespace.clone(); let layout = store.layout(conn, site)?; for (_, table) in &layout.tables { - let table_name = table.name.as_str(); - let indexes = catalog::indexes_for_table(conn, schema_name.as_str(), table_name)?; - let collect: Vec = indexes.into_iter().map(CreateIndex::parse).collect(); - list.indexes.insert(table_name.to_string(), collect); + let indexes = load_indexes_from_table(conn, table, schema_name.as_str())?; + list.indexes.insert(table.name.to_string(), indexes); } Ok(list) } diff --git a/store/postgres/src/relational/prune.rs b/store/postgres/src/relational/prune.rs index 62632549397..f3aba1e4301 100644 --- a/store/postgres/src/relational/prune.rs +++ b/store/postgres/src/relational/prune.rs @@ -1,4 +1,4 @@ -use std::{fmt::Write, sync::Arc}; +use std::{collections::HashMap, fmt::Write, sync::Arc, time::Instant}; use diesel::{ connection::SimpleConnection, @@ -58,7 +58,7 @@ impl TablePair { } else { // In case of pruning we don't do delayed creation of indexes, // as the asumption is that there is not that much data inserted. - dst.as_ddl(schema, catalog, None, &mut query)?; + dst.as_ddl(schema, catalog, Some(&list), &mut query)?; } conn.batch_execute(&query)?; From 086104f9239256403ac216f74e9e81aff2e6113f Mon Sep 17 00:00:00 2001 From: shiyasmohd Date: Tue, 11 Feb 2025 09:54:53 +0530 Subject: [PATCH 033/150] store: fix namespace reference when loading indexes from table --- store/postgres/src/relational/prune.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/store/postgres/src/relational/prune.rs b/store/postgres/src/relational/prune.rs index f3aba1e4301..10d0c7c93f9 100644 --- a/store/postgres/src/relational/prune.rs +++ b/store/postgres/src/relational/prune.rs @@ -56,6 +56,12 @@ impl TablePair { if catalog::table_exists(conn, dst_nsp.as_str(), &dst.name)? { writeln!(query, "truncate table {};", dst.qualified_name)?; } else { + let mut list = IndexList { + indexes: HashMap::new(), + }; + let indexes = load_indexes_from_table(conn, &src, src_nsp.as_str())?; + list.indexes.insert(src.name.to_string(), indexes); + // In case of pruning we don't do delayed creation of indexes, // as the asumption is that there is not that much data inserted. dst.as_ddl(schema, catalog, Some(&list), &mut query)?; From 0b1c10975307f75915bc0be346425868c014906b Mon Sep 17 00:00:00 2001 From: shiyasmohd Date: Thu, 20 Feb 2025 20:08:28 +0530 Subject: [PATCH 034/150] store: update namespace handling in table creation and index loading --- store/postgres/src/relational.rs | 2 +- store/postgres/src/relational/ddl.rs | 8 +------- store/postgres/src/relational/index.rs | 2 +- store/postgres/src/relational/prune.rs | 12 +++++++++--- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/store/postgres/src/relational.rs b/store/postgres/src/relational.rs index c1e7bceef60..8fa896c22d3 100644 --- a/store/postgres/src/relational.rs +++ b/store/postgres/src/relational.rs @@ -1680,7 +1680,7 @@ impl Table { pub fn new_like(&self, namespace: &Namespace, name: &SqlName) -> Arc
{ let other = Table { object: self.object.clone(), - nsp: self.nsp.clone(), + nsp: namespace.clone(), name: name.clone(), qualified_name: SqlName::qualified_name(namespace, name), columns: self.columns.clone(), diff --git a/store/postgres/src/relational/ddl.rs b/store/postgres/src/relational/ddl.rs index 40a02d6051e..55e116272d1 100644 --- a/store/postgres/src/relational/ddl.rs +++ b/store/postgres/src/relational/ddl.rs @@ -403,13 +403,7 @@ impl Table { if index_def.is_some() && ENV_VARS.postpone_attribute_index_creation { let arr = index_def .unwrap() - .indexes_for_table( - &catalog.site.namespace, - &self.name.to_string(), - &self, - false, - false, - ) + .indexes_for_table(&self.nsp, &self.name.to_string(), &self, false, false) .map_err(|_| fmt::Error)?; for (_, sql) in arr { writeln!(out, "{};", sql).expect("properly formated index statements") diff --git a/store/postgres/src/relational/index.rs b/store/postgres/src/relational/index.rs index 4cc354b8e89..4f72e773ee6 100644 --- a/store/postgres/src/relational/index.rs +++ b/store/postgres/src/relational/index.rs @@ -440,7 +440,7 @@ impl CreateIndex { } } - fn with_nsp(&self, nsp2: String) -> Result { + pub fn with_nsp(&self, nsp2: String) -> Result { let s = self.clone(); match s { CreateIndex::Unknown { defn: _ } => Err(anyhow!("Failed to parse the index")), diff --git a/store/postgres/src/relational/prune.rs b/store/postgres/src/relational/prune.rs index 10d0c7c93f9..5c3035ce172 100644 --- a/store/postgres/src/relational/prune.rs +++ b/store/postgres/src/relational/prune.rs @@ -1,4 +1,4 @@ -use std::{collections::HashMap, fmt::Write, sync::Arc, time::Instant}; +use std::{collections::HashMap, fmt::Write, sync::Arc}; use diesel::{ connection::SimpleConnection, @@ -23,7 +23,10 @@ use crate::{ vid_batcher::{VidBatcher, VidRange}, }; -use super::{Catalog, Layout, Namespace}; +use super::{ + index::{load_indexes_from_table, CreateIndex, IndexList}, + Catalog, Layout, Namespace, +}; /// Utility to copy relevant data out of a source table and into a new /// destination table and replace the source table with the destination @@ -59,7 +62,10 @@ impl TablePair { let mut list = IndexList { indexes: HashMap::new(), }; - let indexes = load_indexes_from_table(conn, &src, src_nsp.as_str())?; + let indexes = load_indexes_from_table(conn, &src, src_nsp.as_str())? + .into_iter() + .map(|index| index.with_nsp(dst_nsp.to_string())) + .collect::, _>>()?; list.indexes.insert(src.name.to_string(), indexes); // In case of pruning we don't do delayed creation of indexes, From 0d6e1ca3fc8c61e2c70a57e47853301e3cba3dee Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Feb 2025 21:16:39 +0000 Subject: [PATCH 035/150] build(deps): bump hyper from 1.4.0 to 1.6.0 Bumps [hyper](https://github.com/hyperium/hyper) from 1.4.0 to 1.6.0. - [Release notes](https://github.com/hyperium/hyper/releases) - [Changelog](https://github.com/hyperium/hyper/blob/master/CHANGELOG.md) - [Commits](https://github.com/hyperium/hyper/compare/v1.4.0...v1.6.0) --- updated-dependencies: - dependency-name: hyper dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Cargo.lock | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d5d1bed6ac8..4a5dfe3e050 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -378,7 +378,7 @@ dependencies = [ "http 1.1.0", "http-body 1.0.0", "http-body-util", - "hyper 1.4.0", + "hyper 1.6.0", "hyper-util", "itoa", "matchit", @@ -1815,7 +1815,7 @@ dependencies = [ "http 1.1.0", "http-body-util", "humantime", - "hyper 1.4.0", + "hyper 1.6.0", "hyper-util", "isatty", "itertools 0.13.0", @@ -2527,9 +2527,9 @@ dependencies = [ [[package]] name = "hyper" -version = "1.4.0" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4fe55fb7a772d59a5ff1dfbff4fe0258d19b89fec4b233e75d35d5d2316badc" +checksum = "cc2b571658e38e0c01b1fdca3bbbe93c00d3d71693ff2770043f8c29bc7d6f80" dependencies = [ "bytes", "futures-channel", @@ -2554,7 +2554,7 @@ checksum = "5ee4be2c948921a1a5320b629c4193916ed787a7f7f293fd3f7f5a6c9de74155" dependencies = [ "futures-util", "http 1.1.0", - "hyper 1.4.0", + "hyper 1.6.0", "hyper-util", "rustls 0.23.10", "rustls-native-certs", @@ -2584,7 +2584,7 @@ checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" dependencies = [ "bytes", "http-body-util", - "hyper 1.4.0", + "hyper 1.6.0", "hyper-util", "native-tls", "tokio", @@ -2603,7 +2603,7 @@ dependencies = [ "futures-util", "http 1.1.0", "http-body 1.0.0", - "hyper 1.4.0", + "hyper 1.6.0", "pin-project-lite", "socket2", "tokio", @@ -3286,7 +3286,7 @@ dependencies = [ "chrono", "futures 0.3.30", "humantime", - "hyper 1.4.0", + "hyper 1.6.0", "itertools 0.13.0", "parking_lot", "percent-encoding", @@ -4114,7 +4114,7 @@ dependencies = [ "http 1.1.0", "http-body 1.0.0", "http-body-util", - "hyper 1.4.0", + "hyper 1.6.0", "hyper-rustls", "hyper-tls", "hyper-util", @@ -6505,7 +6505,7 @@ dependencies = [ "futures 0.3.30", "http 1.1.0", "http-body-util", - "hyper 1.4.0", + "hyper 1.6.0", "hyper-util", "log", "once_cell", From 00999bddbeb2c1a74944bb1deba3ebc2dec1230d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 Feb 2025 21:22:34 +0000 Subject: [PATCH 036/150] build(deps): bump async-graphql from 7.0.11 to 7.0.15 Bumps [async-graphql](https://github.com/async-graphql/async-graphql) from 7.0.11 to 7.0.15. - [Changelog](https://github.com/async-graphql/async-graphql/blob/master/CHANGELOG.md) - [Commits](https://github.com/async-graphql/async-graphql/commits) --- updated-dependencies: - dependency-name: async-graphql dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Cargo.lock | 17 ++++++++--------- Cargo.toml | 2 +- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4a5dfe3e050..7f3ffa09be5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -181,9 +181,9 @@ dependencies = [ [[package]] name = "async-graphql" -version = "7.0.11" +version = "7.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ba6d24703c5adc5ba9116901b92ee4e4c0643c01a56c4fd303f3818638d7449" +checksum = "bfff2b17d272a5e3e201feda444e2c24b011fa722951268d1bd8b9b5bc6dc449" dependencies = [ "async-graphql-derive", "async-graphql-parser", @@ -203,7 +203,6 @@ dependencies = [ "mime", "multer", "num-traits", - "once_cell", "pin-project-lite", "regex", "serde", @@ -235,9 +234,9 @@ dependencies = [ [[package]] name = "async-graphql-derive" -version = "7.0.11" +version = "7.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a94c2d176893486bd37cd1b6defadd999f7357bf5804e92f510c08bcf16c538f" +checksum = "d8e5d0c6697def2f79ccbd972fb106b633173a6066e430b480e1ff9376a7561a" dependencies = [ "Inflector", "async-graphql-parser", @@ -252,9 +251,9 @@ dependencies = [ [[package]] name = "async-graphql-parser" -version = "7.0.11" +version = "7.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79272bdbf26af97866e149f05b2b546edb5c00e51b5f916289931ed233e208ad" +checksum = "8531ee6d292c26df31c18c565ff22371e7bdfffe7f5e62b69537db0b8fd554dc" dependencies = [ "async-graphql-value", "pest", @@ -264,9 +263,9 @@ dependencies = [ [[package]] name = "async-graphql-value" -version = "7.0.11" +version = "7.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef5ec94176a12a8cbe985cd73f2e54dc9c702c88c766bdef12f1f3a67cedbee1" +checksum = "741110dda927420a28fbc1c310543d3416f789a6ba96859c2c265843a0a96887" dependencies = [ "bytes", "indexmap 2.2.6", diff --git a/Cargo.toml b/Cargo.toml index c054e146d5d..c19d6939f01 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,7 +36,7 @@ license = "MIT OR Apache-2.0" [workspace.dependencies] anyhow = "1.0" -async-graphql = { version = "7.0.11", features = ["chrono", "uuid"] } +async-graphql = { version = "7.0.15", features = ["chrono", "uuid"] } async-graphql-axum = "7.0.11" axum = "0.7.5" bs58 = "0.5.1" From 33532bfe41dfb9ac153c99ae549c5b812479c492 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Wed, 5 Mar 2025 10:03:39 +0100 Subject: [PATCH 037/150] all: Address various typos --- NEWS.md | 6 +++--- README.md | 2 +- chain/ethereum/proto/ethereum.proto | 16 ++++++++-------- node/src/manager/commands/drop.rs | 4 ++-- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/NEWS.md b/NEWS.md index a8843422fde..719d2f12e49 100644 --- a/NEWS.md +++ b/NEWS.md @@ -446,8 +446,8 @@ Not Relevant @@ -1155,7 +1155,7 @@ storage](./docs/config.md) and spread subgraph deployments, and the load coming from indexing and querying them across multiple independent Postgres databases. -**This feature is considered experimenatal. We encourage users to try this +**This feature is considered experimental. We encourage users to try this out in a test environment, but do not recommend it yet for production use** In particular, the details of how sharding is configured may change in backwards-incompatible ways in the future. diff --git a/README.md b/README.md index ff31fdad758..b856653fd95 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ To build and run this project you need to have the following installed on your s - Note that `rustfmt`, which is part of the default Rust installation, is a build-time requirement. - PostgreSQL – [PostgreSQL Downloads](https://www.postgresql.org/download/) - IPFS – [Installing IPFS](https://docs.ipfs.io/install/) -- Profobuf Compiler - [Installing Protobuf](https://grpc.io/docs/protoc-installation/) +- Protobuf Compiler - [Installing Protobuf](https://grpc.io/docs/protoc-installation/) For Ethereum network data, you can either run your own Ethereum node or use an Ethereum node provider of your choice. diff --git a/chain/ethereum/proto/ethereum.proto b/chain/ethereum/proto/ethereum.proto index 3c9f7378c7d..42adbd0ffa6 100644 --- a/chain/ethereum/proto/ethereum.proto +++ b/chain/ethereum/proto/ethereum.proto @@ -13,7 +13,7 @@ message Block { uint64 size = 4; BlockHeader header = 5; - // Uncles represents block produced with a valid solution but were not actually choosen + // Uncles represents block produced with a valid solution but were not actually chosen // as the canonical block for the given height so they are mostly "forked" blocks. // // If the Block has been produced using the Proof of Stake consensus algorithm, this @@ -285,7 +285,7 @@ message Log { bytes data = 3; // Index is the index of the log relative to the transaction. This index - // is always populated regardless of the state revertion of the the call + // is always populated regardless of the state reversion of the call // that emitted this log. uint32 index = 4; @@ -294,7 +294,7 @@ message Log { // An **important** notice is that this field will be 0 when the call // that emitted the log has been reverted by the chain. // - // Currently, there is two locations where a Log can be obtained: + // Currently, there are two locations where a Log can be obtained: // - block.transaction_traces[].receipt.logs[] // - block.transaction_traces[].calls[].logs[] // @@ -341,7 +341,7 @@ message Call { reserved 29; // In Ethereum, a call can be either: - // - Successfull, execution passes without any problem encountered + // - Successful, execution passes without any problem encountered // - Failed, execution failed, and remaining gas should be consumed // - Reverted, execution failed, but only gas consumed so far is billed, remaining gas is refunded // @@ -355,7 +355,7 @@ message Call { // see above for details about those flags. string failure_reason = 11; - // This field represents wheter or not the state changes performed + // This field represents whether or not the state changes performed // by this call were correctly recorded by the blockchain. // // On Ethereum, a transaction can record state changes even if some @@ -412,7 +412,7 @@ message BalanceChange { BigInt new_value = 3; Reason reason = 4; - // Obtain all balanche change reasons under deep mind repository: + // Obtain all balance change reasons under deep mind repository: // // ```shell // ack -ho 'BalanceChangeReason\(".*"\)' | grep -Eo '".*"' | sort | uniq @@ -466,7 +466,7 @@ message CodeChange { // The gas is computed per actual op codes. Doing them completely might prove // overwhelming in most cases. // -// Hence, we only index some of them, those that are costy like all the calls +// Hence, we only index some of them, those that are costly like all the calls // one, log events, return data, etc. message GasChange { uint64 old_value = 1; @@ -505,4 +505,4 @@ message GasChange { } uint64 ordinal = 4; -} \ No newline at end of file +} diff --git a/node/src/manager/commands/drop.rs b/node/src/manager/commands/drop.rs index 30d724575c5..2c86e88e23a 100644 --- a/node/src/manager/commands/drop.rs +++ b/node/src/manager/commands/drop.rs @@ -9,8 +9,8 @@ use std::sync::Arc; /// Finds, unassigns, record and remove matching deployments. /// -/// Asks for confirmation before removing any data. -/// This is a convenience fuction that to call a series of other graphman commands. +/// Asks for confirmation before removing any data. This is a convenience +/// function that to call a series of other graphman commands. pub async fn run( primary_pool: ConnectionPool, subgraph_store: Arc, From b8bfd45a8b1fab21a837312a3f24dec87b57cef3 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Wed, 5 Mar 2025 10:42:18 +0100 Subject: [PATCH 038/150] docs: Remove scary 'experimental' warning from aggregation docs Also clarify that order by timestamp is only the default since it is now possible to order by other fields --- docs/aggregations.md | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/docs/aggregations.md b/docs/aggregations.md index 301bb70c659..fafbd4d3305 100644 --- a/docs/aggregations.md +++ b/docs/aggregations.md @@ -1,10 +1,5 @@ # Timeseries and aggregations -**This feature is experimental. We very much encourage users to try this -out, but we might still need to make material changes to what's described -here in a manner that is not backwards compatible. That might require -deleting and redeploying any subgraph that uses the features here.** - _This feature is available from spec version 1.1.0 onwards_ ## Overview @@ -188,8 +183,9 @@ accepts the following arguments: - Optional `timestamp_{gte|gt|lt|lte|eq|in}` filters to restrict the range of timestamps to return. The timestamp to filter by must be a string containing microseconds since the epoch. The value `"1704164640000000"` - corresponds to `2024-01-02T03:04Z`. -- Timeseries are always sorted by `timestamp` and `id` in descending order + corresponds to `2024-01-02T03:04Z` +- Timeseries are sorted by `timestamp` and `id` in descending order by + default ```graphql token_stats(interval: "hour", From 279318f2ff7526a55403c565712e87e2b3a35627 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Mar 2025 21:13:10 +0000 Subject: [PATCH 039/150] build(deps): bump blake3 from 1.5.1 to 1.6.1 Bumps [blake3](https://github.com/BLAKE3-team/BLAKE3) from 1.5.1 to 1.6.1. - [Release notes](https://github.com/BLAKE3-team/BLAKE3/releases) - [Commits](https://github.com/BLAKE3-team/BLAKE3/compare/1.5.1...1.6.1) --- updated-dependencies: - dependency-name: blake3 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Cargo.lock | 26 ++++++++++++++++---------- server/index-node/Cargo.toml | 2 +- store/postgres/Cargo.toml | 2 +- 3 files changed, 18 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7f3ffa09be5..18b7c9ed73a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -568,15 +568,15 @@ dependencies = [ [[package]] name = "blake3" -version = "1.5.1" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30cca6d3674597c30ddf2c587bf8d9d65c9a84d2326d941cc79c9842dfe0ef52" +checksum = "675f87afced0413c9bb02843499dbbd3882a237645883f71a2b59644a6d2f753" dependencies = [ "arrayref", "arrayvec 0.7.4", "cc", "cfg-if 1.0.0", - "constant_time_eq 0.3.0", + "constant_time_eq 0.3.1", ] [[package]] @@ -651,13 +651,13 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.105" +version = "1.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5208975e568d83b6b05cc0a063c8e7e9acc2b43bee6da15616a5b73e109d7437" +checksum = "be714c154be609ec7f5dad223a33bf1482fff90472de28f7362806e6d4832b8c" dependencies = [ "jobserver", "libc", - "once_cell", + "shlex", ] [[package]] @@ -782,9 +782,9 @@ checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" [[package]] name = "constant_time_eq" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7144d30dcf0fafbce74250a3963025d8d52177934239851c917d29f1df280c2" +checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" [[package]] name = "convert_case" @@ -2091,7 +2091,7 @@ dependencies = [ name = "graph-server-index-node" version = "0.36.0" dependencies = [ - "blake3 1.5.1", + "blake3 1.6.1", "git-testament", "graph", "graph-chain-arweave", @@ -2125,7 +2125,7 @@ dependencies = [ "Inflector", "anyhow", "async-trait", - "blake3 1.5.1", + "blake3 1.6.1", "chrono", "clap", "derive_more", @@ -4559,6 +4559,12 @@ dependencies = [ "dirs", ] +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + [[package]] name = "signal-hook-registry" version = "1.4.2" diff --git a/server/index-node/Cargo.toml b/server/index-node/Cargo.toml index edc438d1279..74048cc5e0a 100644 --- a/server/index-node/Cargo.toml +++ b/server/index-node/Cargo.toml @@ -4,7 +4,7 @@ version.workspace = true edition.workspace = true [dependencies] -blake3 = "1.5" +blake3 = "1.6" graph = { path = "../../graph" } graph-graphql = { path = "../../graphql" } graph-chain-arweave = { path = "../../chain/arweave" } diff --git a/store/postgres/Cargo.toml b/store/postgres/Cargo.toml index fa9ea5a20c5..f500cb991ee 100644 --- a/store/postgres/Cargo.toml +++ b/store/postgres/Cargo.toml @@ -5,7 +5,7 @@ edition.workspace = true [dependencies] async-trait = "0.1.50" -blake3 = "1.5" +blake3 = "1.6" chrono = { workspace = true } derive_more = { version = "0.99.18" } diesel = { workspace = true } From a1cf8127d172af1d76da6f3cfc5feac1e697a1dc Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Wed, 5 Mar 2025 12:03:20 +0100 Subject: [PATCH 040/150] chain/ethereum: Regenerate proto file The typo fixes from commit 33532bfe require that we rebuild the file; changes are solely in the comments --- .../ethereum/src/protobuf/sf.ethereum.r#type.v2.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/chain/ethereum/src/protobuf/sf.ethereum.r#type.v2.rs b/chain/ethereum/src/protobuf/sf.ethereum.r#type.v2.rs index 6d13e187d14..3f0cc728f39 100644 --- a/chain/ethereum/src/protobuf/sf.ethereum.r#type.v2.rs +++ b/chain/ethereum/src/protobuf/sf.ethereum.r#type.v2.rs @@ -12,7 +12,7 @@ pub struct Block { pub size: u64, #[prost(message, optional, tag = "5")] pub header: ::core::option::Option, - /// Uncles represents block produced with a valid solution but were not actually choosen + /// Uncles represents block produced with a valid solution but were not actually chosen /// as the canonical block for the given height so they are mostly "forked" blocks. /// /// If the Block has been produced using the Proof of Stake consensus algorithm, this @@ -360,7 +360,7 @@ pub struct Log { #[prost(bytes = "vec", tag = "3")] pub data: ::prost::alloc::vec::Vec, /// Index is the index of the log relative to the transaction. This index - /// is always populated regardless of the state revertion of the the call + /// is always populated regardless of the state reversion of the call /// that emitted this log. #[prost(uint32, tag = "4")] pub index: u32, @@ -369,7 +369,7 @@ pub struct Log { /// An **important** notice is that this field will be 0 when the call /// that emitted the log has been reverted by the chain. /// - /// Currently, there is two locations where a Log can be obtained: + /// Currently, there are two locations where a Log can be obtained: /// - block.transaction_traces\[\].receipt.logs\[\] /// - block.transaction_traces\[\].calls\[\].logs\[\] /// @@ -432,7 +432,7 @@ pub struct Call { #[prost(message, repeated, tag = "28")] pub gas_changes: ::prost::alloc::vec::Vec, /// In Ethereum, a call can be either: - /// - Successfull, execution passes without any problem encountered + /// - Successful, execution passes without any problem encountered /// - Failed, execution failed, and remaining gas should be consumed /// - Reverted, execution failed, but only gas consumed so far is billed, remaining gas is refunded /// @@ -447,7 +447,7 @@ pub struct Call { /// see above for details about those flags. #[prost(string, tag = "11")] pub failure_reason: ::prost::alloc::string::String, - /// This field represents wheter or not the state changes performed + /// This field represents whether or not the state changes performed /// by this call were correctly recorded by the blockchain. /// /// On Ethereum, a transaction can record state changes even if some @@ -507,7 +507,7 @@ pub struct BalanceChange { } /// Nested message and enum types in `BalanceChange`. pub mod balance_change { - /// Obtain all balanche change reasons under deep mind repository: + /// Obtain all balance change reasons under deep mind repository: /// /// ```shell /// ack -ho 'BalanceChangeReason\(".*"\)' | grep -Eo '".*"' | sort | uniq @@ -632,7 +632,7 @@ pub struct CodeChange { /// The gas is computed per actual op codes. Doing them completely might prove /// overwhelming in most cases. /// -/// Hence, we only index some of them, those that are costy like all the calls +/// Hence, we only index some of them, those that are costly like all the calls /// one, log events, return data, etc. #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] From 5512b619e33a5e949e2f427d0f97461c50f37ea8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 Feb 2025 21:28:09 +0000 Subject: [PATCH 041/150] build(deps): bump openssl from 0.10.66 to 0.10.71 Bumps [openssl](https://github.com/sfackler/rust-openssl) from 0.10.66 to 0.10.71. - [Release notes](https://github.com/sfackler/rust-openssl/releases) - [Commits](https://github.com/sfackler/rust-openssl/compare/openssl-v0.10.66...openssl-v0.10.71) --- updated-dependencies: - dependency-name: openssl dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Cargo.lock | 8 ++++---- store/postgres/Cargo.toml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 18b7c9ed73a..4c9f672d1f7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3317,9 +3317,9 @@ checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" [[package]] name = "openssl" -version = "0.10.66" +version = "0.10.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9529f4786b70a3e8c61e11179af17ab6188ad8d0ded78c5529441ed39d4bd9c1" +checksum = "5e14130c6a98cd258fdcb0fb6d744152343ff729cbfcb28c656a9d12b999fbcd" dependencies = [ "bitflags 2.6.0", "cfg-if 1.0.0", @@ -3349,9 +3349,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.103" +version = "0.9.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f9e8deee91df40a943c71b917e5874b951d32a802526c85721ce3b776c929d6" +checksum = "8bb61ea9811cc39e3c2069f40b8b8e2e70d8569b361f879786cc7ed48b777cdd" dependencies = [ "cc", "libc", diff --git a/store/postgres/Cargo.toml b/store/postgres/Cargo.toml index f500cb991ee..a70c2b8b614 100644 --- a/store/postgres/Cargo.toml +++ b/store/postgres/Cargo.toml @@ -21,7 +21,7 @@ lazy_static = "1.5" lru_time_cache = "0.11" maybe-owned = "0.3.4" postgres = "0.19.1" -openssl = "0.10.64" +openssl = "0.10.71" postgres-openssl = "0.5.0" rand = "0.8.4" serde = { workspace = true } From 2d7abf5cf2ad7f5bba7656cd730b796fc9db72c7 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Tue, 18 Feb 2025 09:13:19 -0800 Subject: [PATCH 042/150] all: Remove cosmos support Cosmos support has been deprecated for a while and is not used anymore --- Cargo.lock | 19 - chain/cosmos/Cargo.toml | 19 - chain/cosmos/build.rs | 54 -- chain/cosmos/proto/cosmos.proto | 368 -------- chain/cosmos/proto/cosmos_proto/cosmos.proto | 97 -- chain/cosmos/proto/firehose/annotations.proto | 11 - chain/cosmos/proto/gogoproto/gogo.proto | 145 --- chain/cosmos/src/adapter.rs | 159 ---- chain/cosmos/src/chain.rs | 715 --------------- chain/cosmos/src/codec.rs | 198 ----- chain/cosmos/src/data_source.rs | 726 --------------- chain/cosmos/src/lib.rs | 16 - chain/cosmos/src/protobuf/.gitignore | 4 - chain/cosmos/src/protobuf/mod.rs | 8 - .../src/protobuf/sf.cosmos.r#type.v1.rs | 839 ------------------ chain/cosmos/src/runtime/abi.rs | 80 -- chain/cosmos/src/runtime/mod.rs | 348 -------- chain/cosmos/src/trigger.rs | 364 -------- core/Cargo.toml | 1 - core/src/subgraph/instance_manager.rs | 15 - core/src/subgraph/registrar.rs | 18 - graph/build.rs | 1 - graph/src/blockchain/mod.rs | 5 - graph/src/firehose/codec.rs | 5 - graph/src/firehose/sf.cosmos.transform.v1.rs | 7 - graph/src/runtime/mod.rs | 72 +- node/Cargo.toml | 1 - node/src/chain.rs | 27 - node/src/network_setup.rs | 3 - server/index-node/Cargo.toml | 1 - server/index-node/src/resolver.rs | 19 - substreams/substreams-trigger-filter/build.rs | 1 - 32 files changed, 1 insertion(+), 4345 deletions(-) delete mode 100644 chain/cosmos/Cargo.toml delete mode 100644 chain/cosmos/build.rs delete mode 100644 chain/cosmos/proto/cosmos.proto delete mode 100644 chain/cosmos/proto/cosmos_proto/cosmos.proto delete mode 100644 chain/cosmos/proto/firehose/annotations.proto delete mode 100644 chain/cosmos/proto/gogoproto/gogo.proto delete mode 100644 chain/cosmos/src/adapter.rs delete mode 100644 chain/cosmos/src/chain.rs delete mode 100644 chain/cosmos/src/codec.rs delete mode 100644 chain/cosmos/src/data_source.rs delete mode 100644 chain/cosmos/src/lib.rs delete mode 100644 chain/cosmos/src/protobuf/.gitignore delete mode 100644 chain/cosmos/src/protobuf/mod.rs delete mode 100644 chain/cosmos/src/protobuf/sf.cosmos.r#type.v1.rs delete mode 100644 chain/cosmos/src/runtime/abi.rs delete mode 100644 chain/cosmos/src/runtime/mod.rs delete mode 100644 chain/cosmos/src/trigger.rs delete mode 100644 graph/src/firehose/sf.cosmos.transform.v1.rs diff --git a/Cargo.lock b/Cargo.lock index 4c9f672d1f7..384e357019a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1889,22 +1889,6 @@ dependencies = [ "protobuf-parse", ] -[[package]] -name = "graph-chain-cosmos" -version = "0.36.0" -dependencies = [ - "anyhow", - "graph", - "graph-chain-common", - "graph-runtime-derive", - "graph-runtime-wasm", - "prost 0.12.6", - "prost-types 0.12.6", - "semver", - "serde", - "tonic-build", -] - [[package]] name = "graph-chain-ethereum" version = "0.36.0" @@ -1972,7 +1956,6 @@ dependencies = [ "cid", "graph", "graph-chain-arweave", - "graph-chain-cosmos", "graph-chain-ethereum", "graph-chain-near", "graph-chain-substreams", @@ -2010,7 +1993,6 @@ dependencies = [ "git-testament", "graph", "graph-chain-arweave", - "graph-chain-cosmos", "graph-chain-ethereum", "graph-chain-near", "graph-chain-substreams", @@ -2095,7 +2077,6 @@ dependencies = [ "git-testament", "graph", "graph-chain-arweave", - "graph-chain-cosmos", "graph-chain-ethereum", "graph-chain-near", "graph-chain-substreams", diff --git a/chain/cosmos/Cargo.toml b/chain/cosmos/Cargo.toml deleted file mode 100644 index 4d3b598d046..00000000000 --- a/chain/cosmos/Cargo.toml +++ /dev/null @@ -1,19 +0,0 @@ -[package] -name = "graph-chain-cosmos" -version.workspace = true -edition = "2018" - -[build-dependencies] -tonic-build = { workspace = true } -graph-chain-common = { path = "../common" } - -[dependencies] -graph = { path = "../../graph" } -prost = { workspace = true } -prost-types = { workspace = true } -serde = { workspace = true } -anyhow = "1.0" -semver = "1.0.23" - -graph-runtime-wasm = { path = "../../runtime/wasm" } -graph-runtime-derive = { path = "../../runtime/derive" } diff --git a/chain/cosmos/build.rs b/chain/cosmos/build.rs deleted file mode 100644 index c618d3b466d..00000000000 --- a/chain/cosmos/build.rs +++ /dev/null @@ -1,54 +0,0 @@ -const PROTO_FILE: &str = "proto/cosmos.proto"; - -fn main() { - println!("cargo:rerun-if-changed=proto"); - - let types = - graph_chain_common::parse_proto_file(PROTO_FILE).expect("Unable to parse proto file!"); - - let array_types = types - .iter() - .flat_map(|(_, t)| t.fields.iter()) - .filter(|t| t.is_array) - .map(|t| t.type_name.clone()) - .collect::>(); - - let mut builder = tonic_build::configure().out_dir("src/protobuf"); - - for (name, ptype) in types { - //generate Asc - builder = builder.type_attribute( - name.clone(), - format!( - "#[graph_runtime_derive::generate_asc_type({})]", - ptype.fields().unwrap_or_default() - ), - ); - - //generate data index id - builder = builder.type_attribute( - name.clone(), - "#[graph_runtime_derive::generate_network_type_id(Cosmos)]", - ); - - //generate conversion from rust type to asc - builder = builder.type_attribute( - name.clone(), - format!( - "#[graph_runtime_derive::generate_from_rust_type({})]", - ptype.fields().unwrap_or_default() - ), - ); - - if array_types.contains(&ptype.name) { - builder = builder.type_attribute( - name.clone(), - "#[graph_runtime_derive::generate_array_type(Cosmos)]", - ); - } - } - - builder - .compile(&[PROTO_FILE], &["proto"]) - .expect("Failed to compile Firehose Cosmos proto(s)"); -} diff --git a/chain/cosmos/proto/cosmos.proto b/chain/cosmos/proto/cosmos.proto deleted file mode 100644 index c32502da1e9..00000000000 --- a/chain/cosmos/proto/cosmos.proto +++ /dev/null @@ -1,368 +0,0 @@ -syntax = "proto3"; - -package sf.cosmos.type.v1; - -option go_package = "github.com/figment-networks/proto-cosmos/pb/sf/cosmos/type/v1;pbcosmos"; - -import "google/protobuf/descriptor.proto"; -import "google/protobuf/any.proto"; -import "gogoproto/gogo.proto"; -import "cosmos_proto/cosmos.proto"; -import "firehose/annotations.proto"; - -message Block { - Header header = 1 [(firehose.required) = true, (gogoproto.nullable) = false]; - EvidenceList evidence = 2 [(gogoproto.nullable) = false]; - Commit last_commit = 3; - ResponseBeginBlock result_begin_block = 4 [(firehose.required) = true]; - ResponseEndBlock result_end_block = 5 [(firehose.required) = true]; - repeated TxResult transactions = 7; - repeated Validator validator_updates = 8; -} - -// HeaderOnlyBlock is a standard [Block] structure where all other fields are -// removed so that hydrating that object from a [Block] bytes payload will -// drastically reduce allocated memory required to hold the full block. -// -// This can be used to unpack a [Block] when only the [Header] information -// is required and greatly reduce required memory. -message HeaderOnlyBlock { - Header header = 1 [(firehose.required) = true, (gogoproto.nullable) = false]; -} - -message EventData { - Event event = 1 [(firehose.required) = true]; - HeaderOnlyBlock block = 2 [(firehose.required) = true]; - TransactionContext tx = 3; -} - -message TransactionData { - TxResult tx = 1 [(firehose.required) = true]; - HeaderOnlyBlock block = 2 [(firehose.required) = true]; -} - -message MessageData { - google.protobuf.Any message = 1 [(firehose.required) = true]; - HeaderOnlyBlock block = 2 [(firehose.required) = true]; - TransactionContext tx = 3 [(firehose.required) = true]; -} - -message TransactionContext { - bytes hash = 1; - uint32 index = 2; - uint32 code = 3; - int64 gas_wanted = 4; - int64 gas_used = 5; -} - -message Header { - Consensus version = 1 [(gogoproto.nullable) = false]; - string chain_id = 2 [(gogoproto.customname) = "ChainID"]; - uint64 height = 3; - Timestamp time = 4 [(gogoproto.nullable) = false, (gogoproto.stdtime) = true]; - BlockID last_block_id = 5 [(firehose.required) = true, (gogoproto.nullable) = false]; - bytes last_commit_hash = 6; - bytes data_hash = 7; - bytes validators_hash = 8; - bytes next_validators_hash = 9; - bytes consensus_hash = 10; - bytes app_hash = 11; - bytes last_results_hash = 12; - bytes evidence_hash = 13; - bytes proposer_address = 14; - bytes hash = 15; -} - -message Consensus { - option (gogoproto.equal) = true; - - uint64 block = 1; - uint64 app = 2; -} - -message Timestamp { - int64 seconds = 1; - int32 nanos = 2; -} - -message BlockID { - bytes hash = 1; - PartSetHeader part_set_header = 2 [(gogoproto.nullable) = false]; -} - -message PartSetHeader { - uint32 total = 1; - bytes hash = 2; -} - -message EvidenceList { - repeated Evidence evidence = 1 [(gogoproto.nullable) = false]; -} - -message Evidence { - oneof sum { - DuplicateVoteEvidence duplicate_vote_evidence = 1; - LightClientAttackEvidence light_client_attack_evidence = 2; - } -} - -message DuplicateVoteEvidence { - EventVote vote_a = 1; - EventVote vote_b = 2; - int64 total_voting_power = 3; - int64 validator_power = 4; - Timestamp timestamp = 5 [(gogoproto.nullable) = false, (gogoproto.stdtime) = true]; -} - -message EventVote { - SignedMsgType event_vote_type = 1 [json_name = "type"]; - uint64 height = 2; - int32 round = 3; - BlockID block_id = 4 [(gogoproto.nullable) = false, (gogoproto.customname) = "BlockID"]; - Timestamp timestamp = 5 [(gogoproto.nullable) = false, (gogoproto.stdtime) = true]; - bytes validator_address = 6; - int32 validator_index = 7; - bytes signature = 8; -} - -enum SignedMsgType { - option (gogoproto.goproto_enum_stringer) = true; - option (gogoproto.goproto_enum_prefix) = false; - - SIGNED_MSG_TYPE_UNKNOWN = 0 [(gogoproto.enumvalue_customname) = "UnknownType"]; - SIGNED_MSG_TYPE_PREVOTE = 1 [(gogoproto.enumvalue_customname) = "PrevoteType"]; - SIGNED_MSG_TYPE_PRECOMMIT = 2 [(gogoproto.enumvalue_customname) = "PrecommitType"]; - SIGNED_MSG_TYPE_PROPOSAL = 32 [(gogoproto.enumvalue_customname) = "ProposalType"]; -} - -message LightClientAttackEvidence { - LightBlock conflicting_block = 1; - int64 common_height = 2; - repeated Validator byzantine_validators = 3; - int64 total_voting_power = 4; - Timestamp timestamp = 5 [(gogoproto.nullable) = false, (gogoproto.stdtime) = true]; -} - -message LightBlock { - SignedHeader signed_header = 1; - ValidatorSet validator_set = 2; -} - -message SignedHeader { - Header header = 1; - Commit commit = 2; -} - -message Commit { - int64 height = 1; - int32 round = 2; - BlockID block_id = 3 [(gogoproto.nullable) = false, (gogoproto.customname) = "BlockID"]; - repeated CommitSig signatures = 4 [(gogoproto.nullable) = false]; -} - -message CommitSig { - BlockIDFlag block_id_flag = 1; - bytes validator_address = 2; - Timestamp timestamp = 3 [(gogoproto.nullable) = false, (gogoproto.stdtime) = true]; - bytes signature = 4; -} - -enum BlockIDFlag { - option (gogoproto.goproto_enum_stringer) = true; - option (gogoproto.goproto_enum_prefix) = false; - - BLOCK_ID_FLAG_UNKNOWN = 0 [(gogoproto.enumvalue_customname) = "BlockIDFlagUnknown"]; - BLOCK_ID_FLAG_ABSENT = 1 [(gogoproto.enumvalue_customname) = "BlockIDFlagAbsent"]; - BLOCK_ID_FLAG_COMMIT = 2 [(gogoproto.enumvalue_customname) = "BlockIDFlagCommit"]; - BLOCK_ID_FLAG_NIL = 3 [(gogoproto.enumvalue_customname) = "BlockIDFlagNil"]; -} - -message ValidatorSet { - repeated Validator validators = 1; - Validator proposer = 2; - int64 total_voting_power = 3; -} - -message Validator { - bytes address = 1; - PublicKey pub_key = 2 [(gogoproto.nullable) = false]; - int64 voting_power = 3; - int64 proposer_priority = 4; -} - -message PublicKey { - option (gogoproto.compare) = true; - option (gogoproto.equal) = true; - - oneof sum { - bytes ed25519 = 1; - bytes secp256k1 = 2; - } -} - -message ResponseBeginBlock { - repeated Event events = 1 [(gogoproto.nullable) = false, (gogoproto.jsontag) = "events,omitempty"]; -} - -message Event { - string event_type = 1 [json_name = "type"]; - repeated EventAttribute attributes = 2 [(gogoproto.nullable) = false, (gogoproto.jsontag) = "attributes,omitempty"]; -} - -message EventAttribute { - string key = 1; - string value = 2; - bool index = 3; -} - -message ResponseEndBlock { - repeated ValidatorUpdate validator_updates = 1; - ConsensusParams consensus_param_updates = 2; - repeated Event events = 3; -} - -message ValidatorUpdate { - bytes address = 1; - PublicKey pub_key = 2 [(gogoproto.nullable) = false]; - int64 power = 3; -} - -message ConsensusParams { - BlockParams block = 1 [(gogoproto.nullable) = false]; - EvidenceParams evidence = 2 [(gogoproto.nullable) = false]; - ValidatorParams validator = 3 [(gogoproto.nullable) = false]; - VersionParams version = 4 [(gogoproto.nullable) = false]; -} - -message BlockParams { - int64 max_bytes = 1; - int64 max_gas = 2; -} - -message EvidenceParams { - int64 max_age_num_blocks = 1; - Duration max_age_duration = 2 [(gogoproto.nullable) = false, (gogoproto.stdduration) = true]; - int64 max_bytes = 3; -} - -message Duration { - int64 seconds = 1; - int32 nanos = 2; -} - -message ValidatorParams { - option (gogoproto.populate) = true; - option (gogoproto.equal) = true; - - repeated string pub_key_types = 1; -} - -message VersionParams { - option (gogoproto.populate) = true; - option (gogoproto.equal) = true; - - uint64 app_version = 1; -} - -message TxResult { - uint64 height = 1; - uint32 index = 2; - Tx tx = 3 [(firehose.required) = true]; - ResponseDeliverTx result = 4 [(firehose.required) = true]; - bytes hash = 5; -} - -message Tx { - TxBody body = 1 [(firehose.required) = true]; - AuthInfo auth_info = 2; - repeated bytes signatures = 3; -} - -message TxBody { - repeated google.protobuf.Any messages = 1; - string memo = 2; - uint64 timeout_height = 3; - repeated google.protobuf.Any extension_options = 1023; - repeated google.protobuf.Any non_critical_extension_options = 2047; -} - -message Any { - string type_url = 1; - bytes value = 2; -} - -message AuthInfo { - repeated SignerInfo signer_infos = 1; - Fee fee = 2; - Tip tip = 3; -} - -message SignerInfo { - google.protobuf.Any public_key = 1; - ModeInfo mode_info = 2; - uint64 sequence = 3; -} - -message ModeInfo { - oneof sum { - ModeInfoSingle single = 1; - ModeInfoMulti multi = 2; - } -} - -message ModeInfoSingle { - SignMode mode = 1; -} - -enum SignMode { - SIGN_MODE_UNSPECIFIED = 0; - SIGN_MODE_DIRECT = 1; - SIGN_MODE_TEXTUAL = 2; - SIGN_MODE_LEGACY_AMINO_JSON = 127; -} - -message ModeInfoMulti { - CompactBitArray bitarray = 1; - repeated ModeInfo mode_infos = 2; -} - -message CompactBitArray { - option (gogoproto.goproto_stringer) = false; - - uint32 extra_bits_stored = 1; - bytes elems = 2; -} - -message Fee { - repeated Coin amount = 1 [(gogoproto.nullable) = false, (gogoproto.castrepeated) = "github.com/cosmos/cosmos-sdk/types.Coins"]; - uint64 gas_limit = 2; - string payer = 3 [(cosmos_proto.scalar) = "cosmos.AddressString"]; - string granter = 4 [(cosmos_proto.scalar) = "cosmos.AddressString"]; -} - -message Coin { - option (gogoproto.equal) = true; - - string denom = 1; - string amount = 2 [(gogoproto.customtype) = "Int", (gogoproto.nullable) = false]; -} - -message Tip { - repeated Coin amount = 1 [(gogoproto.nullable) = false, (gogoproto.castrepeated) = "github.com/cosmos/cosmos-sdk/types.Coins"]; - string tipper = 2 [(cosmos_proto.scalar) = "cosmos.AddressString"]; -} - -message ResponseDeliverTx { - uint32 code = 1; - bytes data = 2; - string log = 3; - string info = 4; - int64 gas_wanted = 5; - int64 gas_used = 6; - repeated Event events = 7 [(gogoproto.nullable) = false, (gogoproto.jsontag) = "events,omitempty"]; - string codespace = 8; -} - -message ValidatorSetUpdates { - repeated Validator validator_updates = 1; -} diff --git a/chain/cosmos/proto/cosmos_proto/cosmos.proto b/chain/cosmos/proto/cosmos_proto/cosmos.proto deleted file mode 100644 index 5c63b86f063..00000000000 --- a/chain/cosmos/proto/cosmos_proto/cosmos.proto +++ /dev/null @@ -1,97 +0,0 @@ -syntax = "proto3"; -package cosmos_proto; - -import "google/protobuf/descriptor.proto"; - -option go_package = "github.com/cosmos/cosmos-proto;cosmos_proto"; - -extend google.protobuf.MessageOptions { - - // implements_interface is used to indicate the type name of the interface - // that a message implements so that it can be used in google.protobuf.Any - // fields that accept that interface. A message can implement multiple - // interfaces. Interfaces should be declared using a declare_interface - // file option. - repeated string implements_interface = 93001; -} - -extend google.protobuf.FieldOptions { - - // accepts_interface is used to annotate that a google.protobuf.Any - // field accepts messages that implement the specified interface. - // Interfaces should be declared using a declare_interface file option. - string accepts_interface = 93001; - - // scalar is used to indicate that this field follows the formatting defined - // by the named scalar which should be declared with declare_scalar. Code - // generators may choose to use this information to map this field to a - // language-specific type representing the scalar. - string scalar = 93002; -} - -extend google.protobuf.FileOptions { - - // declare_interface declares an interface type to be used with - // accepts_interface and implements_interface. Interface names are - // expected to follow the following convention such that their declaration - // can be discovered by tools: for a given interface type a.b.C, it is - // expected that the declaration will be found in a protobuf file named - // a/b/interfaces.proto in the file descriptor set. - repeated InterfaceDescriptor declare_interface = 793021; - - // declare_scalar declares a scalar type to be used with - // the scalar field option. Scalar names are - // expected to follow the following convention such that their declaration - // can be discovered by tools: for a given scalar type a.b.C, it is - // expected that the declaration will be found in a protobuf file named - // a/b/scalars.proto in the file descriptor set. - repeated ScalarDescriptor declare_scalar = 793022; -} - -// InterfaceDescriptor describes an interface type to be used with -// accepts_interface and implements_interface and declared by declare_interface. -message InterfaceDescriptor { - - // name is the name of the interface. It should be a short-name (without - // a period) such that the fully qualified name of the interface will be - // package.name, ex. for the package a.b and interface named C, the - // fully-qualified name will be a.b.C. - string name = 1; - - // description is a human-readable description of the interface and its - // purpose. - string description = 2; -} - -// ScalarDescriptor describes an scalar type to be used with -// the scalar field option and declared by declare_scalar. -// Scalars extend simple protobuf built-in types with additional -// syntax and semantics, for instance to represent big integers. -// Scalars should ideally define an encoding such that there is only one -// valid syntactical representation for a given semantic meaning, -// i.e. the encoding should be deterministic. -message ScalarDescriptor { - - // name is the name of the scalar. It should be a short-name (without - // a period) such that the fully qualified name of the scalar will be - // package.name, ex. for the package a.b and scalar named C, the - // fully-qualified name will be a.b.C. - string name = 1; - - // description is a human-readable description of the scalar and its - // encoding format. For instance a big integer or decimal scalar should - // specify precisely the expected encoding format. - string description = 2; - - // field_type is the type of field with which this scalar can be used. - // Scalars can be used with one and only one type of field so that - // encoding standards and simple and clear. Currently only string and - // bytes fields are supported for scalars. - repeated ScalarType field_type = 3; -} - -enum ScalarType { - SCALAR_TYPE_UNSPECIFIED = 0; - SCALAR_TYPE_STRING = 1; - SCALAR_TYPE_BYTES = 2; -} diff --git a/chain/cosmos/proto/firehose/annotations.proto b/chain/cosmos/proto/firehose/annotations.proto deleted file mode 100644 index 1476c1ab08d..00000000000 --- a/chain/cosmos/proto/firehose/annotations.proto +++ /dev/null @@ -1,11 +0,0 @@ -syntax = "proto3"; - -package firehose; - -option go_package = "github.com/streamingfast/pbgo/sf/firehose/v1;pbfirehose"; - -import "google/protobuf/descriptor.proto"; - -extend google.protobuf.FieldOptions { - optional bool required = 77001; -} diff --git a/chain/cosmos/proto/gogoproto/gogo.proto b/chain/cosmos/proto/gogoproto/gogo.proto deleted file mode 100644 index 49e78f99fe5..00000000000 --- a/chain/cosmos/proto/gogoproto/gogo.proto +++ /dev/null @@ -1,145 +0,0 @@ -// Protocol Buffers for Go with Gadgets -// -// Copyright (c) 2013, The GoGo Authors. All rights reserved. -// http://github.com/gogo/protobuf -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -syntax = "proto2"; -package gogoproto; - -import "google/protobuf/descriptor.proto"; - -option java_package = "com.google.protobuf"; -option java_outer_classname = "GoGoProtos"; -option go_package = "github.com/gogo/protobuf/gogoproto"; - -extend google.protobuf.EnumOptions { - optional bool goproto_enum_prefix = 62001; - optional bool goproto_enum_stringer = 62021; - optional bool enum_stringer = 62022; - optional string enum_customname = 62023; - optional bool enumdecl = 62024; -} - -extend google.protobuf.EnumValueOptions { - optional string enumvalue_customname = 66001; -} - -extend google.protobuf.FileOptions { - optional bool goproto_getters_all = 63001; - optional bool goproto_enum_prefix_all = 63002; - optional bool goproto_stringer_all = 63003; - optional bool verbose_equal_all = 63004; - optional bool face_all = 63005; - optional bool gostring_all = 63006; - optional bool populate_all = 63007; - optional bool stringer_all = 63008; - optional bool onlyone_all = 63009; - - optional bool equal_all = 63013; - optional bool description_all = 63014; - optional bool testgen_all = 63015; - optional bool benchgen_all = 63016; - optional bool marshaler_all = 63017; - optional bool unmarshaler_all = 63018; - optional bool stable_marshaler_all = 63019; - - optional bool sizer_all = 63020; - - optional bool goproto_enum_stringer_all = 63021; - optional bool enum_stringer_all = 63022; - - optional bool unsafe_marshaler_all = 63023; - optional bool unsafe_unmarshaler_all = 63024; - - optional bool goproto_extensions_map_all = 63025; - optional bool goproto_unrecognized_all = 63026; - optional bool gogoproto_import = 63027; - optional bool protosizer_all = 63028; - optional bool compare_all = 63029; - optional bool typedecl_all = 63030; - optional bool enumdecl_all = 63031; - - optional bool goproto_registration = 63032; - optional bool messagename_all = 63033; - - optional bool goproto_sizecache_all = 63034; - optional bool goproto_unkeyed_all = 63035; -} - -extend google.protobuf.MessageOptions { - optional bool goproto_getters = 64001; - optional bool goproto_stringer = 64003; - optional bool verbose_equal = 64004; - optional bool face = 64005; - optional bool gostring = 64006; - optional bool populate = 64007; - optional bool stringer = 67008; - optional bool onlyone = 64009; - - optional bool equal = 64013; - optional bool description = 64014; - optional bool testgen = 64015; - optional bool benchgen = 64016; - optional bool marshaler = 64017; - optional bool unmarshaler = 64018; - optional bool stable_marshaler = 64019; - - optional bool sizer = 64020; - - optional bool unsafe_marshaler = 64023; - optional bool unsafe_unmarshaler = 64024; - - optional bool goproto_extensions_map = 64025; - optional bool goproto_unrecognized = 64026; - - optional bool protosizer = 64028; - optional bool compare = 64029; - - optional bool typedecl = 64030; - - optional bool messagename = 64033; - - optional bool goproto_sizecache = 64034; - optional bool goproto_unkeyed = 64035; -} - -extend google.protobuf.FieldOptions { - optional bool nullable = 65001; - optional bool embed = 65002; - optional string customtype = 65003; - optional string customname = 65004; - optional string jsontag = 65005; - optional string moretags = 65006; - optional string casttype = 65007; - optional string castkey = 65008; - optional string castvalue = 65009; - - optional bool stdtime = 65010; - optional bool stdduration = 65011; - optional bool wktpointer = 65012; - - optional string castrepeated = 65013; -} diff --git a/chain/cosmos/src/adapter.rs b/chain/cosmos/src/adapter.rs deleted file mode 100644 index 746c91e2e07..00000000000 --- a/chain/cosmos/src/adapter.rs +++ /dev/null @@ -1,159 +0,0 @@ -use std::collections::HashSet; - -use prost::Message; -use prost_types::Any; - -use crate::{data_source::DataSource, Chain}; -use graph::blockchain as bc; -use graph::firehose::EventTypeFilter; -use graph::prelude::*; - -const EVENT_TYPE_FILTER_TYPE_URL: &str = - "type.googleapis.com/sf.cosmos.transform.v1.EventTypeFilter"; - -#[derive(Clone, Debug, Default)] -pub struct TriggerFilter { - pub(crate) event_type_filter: CosmosEventTypeFilter, - pub(crate) block_filter: CosmosBlockFilter, -} - -impl bc::TriggerFilter for TriggerFilter { - fn extend<'a>(&mut self, data_sources: impl Iterator + Clone) { - self.event_type_filter - .extend_from_data_sources(data_sources.clone()); - self.block_filter.extend_from_data_sources(data_sources); - } - - fn node_capabilities(&self) -> bc::EmptyNodeCapabilities { - bc::EmptyNodeCapabilities::default() - } - - fn extend_with_template( - &mut self, - _data_source: impl Iterator::DataSourceTemplate>, - ) { - } - - fn to_firehose_filter(self) -> Vec { - if self.block_filter.trigger_every_block { - return vec![]; - } - - if self.event_type_filter.event_types.is_empty() { - return vec![]; - } - - let filter = EventTypeFilter { - event_types: Vec::from_iter(self.event_type_filter.event_types), - }; - - vec![Any { - type_url: EVENT_TYPE_FILTER_TYPE_URL.to_string(), - value: filter.encode_to_vec(), - }] - } -} - -pub type EventType = String; - -#[derive(Clone, Debug, Default)] -pub(crate) struct CosmosEventTypeFilter { - pub event_types: HashSet, -} - -impl CosmosEventTypeFilter { - pub(crate) fn matches(&self, event_type: &EventType) -> bool { - self.event_types.contains(event_type) - } - - fn extend_from_data_sources<'a>(&mut self, data_sources: impl Iterator) { - self.event_types.extend( - data_sources.flat_map(|data_source| data_source.events().map(ToString::to_string)), - ); - } -} - -#[derive(Clone, Debug, Default)] -pub(crate) struct CosmosBlockFilter { - pub trigger_every_block: bool, -} - -impl CosmosBlockFilter { - fn extend_from_data_sources<'a>( - &mut self, - mut data_sources: impl Iterator, - ) { - if !self.trigger_every_block { - self.trigger_every_block = data_sources.any(DataSource::has_block_handler); - } - } -} - -#[cfg(test)] -mod test { - use graph::blockchain::TriggerFilter as _; - - use super::*; - - #[test] - fn test_trigger_filters() { - let cases = [ - (TriggerFilter::test_new(false, &[]), None), - (TriggerFilter::test_new(true, &[]), None), - (TriggerFilter::test_new(true, &["event_1", "event_2"]), None), - ( - TriggerFilter::test_new(false, &["event_1", "event_2", "event_3"]), - Some(event_type_filter_with(&["event_1", "event_2", "event_3"])), - ), - ]; - - for (trigger_filter, expected_filter) in cases { - let firehose_filter = trigger_filter.to_firehose_filter(); - let decoded_filter = decode_filter(firehose_filter); - - assert_eq!(decoded_filter.is_some(), expected_filter.is_some()); - - if let (Some(mut expected_filter), Some(mut decoded_filter)) = - (expected_filter, decoded_filter) - { - // event types may be in different order - expected_filter.event_types.sort(); - decoded_filter.event_types.sort(); - - assert_eq!(decoded_filter, expected_filter); - } - } - } - - impl TriggerFilter { - pub(crate) fn test_new(trigger_every_block: bool, event_types: &[&str]) -> TriggerFilter { - TriggerFilter { - event_type_filter: CosmosEventTypeFilter { - event_types: event_types.iter().map(ToString::to_string).collect(), - }, - block_filter: CosmosBlockFilter { - trigger_every_block, - }, - } - } - } - - fn event_type_filter_with(event_types: &[&str]) -> EventTypeFilter { - EventTypeFilter { - event_types: event_types.iter().map(ToString::to_string).collect(), - } - } - - fn decode_filter(proto_filters: Vec) -> Option { - assert!(proto_filters.len() <= 1); - - let proto_filter = proto_filters.get(0)?; - - assert_eq!(proto_filter.type_url, EVENT_TYPE_FILTER_TYPE_URL); - - let firehose_filter = EventTypeFilter::decode(&*proto_filter.value) - .expect("Could not decode EventTypeFilter from protobuf Any"); - - Some(firehose_filter) - } -} diff --git a/chain/cosmos/src/chain.rs b/chain/cosmos/src/chain.rs deleted file mode 100644 index 353b1e4dbbe..00000000000 --- a/chain/cosmos/src/chain.rs +++ /dev/null @@ -1,715 +0,0 @@ -use graph::blockchain::firehose_block_ingestor::FirehoseBlockIngestor; -use graph::blockchain::{BlockIngestor, NoopDecoderHook, TriggerFilterWrapper}; -use graph::components::network_provider::ChainName; -use graph::env::EnvVars; -use graph::prelude::MetricsRegistry; -use graph::substreams::Clock; -use std::collections::BTreeSet; -use std::convert::TryFrom; -use std::sync::Arc; - -use graph::blockchain::block_stream::{BlockStreamError, BlockStreamMapper, FirehoseCursor}; -use graph::blockchain::client::ChainClient; -use graph::blockchain::{BasicBlockchainBuilder, BlockchainBuilder, NoopRuntimeAdapter}; -use graph::cheap_clone::CheapClone; -use graph::components::store::{DeploymentCursorTracker, SourceableStore}; -use graph::data::subgraph::UnifiedMappingApiVersion; -use graph::{ - blockchain::{ - block_stream::{ - BlockStream, BlockStreamEvent, BlockWithTriggers, FirehoseError, - FirehoseMapper as FirehoseMapperTrait, TriggersAdapter as TriggersAdapterTrait, - }, - firehose_block_stream::FirehoseBlockStream, - Block as _, BlockHash, BlockPtr, Blockchain, BlockchainKind, EmptyNodeCapabilities, - IngestorError, RuntimeAdapter as RuntimeAdapterTrait, - }, - components::store::DeploymentLocator, - firehose::{self, FirehoseEndpoint, ForkStep}, - prelude::{async_trait, o, BlockNumber, ChainStore, Error, Logger, LoggerFactory}, -}; -use prost::Message; - -use crate::data_source::{ - DataSource, DataSourceTemplate, EventOrigin, UnresolvedDataSource, UnresolvedDataSourceTemplate, -}; -use crate::trigger::CosmosTrigger; -use crate::{codec, Block, TriggerFilter}; - -pub struct Chain { - logger_factory: LoggerFactory, - name: ChainName, - client: Arc>, - chain_store: Arc, - metrics_registry: Arc, -} - -impl std::fmt::Debug for Chain { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "chain: cosmos") - } -} - -#[async_trait] -impl BlockchainBuilder for BasicBlockchainBuilder { - async fn build(self, _config: &Arc) -> Chain { - Chain { - logger_factory: self.logger_factory, - name: self.name, - client: Arc::new(ChainClient::new_firehose(self.firehose_endpoints)), - chain_store: self.chain_store, - metrics_registry: self.metrics_registry, - } - } -} - -#[async_trait] -impl Blockchain for Chain { - const KIND: BlockchainKind = BlockchainKind::Cosmos; - - type Client = (); - type Block = codec::Block; - - type DataSource = DataSource; - - type UnresolvedDataSource = UnresolvedDataSource; - - type DataSourceTemplate = DataSourceTemplate; - - type UnresolvedDataSourceTemplate = UnresolvedDataSourceTemplate; - - type TriggerData = CosmosTrigger; - - type MappingTrigger = CosmosTrigger; - - type TriggerFilter = TriggerFilter; - - type NodeCapabilities = EmptyNodeCapabilities; - - type DecoderHook = NoopDecoderHook; - - fn is_refetch_block_required(&self) -> bool { - false - } - async fn refetch_firehose_block( - &self, - _logger: &Logger, - _cursor: FirehoseCursor, - ) -> Result { - unimplemented!("This chain does not support Dynamic Data Sources. is_refetch_block_required always returns false, this shouldn't be called.") - } - - fn triggers_adapter( - &self, - _loc: &DeploymentLocator, - _capabilities: &Self::NodeCapabilities, - _unified_api_version: UnifiedMappingApiVersion, - ) -> Result>, Error> { - let adapter = TriggersAdapter {}; - Ok(Arc::new(adapter)) - } - - async fn new_block_stream( - &self, - deployment: DeploymentLocator, - store: impl DeploymentCursorTracker, - start_blocks: Vec, - _source_subgraph_stores: Vec>, - filter: Arc>, - unified_api_version: UnifiedMappingApiVersion, - ) -> Result>, Error> { - let adapter = self - .triggers_adapter( - &deployment, - &EmptyNodeCapabilities::default(), - unified_api_version, - ) - .unwrap_or_else(|_| panic!("no adapter for network {}", self.name)); - - let logger = self - .logger_factory - .subgraph_logger(&deployment) - .new(o!("component" => "FirehoseBlockStream")); - - let firehose_mapper = Arc::new(FirehoseMapper { - adapter, - filter: filter.chain_filter.clone(), - }); - - Ok(Box::new(FirehoseBlockStream::new( - deployment.hash, - self.chain_client(), - store.block_ptr(), - store.firehose_cursor(), - firehose_mapper, - start_blocks, - logger, - self.metrics_registry.clone(), - ))) - } - - fn chain_store(&self) -> Arc { - self.chain_store.cheap_clone() - } - - async fn block_pointer_from_number( - &self, - logger: &Logger, - number: BlockNumber, - ) -> Result { - let firehose_endpoint = self.client.firehose_endpoint().await?; - - firehose_endpoint - .block_ptr_for_number::(logger, number) - .await - .map_err(Into::into) - } - - fn runtime(&self) -> anyhow::Result<(Arc>, Self::DecoderHook)> { - Ok((Arc::new(NoopRuntimeAdapter::default()), NoopDecoderHook)) - } - - fn chain_client(&self) -> Arc> { - self.client.clone() - } - - async fn block_ingestor(&self) -> anyhow::Result> { - let ingestor = FirehoseBlockIngestor::::new( - self.chain_store.cheap_clone(), - self.chain_client(), - self.logger_factory - .component_logger("CosmosFirehoseBlockIngestor", None), - self.name.clone(), - ); - Ok(Box::new(ingestor)) - } -} - -pub struct TriggersAdapter {} - -#[async_trait] -impl TriggersAdapterTrait for TriggersAdapter { - async fn ancestor_block( - &self, - _ptr: BlockPtr, - _offset: BlockNumber, - _root: Option, - ) -> Result, Error> { - panic!("Should never be called since not used by FirehoseBlockStream") - } - async fn load_block_ptrs_by_numbers( - &self, - _logger: Logger, - _block_numbers: BTreeSet, - ) -> Result, Error> { - todo!() - } - async fn chain_head_ptr(&self) -> Result, Error> { - unimplemented!() - } - - async fn scan_triggers( - &self, - _from: BlockNumber, - _to: BlockNumber, - _filter: &TriggerFilter, - ) -> Result<(Vec>, BlockNumber), Error> { - panic!("Should never be called since not used by FirehoseBlockStream") - } - - async fn triggers_in_block( - &self, - logger: &Logger, - block: codec::Block, - filter: &TriggerFilter, - ) -> Result, Error> { - let shared_block = Arc::new(block.clone()); - - let header_only_block = codec::HeaderOnlyBlock::from(&block); - - let mut triggers: Vec<_> = shared_block - .begin_block_events()? - .cloned() - // FIXME (Cosmos): Optimize. Should use an Arc instead of cloning the - // block. This is not currently possible because EventData is automatically - // generated. - .filter_map(|event| { - filter_event_trigger( - filter, - event, - &header_only_block, - None, - EventOrigin::BeginBlock, - ) - }) - .chain(shared_block.transactions().flat_map(|tx| { - tx.result - .as_ref() - .unwrap() - .events - .iter() - .filter_map(|e| { - filter_event_trigger( - filter, - e.clone(), - &header_only_block, - Some(build_tx_context(tx)), - EventOrigin::DeliverTx, - ) - }) - .collect::>() - })) - .chain( - shared_block - .end_block_events()? - .cloned() - .filter_map(|event| { - filter_event_trigger( - filter, - event, - &header_only_block, - None, - EventOrigin::EndBlock, - ) - }), - ) - .collect(); - - triggers.extend(shared_block.transactions().cloned().flat_map(|tx_result| { - let mut triggers: Vec<_> = Vec::new(); - if let Some(tx) = tx_result.tx.clone() { - if let Some(tx_body) = tx.body { - triggers.extend(tx_body.messages.into_iter().map(|message| { - CosmosTrigger::with_message( - message, - header_only_block.clone(), - build_tx_context(&tx_result), - ) - })); - } - } - triggers.push(CosmosTrigger::with_transaction( - tx_result, - header_only_block.clone(), - )); - triggers - })); - - if filter.block_filter.trigger_every_block { - triggers.push(CosmosTrigger::Block(shared_block.cheap_clone())); - } - - Ok(BlockWithTriggers::new(block, triggers, logger)) - } - - async fn is_on_main_chain(&self, _ptr: BlockPtr) -> Result { - panic!("Should never be called since not used by FirehoseBlockStream") - } - - /// Panics if `block` is genesis. - /// But that's ok since this is only called when reverting `block`. - async fn parent_ptr(&self, block: &BlockPtr) -> Result, Error> { - Ok(Some(BlockPtr { - hash: BlockHash::from(vec![0xff; 32]), - number: block.number.saturating_sub(1), - })) - } -} - -/// Returns a new event trigger only if the given event matches the event filter. -fn filter_event_trigger( - filter: &TriggerFilter, - event: codec::Event, - block: &codec::HeaderOnlyBlock, - tx_context: Option, - origin: EventOrigin, -) -> Option { - if filter.event_type_filter.matches(&event.event_type) { - Some(CosmosTrigger::with_event( - event, - block.clone(), - tx_context, - origin, - )) - } else { - None - } -} - -fn build_tx_context(tx: &codec::TxResult) -> codec::TransactionContext { - codec::TransactionContext { - hash: tx.hash.clone(), - index: tx.index, - code: tx.result.as_ref().unwrap().code, - gas_wanted: tx.result.as_ref().unwrap().gas_wanted, - gas_used: tx.result.as_ref().unwrap().gas_used, - } -} - -pub struct FirehoseMapper { - adapter: Arc>, - filter: Arc, -} - -#[async_trait] -impl BlockStreamMapper for FirehoseMapper { - fn decode_block( - &self, - output: Option<&[u8]>, - ) -> Result, BlockStreamError> { - let block = match output { - Some(block) => crate::Block::decode(block)?, - None => Err(anyhow::anyhow!( - "cosmos mapper is expected to always have a block" - ))?, - }; - - Ok(Some(block)) - } - - async fn block_with_triggers( - &self, - logger: &Logger, - block: crate::Block, - ) -> Result, BlockStreamError> { - self.adapter - .triggers_in_block(logger, block, self.filter.as_ref()) - .await - .map_err(BlockStreamError::from) - } - - async fn handle_substreams_block( - &self, - _logger: &Logger, - _clock: Clock, - _cursor: FirehoseCursor, - _block: Vec, - ) -> Result, BlockStreamError> { - unimplemented!() - } -} - -#[async_trait] -impl FirehoseMapperTrait for FirehoseMapper { - fn trigger_filter(&self) -> &TriggerFilter { - self.filter.as_ref() - } - - async fn to_block_stream_event( - &self, - logger: &Logger, - response: &firehose::Response, - ) -> Result, FirehoseError> { - let step = ForkStep::try_from(response.step).unwrap_or_else(|_| { - panic!( - "unknown step i32 value {}, maybe you forgot update & re-regenerate the protobuf definitions?", - response.step - ) - }); - - let any_block = response - .block - .as_ref() - .expect("block payload information should always be present"); - - // Right now, this is done in all cases but in reality, with how the BlockStreamEvent::Revert - // is defined right now, only block hash and block number is necessary. However, this information - // is not part of the actual bstream::BlockResponseV2 payload. As such, we need to decode the full - // block which is useless. - // - // Check about adding basic information about the block in the bstream::BlockResponseV2 or maybe - // define a slimmed down struct that would decode only a few fields and ignore all the rest. - // unwrap: Input cannot be None so output will be error or block. - let block = self - .decode_block(Some(any_block.value.as_ref())) - .map_err(Error::from)? - .unwrap(); - - match step { - ForkStep::StepNew => Ok(BlockStreamEvent::ProcessBlock( - self.block_with_triggers(logger, block) - .await - .map_err(Error::from)?, - FirehoseCursor::from(response.cursor.clone()), - )), - - ForkStep::StepUndo => { - let parent_ptr = block - .parent_ptr() - .map_err(FirehoseError::from)? - .expect("Genesis block should never be reverted"); - - Ok(BlockStreamEvent::Revert( - parent_ptr, - FirehoseCursor::from(response.cursor.clone()), - )) - } - - ForkStep::StepFinal => { - panic!( - "final step is not handled and should not be requested in the Firehose request" - ) - } - - ForkStep::StepUnset => { - panic!("unknown step should not happen in the Firehose response") - } - } - } - - async fn block_ptr_for_number( - &self, - logger: &Logger, - endpoint: &Arc, - number: BlockNumber, - ) -> Result { - endpoint - .block_ptr_for_number::(logger, number) - .await - } - - async fn final_block_ptr_for( - &self, - logger: &Logger, - endpoint: &Arc, - block: &codec::Block, - ) -> Result { - // Cosmos provides instant block finality. - self.block_ptr_for_number(logger, endpoint, block.number()) - .await - } -} - -#[cfg(test)] -mod test { - use graph::{ - blockchain::Trigger, - prelude::{ - slog::{o, Discard, Logger}, - tokio, - }, - }; - - use super::*; - - use codec::{ - Block, Event, Header, HeaderOnlyBlock, ResponseBeginBlock, ResponseDeliverTx, - ResponseEndBlock, TxResult, - }; - - #[tokio::test] - async fn test_trigger_filters() { - let adapter = TriggersAdapter {}; - let logger = Logger::root(Discard, o!()); - - let block_with_events = Block::test_with_event_types( - vec!["begin_event_1", "begin_event_2", "begin_event_3"], - vec!["tx_event_1", "tx_event_2", "tx_event_3"], - vec!["end_event_1", "end_event_2", "end_event_3"], - ); - - let header_only_block = HeaderOnlyBlock::from(&block_with_events); - - let cases = [ - ( - Block::test_new(), - TriggerFilter::test_new(false, &[]), - vec![], - ), - ( - Block::test_new(), - TriggerFilter::test_new(true, &[]), - vec![CosmosTrigger::Block(Arc::new(Block::test_new()))], - ), - ( - Block::test_new(), - TriggerFilter::test_new(false, &["event_1", "event_2", "event_3"]), - vec![], - ), - ( - block_with_events.clone(), - TriggerFilter::test_new(false, &["begin_event_3", "tx_event_3", "end_event_3"]), - vec![ - CosmosTrigger::with_event( - Event::test_with_type("begin_event_3"), - header_only_block.clone(), - None, - EventOrigin::BeginBlock, - ), - CosmosTrigger::with_event( - Event::test_with_type("tx_event_3"), - header_only_block.clone(), - Some(build_tx_context(&block_with_events.transactions[2])), - EventOrigin::DeliverTx, - ), - CosmosTrigger::with_event( - Event::test_with_type("end_event_3"), - header_only_block.clone(), - None, - EventOrigin::EndBlock, - ), - CosmosTrigger::with_transaction( - TxResult::test_with_event_type("tx_event_1"), - header_only_block.clone(), - ), - CosmosTrigger::with_transaction( - TxResult::test_with_event_type("tx_event_2"), - header_only_block.clone(), - ), - CosmosTrigger::with_transaction( - TxResult::test_with_event_type("tx_event_3"), - header_only_block.clone(), - ), - ], - ), - ( - block_with_events.clone(), - TriggerFilter::test_new(true, &["begin_event_3", "tx_event_2", "end_event_1"]), - vec![ - CosmosTrigger::Block(Arc::new(block_with_events.clone())), - CosmosTrigger::with_event( - Event::test_with_type("begin_event_3"), - header_only_block.clone(), - None, - EventOrigin::BeginBlock, - ), - CosmosTrigger::with_event( - Event::test_with_type("tx_event_2"), - header_only_block.clone(), - Some(build_tx_context(&block_with_events.transactions[1])), - EventOrigin::DeliverTx, - ), - CosmosTrigger::with_event( - Event::test_with_type("end_event_1"), - header_only_block.clone(), - None, - EventOrigin::EndBlock, - ), - CosmosTrigger::with_transaction( - TxResult::test_with_event_type("tx_event_1"), - header_only_block.clone(), - ), - CosmosTrigger::with_transaction( - TxResult::test_with_event_type("tx_event_2"), - header_only_block.clone(), - ), - CosmosTrigger::with_transaction( - TxResult::test_with_event_type("tx_event_3"), - header_only_block.clone(), - ), - ], - ), - ]; - - for (block, trigger_filter, expected_triggers) in cases { - let triggers = adapter - .triggers_in_block(&logger, block, &trigger_filter) - .await - .expect("failed to get triggers in block"); - - assert_eq!( - triggers.trigger_data.len(), - expected_triggers.len(), - "Expected trigger list to contain exactly {:?}, but it didn't: {:?}", - expected_triggers, - triggers.trigger_data - ); - - // they may not be in the same order - for trigger in expected_triggers { - assert!( - triggers.trigger_data.iter().any(|t| match t { - Trigger::Chain(t) => t == &trigger, - _ => false, - }), - "Expected trigger list to contain {:?}, but it only contains: {:?}", - trigger, - triggers.trigger_data - ); - } - } - } - - impl Block { - fn test_new() -> Block { - Block::test_with_event_types(vec![], vec![], vec![]) - } - - fn test_with_event_types( - begin_event_types: Vec<&str>, - tx_event_types: Vec<&str>, - end_event_types: Vec<&str>, - ) -> Block { - Block { - header: Some(Header { - version: None, - chain_id: "test".to_string(), - height: 1, - time: None, - last_block_id: None, - last_commit_hash: vec![], - data_hash: vec![], - validators_hash: vec![], - next_validators_hash: vec![], - consensus_hash: vec![], - app_hash: vec![], - last_results_hash: vec![], - evidence_hash: vec![], - proposer_address: vec![], - hash: vec![], - }), - evidence: None, - last_commit: None, - result_begin_block: Some(ResponseBeginBlock { - events: begin_event_types - .into_iter() - .map(Event::test_with_type) - .collect(), - }), - result_end_block: Some(ResponseEndBlock { - validator_updates: vec![], - consensus_param_updates: None, - events: end_event_types - .into_iter() - .map(Event::test_with_type) - .collect(), - }), - transactions: tx_event_types - .into_iter() - .map(TxResult::test_with_event_type) - .collect(), - validator_updates: vec![], - } - } - } - - impl Event { - fn test_with_type(event_type: &str) -> Event { - Event { - event_type: event_type.to_string(), - attributes: vec![], - } - } - } - - impl TxResult { - fn test_with_event_type(event_type: &str) -> TxResult { - TxResult { - height: 1, - index: 1, - tx: None, - result: Some(ResponseDeliverTx { - code: 1, - data: vec![], - log: "".to_string(), - info: "".to_string(), - gas_wanted: 1, - gas_used: 1, - codespace: "".to_string(), - events: vec![Event::test_with_type(event_type)], - }), - hash: vec![], - } - } - } -} diff --git a/chain/cosmos/src/codec.rs b/chain/cosmos/src/codec.rs deleted file mode 100644 index bdc05c1b820..00000000000 --- a/chain/cosmos/src/codec.rs +++ /dev/null @@ -1,198 +0,0 @@ -pub(crate) use crate::protobuf::pbcodec::*; - -use graph::blockchain::{Block as BlockchainBlock, BlockTime}; -use graph::{ - blockchain::BlockPtr, - prelude::{anyhow::anyhow, BlockNumber, Error}, -}; - -use std::convert::TryFrom; - -impl Block { - pub fn header(&self) -> Result<&Header, Error> { - self.header - .as_ref() - .ok_or_else(|| anyhow!("block data missing header field")) - } - - pub fn begin_block_events(&self) -> Result, Error> { - let events = self - .result_begin_block - .as_ref() - .ok_or_else(|| anyhow!("block data missing result_begin_block field"))? - .events - .iter(); - - Ok(events) - } - - pub fn end_block_events(&self) -> Result, Error> { - let events = self - .result_end_block - .as_ref() - .ok_or_else(|| anyhow!("block data missing result_end_block field"))? - .events - .iter(); - - Ok(events) - } - - pub fn transactions(&self) -> impl Iterator { - self.transactions.iter() - } - - pub fn parent_ptr(&self) -> Result, Error> { - let header = self.header()?; - - Ok(header - .last_block_id - .as_ref() - .map(|last_block_id| BlockPtr::from((last_block_id.hash.clone(), header.height - 1)))) - } -} - -impl TryFrom for BlockPtr { - type Error = Error; - - fn try_from(b: Block) -> Result { - BlockPtr::try_from(&b) - } -} - -impl<'a> TryFrom<&'a Block> for BlockPtr { - type Error = Error; - - fn try_from(b: &'a Block) -> Result { - let header = b.header()?; - Ok(BlockPtr::from((header.hash.clone(), header.height))) - } -} - -impl BlockchainBlock for Block { - fn number(&self) -> i32 { - BlockNumber::try_from(self.header().unwrap().height).unwrap() - } - - fn ptr(&self) -> BlockPtr { - BlockPtr::try_from(self).unwrap() - } - - fn parent_ptr(&self) -> Option { - self.parent_ptr().unwrap() - } - - fn timestamp(&self) -> BlockTime { - let time = self.header().unwrap().time.as_ref().unwrap(); - BlockTime::since_epoch(time.seconds, time.nanos as u32) - } -} - -impl HeaderOnlyBlock { - pub fn header(&self) -> Result<&Header, Error> { - self.header - .as_ref() - .ok_or_else(|| anyhow!("block data missing header field")) - } - - pub fn parent_ptr(&self) -> Result, Error> { - let header = self.header()?; - - Ok(header - .last_block_id - .as_ref() - .map(|last_block_id| BlockPtr::from((last_block_id.hash.clone(), header.height - 1)))) - } -} - -impl From<&Block> for HeaderOnlyBlock { - fn from(b: &Block) -> HeaderOnlyBlock { - HeaderOnlyBlock { - header: b.header.clone(), - } - } -} - -impl TryFrom for BlockPtr { - type Error = Error; - - fn try_from(b: HeaderOnlyBlock) -> Result { - BlockPtr::try_from(&b) - } -} - -impl<'a> TryFrom<&'a HeaderOnlyBlock> for BlockPtr { - type Error = Error; - - fn try_from(b: &'a HeaderOnlyBlock) -> Result { - let header = b.header()?; - - Ok(BlockPtr::from((header.hash.clone(), header.height))) - } -} - -impl BlockchainBlock for HeaderOnlyBlock { - fn number(&self) -> i32 { - BlockNumber::try_from(self.header().unwrap().height).unwrap() - } - - fn ptr(&self) -> BlockPtr { - BlockPtr::try_from(self).unwrap() - } - - fn parent_ptr(&self) -> Option { - self.parent_ptr().unwrap() - } - - fn timestamp(&self) -> BlockTime { - let time = self.header().unwrap().time.as_ref().unwrap(); - BlockTime::since_epoch(time.seconds, time.nanos as u32) - } -} - -impl EventData { - pub fn event(&self) -> Result<&Event, Error> { - self.event - .as_ref() - .ok_or_else(|| anyhow!("event data missing event field")) - } - pub fn block(&self) -> Result<&HeaderOnlyBlock, Error> { - self.block - .as_ref() - .ok_or_else(|| anyhow!("event data missing block field")) - } -} - -impl TransactionData { - pub fn tx_result(&self) -> Result<&TxResult, Error> { - self.tx - .as_ref() - .ok_or_else(|| anyhow!("transaction data missing tx field")) - } - - pub fn response_deliver_tx(&self) -> Result<&ResponseDeliverTx, Error> { - self.tx_result()? - .result - .as_ref() - .ok_or_else(|| anyhow!("transaction data missing result field")) - } - - pub fn block(&self) -> Result<&HeaderOnlyBlock, Error> { - self.block - .as_ref() - .ok_or_else(|| anyhow!("transaction data missing block field")) - } -} - -impl MessageData { - pub fn message(&self) -> Result<&prost_types::Any, Error> { - self.message - .as_ref() - .ok_or_else(|| anyhow!("message data missing message field")) - } - - pub fn block(&self) -> Result<&HeaderOnlyBlock, Error> { - self.block - .as_ref() - .ok_or_else(|| anyhow!("message data missing block field")) - } -} diff --git a/chain/cosmos/src/data_source.rs b/chain/cosmos/src/data_source.rs deleted file mode 100644 index f09448ecbe8..00000000000 --- a/chain/cosmos/src/data_source.rs +++ /dev/null @@ -1,726 +0,0 @@ -use std::collections::{HashMap, HashSet}; -use std::sync::Arc; - -use anyhow::{Context, Error, Result}; - -use graph::components::subgraph::InstanceDSTemplateInfo; -use graph::{ - blockchain::{self, Block, Blockchain, TriggerWithHandler}, - components::store::StoredDynamicDataSource, - data::subgraph::DataSourceContext, - derive::CheapClone, - prelude::{ - anyhow, async_trait, BlockNumber, CheapClone, Deserialize, Link, LinkResolver, Logger, - }, -}; - -use crate::chain::Chain; -use crate::codec; -use crate::trigger::CosmosTrigger; - -pub const COSMOS_KIND: &str = "cosmos"; -const BLOCK_HANDLER_KIND: &str = "block"; -const EVENT_HANDLER_KIND: &str = "event"; -const TRANSACTION_HANDLER_KIND: &str = "transaction"; -const MESSAGE_HANDLER_KIND: &str = "message"; - -const DYNAMIC_DATA_SOURCE_ERROR: &str = "Cosmos subgraphs do not support dynamic data sources"; -const TEMPLATE_ERROR: &str = "Cosmos subgraphs do not support templates"; - -/// Runtime representation of a data source. -// Note: Not great for memory usage that this needs to be `Clone`, considering how there may be tens -// of thousands of data sources in memory at once. -#[derive(Clone, Debug)] -pub struct DataSource { - pub kind: String, - pub network: Option, - pub name: String, - pub source: Source, - pub mapping: Mapping, - pub context: Arc>, - pub creation_block: Option, -} - -impl blockchain::DataSource for DataSource { - fn from_template_info( - _info: InstanceDSTemplateInfo, - _template: &graph::data_source::DataSourceTemplate, - ) -> Result { - Err(anyhow!(TEMPLATE_ERROR)) - } - - fn address(&self) -> Option<&[u8]> { - None - } - - fn start_block(&self) -> BlockNumber { - self.source.start_block - } - - fn handler_kinds(&self) -> HashSet<&str> { - let mut kinds = HashSet::new(); - - let Mapping { - block_handlers, - event_handlers, - transaction_handlers, - message_handlers, - .. - } = &self.mapping; - - if !block_handlers.is_empty() { - kinds.insert(BLOCK_HANDLER_KIND); - } - - if !event_handlers.is_empty() { - kinds.insert(EVENT_HANDLER_KIND); - } - - if !transaction_handlers.is_empty() { - kinds.insert(TRANSACTION_HANDLER_KIND); - } - - if !message_handlers.is_empty() { - kinds.insert(MESSAGE_HANDLER_KIND); - } - - kinds - } - - fn end_block(&self) -> Option { - self.source.end_block - } - - fn match_and_decode( - &self, - trigger: &::TriggerData, - block: &Arc<::Block>, - _logger: &Logger, - ) -> Result>> { - if self.source.start_block > block.number() { - return Ok(None); - } - - let handler = match trigger { - CosmosTrigger::Block(_) => match self.handler_for_block() { - Some(handler) => handler.handler, - None => return Ok(None), - }, - - CosmosTrigger::Event { event_data, origin } => { - match self.handler_for_event(event_data.event()?, *origin) { - Some(handler) => handler.handler, - None => return Ok(None), - } - } - - CosmosTrigger::Transaction(_) => match self.handler_for_transaction() { - Some(handler) => handler.handler, - None => return Ok(None), - }, - - CosmosTrigger::Message(message_data) => { - match self.handler_for_message(message_data.message()?) { - Some(handler) => handler.handler, - None => return Ok(None), - } - } - }; - - Ok(Some(TriggerWithHandler::::new( - trigger.cheap_clone(), - handler, - block.ptr(), - block.timestamp(), - ))) - } - - fn name(&self) -> &str { - &self.name - } - - fn kind(&self) -> &str { - &self.kind - } - - fn network(&self) -> Option<&str> { - self.network.as_deref() - } - - fn context(&self) -> Arc> { - self.context.cheap_clone() - } - - fn creation_block(&self) -> Option { - self.creation_block - } - - fn is_duplicate_of(&self, other: &Self) -> bool { - let DataSource { - kind, - network, - name, - source, - mapping, - context, - - // The creation block is ignored for detection duplicate data sources. - // Contract ABI equality is implicit in `source` and `mapping.abis` equality. - creation_block: _, - } = self; - - // mapping_request_sender, host_metrics, and (most of) host_exports are operational structs - // used at runtime but not needed to define uniqueness; each runtime host should be for a - // unique data source. - kind == &other.kind - && network == &other.network - && name == &other.name - && source == &other.source - && mapping.block_handlers == other.mapping.block_handlers - && mapping.event_handlers == other.mapping.event_handlers - && mapping.transaction_handlers == other.mapping.transaction_handlers - && mapping.message_handlers == other.mapping.message_handlers - && context == &other.context - } - - fn as_stored_dynamic_data_source(&self) -> StoredDynamicDataSource { - unimplemented!("{}", DYNAMIC_DATA_SOURCE_ERROR); - } - - fn from_stored_dynamic_data_source( - _template: &DataSourceTemplate, - _stored: StoredDynamicDataSource, - ) -> Result { - Err(anyhow!(DYNAMIC_DATA_SOURCE_ERROR)) - } - - fn validate(&self, _: &semver::Version) -> Vec { - let mut errors = Vec::new(); - - if self.kind != COSMOS_KIND { - errors.push(anyhow!( - "data source has invalid `kind`, expected {} but found {}", - COSMOS_KIND, - self.kind - )) - } - - // Ensure there is only one block handler - if self.mapping.block_handlers.len() > 1 { - errors.push(anyhow!("data source has duplicated block handlers")); - } - - // Ensure there is only one transaction handler - if self.mapping.transaction_handlers.len() > 1 { - errors.push(anyhow!("data source has duplicated transaction handlers")); - } - - // Ensure that each event type + origin filter combination has only one handler - - // group handler origin filters by event type - let mut event_types = HashMap::with_capacity(self.mapping.event_handlers.len()); - for event_handler in self.mapping.event_handlers.iter() { - let origins = event_types - .entry(&event_handler.event) - // 3 is the maximum number of valid handlers for an event type (1 for each origin) - .or_insert(HashSet::with_capacity(3)); - - // insert returns false if value was already in the set - if !origins.insert(event_handler.origin) { - errors.push(multiple_origin_err( - &event_handler.event, - event_handler.origin, - )) - } - } - - // Ensure each event type either has: - // 1 handler with no origin filter - // OR - // 1 or more handlers with origin filter - for (event_type, origins) in event_types.iter() { - if origins.len() > 1 && !origins.iter().all(Option::is_some) { - errors.push(combined_origins_err(event_type)) - } - } - - // Ensure each message handlers is unique - let mut message_type_urls = HashSet::with_capacity(self.mapping.message_handlers.len()); - for message_handler in self.mapping.message_handlers.iter() { - if !message_type_urls.insert(message_handler.message.clone()) { - errors.push(duplicate_url_type(&message_handler.message)) - } - } - - errors - } - - fn api_version(&self) -> semver::Version { - self.mapping.api_version.clone() - } - - fn runtime(&self) -> Option>> { - Some(self.mapping.runtime.cheap_clone()) - } -} - -impl DataSource { - fn from_manifest( - kind: String, - network: Option, - name: String, - source: Source, - mapping: Mapping, - context: Option, - ) -> Result { - // Data sources in the manifest are created "before genesis" so they have no creation block. - let creation_block = None; - - Ok(DataSource { - kind, - network, - name, - source, - mapping, - context: Arc::new(context), - creation_block, - }) - } - - fn handler_for_block(&self) -> Option { - self.mapping.block_handlers.first().cloned() - } - - fn handler_for_transaction(&self) -> Option { - self.mapping.transaction_handlers.first().cloned() - } - - fn handler_for_message(&self, message: &::prost_types::Any) -> Option { - self.mapping - .message_handlers - .iter() - .find(|handler| handler.message == message.type_url) - .cloned() - } - - fn handler_for_event( - &self, - event: &codec::Event, - event_origin: EventOrigin, - ) -> Option { - self.mapping - .event_handlers - .iter() - .find(|handler| { - let event_type_matches = event.event_type == handler.event; - - if let Some(handler_origin) = handler.origin { - event_type_matches && event_origin == handler_origin - } else { - event_type_matches - } - }) - .cloned() - } - - pub(crate) fn has_block_handler(&self) -> bool { - !self.mapping.block_handlers.is_empty() - } - - /// Return an iterator over all event types from event handlers. - pub(crate) fn events(&self) -> impl Iterator { - self.mapping - .event_handlers - .iter() - .map(|handler| handler.event.as_str()) - } -} - -#[derive(Clone, Debug, Eq, PartialEq, Deserialize)] -pub struct UnresolvedDataSource { - pub kind: String, - pub network: Option, - pub name: String, - pub source: Source, - pub mapping: UnresolvedMapping, - pub context: Option, -} - -#[async_trait] -impl blockchain::UnresolvedDataSource for UnresolvedDataSource { - async fn resolve( - self, - resolver: &Arc, - logger: &Logger, - _manifest_idx: u32, - ) -> Result { - let UnresolvedDataSource { - kind, - network, - name, - source, - mapping, - context, - } = self; - - let mapping = mapping.resolve(resolver, logger).await.with_context(|| { - format!( - "failed to resolve data source {} with source {}", - name, source.start_block - ) - })?; - - DataSource::from_manifest(kind, network, name, source, mapping, context) - } -} - -#[derive(Clone, Debug, Default, Hash, Eq, PartialEq, Deserialize)] -pub struct BaseDataSourceTemplate { - pub kind: String, - pub network: Option, - pub name: String, - pub mapping: M, -} - -pub type UnresolvedDataSourceTemplate = BaseDataSourceTemplate; -pub type DataSourceTemplate = BaseDataSourceTemplate; - -#[async_trait] -impl blockchain::UnresolvedDataSourceTemplate for UnresolvedDataSourceTemplate { - async fn resolve( - self, - _resolver: &Arc, - _logger: &Logger, - _manifest_idx: u32, - ) -> Result { - Err(anyhow!(TEMPLATE_ERROR)) - } -} - -impl blockchain::DataSourceTemplate for DataSourceTemplate { - fn name(&self) -> &str { - unimplemented!("{}", TEMPLATE_ERROR); - } - - fn api_version(&self) -> semver::Version { - unimplemented!("{}", TEMPLATE_ERROR); - } - - fn runtime(&self) -> Option>> { - unimplemented!("{}", TEMPLATE_ERROR); - } - - fn manifest_idx(&self) -> u32 { - unimplemented!("{}", TEMPLATE_ERROR); - } - - fn kind(&self) -> &str { - &self.kind - } -} - -#[derive(Clone, Debug, Default, Hash, Eq, PartialEq, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct UnresolvedMapping { - pub api_version: String, - pub language: String, - pub entities: Vec, - #[serde(default)] - pub block_handlers: Vec, - #[serde(default)] - pub event_handlers: Vec, - #[serde(default)] - pub transaction_handlers: Vec, - #[serde(default)] - pub message_handlers: Vec, - pub file: Link, -} - -impl UnresolvedMapping { - pub async fn resolve( - self, - resolver: &Arc, - logger: &Logger, - ) -> Result { - let UnresolvedMapping { - api_version, - language, - entities, - block_handlers, - event_handlers, - transaction_handlers, - message_handlers, - file: link, - } = self; - - let api_version = semver::Version::parse(&api_version)?; - - let module_bytes = resolver - .cat(logger, &link) - .await - .with_context(|| format!("failed to resolve mapping {}", link.link))?; - - Ok(Mapping { - api_version, - language, - entities, - block_handlers: block_handlers.clone(), - event_handlers: event_handlers.clone(), - transaction_handlers: transaction_handlers.clone(), - message_handlers: message_handlers.clone(), - runtime: Arc::new(module_bytes), - link, - }) - } -} - -#[derive(Clone, Debug)] -pub struct Mapping { - pub api_version: semver::Version, - pub language: String, - pub entities: Vec, - pub block_handlers: Vec, - pub event_handlers: Vec, - pub transaction_handlers: Vec, - pub message_handlers: Vec, - pub runtime: Arc>, - pub link: Link, -} - -#[derive(Clone, Debug, Hash, Eq, PartialEq, Deserialize)] -pub struct MappingBlockHandler { - pub handler: String, -} - -#[derive(Clone, Debug, Hash, Eq, PartialEq, Deserialize)] -pub struct MappingEventHandler { - pub event: String, - pub origin: Option, - pub handler: String, -} - -#[derive(Clone, Debug, Hash, Eq, PartialEq, Deserialize)] -pub struct MappingTransactionHandler { - pub handler: String, -} - -#[derive(Clone, Debug, Hash, Eq, PartialEq, Deserialize)] -pub struct MappingMessageHandler { - pub message: String, - pub handler: String, -} - -#[derive(Clone, Debug, Hash, Eq, PartialEq, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct Source { - #[serde(default)] - pub start_block: BlockNumber, - pub(crate) end_block: Option, -} - -#[derive(Clone, Copy, CheapClone, Debug, Hash, Eq, PartialEq, Deserialize)] -pub enum EventOrigin { - BeginBlock, - DeliverTx, - EndBlock, -} - -fn multiple_origin_err(event_type: &str, origin: Option) -> Error { - let origin_err_name = match origin { - Some(origin) => format!("{:?}", origin), - None => "no".to_string(), - }; - - anyhow!( - "data source has multiple {} event handlers with {} origin", - event_type, - origin_err_name, - ) -} - -fn combined_origins_err(event_type: &str) -> Error { - anyhow!( - "data source has combined origin and no-origin {} event handlers", - event_type - ) -} - -fn duplicate_url_type(message: &str) -> Error { - anyhow!( - "data source has more than one message handler for message {} ", - message - ) -} - -#[cfg(test)] -mod tests { - use super::*; - - use graph::{blockchain::DataSource as _, data::subgraph::LATEST_VERSION}; - - #[test] - fn test_event_handlers_origin_validation() { - let cases = [ - ( - DataSource::with_event_handlers(vec![ - MappingEventHandler::with_origin("event_1", None), - MappingEventHandler::with_origin("event_2", None), - MappingEventHandler::with_origin("event_3", None), - ]), - vec![], - ), - ( - DataSource::with_event_handlers(vec![ - MappingEventHandler::with_origin("event_1", Some(EventOrigin::BeginBlock)), - MappingEventHandler::with_origin("event_2", Some(EventOrigin::BeginBlock)), - MappingEventHandler::with_origin("event_1", Some(EventOrigin::DeliverTx)), - MappingEventHandler::with_origin("event_1", Some(EventOrigin::EndBlock)), - MappingEventHandler::with_origin("event_2", Some(EventOrigin::DeliverTx)), - MappingEventHandler::with_origin("event_2", Some(EventOrigin::EndBlock)), - ]), - vec![], - ), - ( - DataSource::with_event_handlers(vec![ - MappingEventHandler::with_origin("event_1", None), - MappingEventHandler::with_origin("event_1", None), - MappingEventHandler::with_origin("event_2", None), - MappingEventHandler::with_origin("event_2", Some(EventOrigin::BeginBlock)), - MappingEventHandler::with_origin("event_3", Some(EventOrigin::EndBlock)), - MappingEventHandler::with_origin("event_3", Some(EventOrigin::EndBlock)), - ]), - vec![ - multiple_origin_err("event_1", None), - combined_origins_err("event_2"), - multiple_origin_err("event_3", Some(EventOrigin::EndBlock)), - ], - ), - ]; - - for (data_source, errors) in &cases { - let validation_errors = data_source.validate(&LATEST_VERSION); - - assert_eq!(errors.len(), validation_errors.len()); - - for error in errors.iter() { - assert!( - validation_errors - .iter() - .any(|validation_error| validation_error.to_string() == error.to_string()), - r#"expected "{}" to be in validation errors, but it wasn't"#, - error - ); - } - } - } - - #[test] - fn test_message_handlers_duplicate() { - let cases = [ - ( - DataSource::with_message_handlers(vec![ - MappingMessageHandler { - handler: "handler".to_string(), - message: "message_0".to_string(), - }, - MappingMessageHandler { - handler: "handler".to_string(), - message: "message_1".to_string(), - }, - ]), - vec![], - ), - ( - DataSource::with_message_handlers(vec![ - MappingMessageHandler { - handler: "handler".to_string(), - message: "message_0".to_string(), - }, - MappingMessageHandler { - handler: "handler".to_string(), - message: "message_0".to_string(), - }, - ]), - vec![duplicate_url_type("message_0")], - ), - ]; - - for (data_source, errors) in &cases { - let validation_errors = data_source.validate(&LATEST_VERSION); - - assert_eq!(errors.len(), validation_errors.len()); - - for error in errors.iter() { - assert!( - validation_errors - .iter() - .any(|validation_error| validation_error.to_string() == error.to_string()), - r#"expected "{}" to be in validation errors, but it wasn't"#, - error - ); - } - } - } - - impl DataSource { - fn with_event_handlers(event_handlers: Vec) -> DataSource { - DataSource { - kind: "cosmos".to_string(), - network: None, - name: "Test".to_string(), - source: Source { - start_block: 1, - end_block: None, - }, - mapping: Mapping { - api_version: semver::Version::new(0, 0, 0), - language: "".to_string(), - entities: vec![], - block_handlers: vec![], - event_handlers, - transaction_handlers: vec![], - message_handlers: vec![], - runtime: Arc::new(vec![]), - link: "test".to_string().into(), - }, - context: Arc::new(None), - creation_block: None, - } - } - - fn with_message_handlers(message_handlers: Vec) -> DataSource { - DataSource { - kind: "cosmos".to_string(), - network: None, - name: "Test".to_string(), - source: Source { - start_block: 1, - end_block: None, - }, - mapping: Mapping { - api_version: semver::Version::new(0, 0, 0), - language: "".to_string(), - entities: vec![], - block_handlers: vec![], - event_handlers: vec![], - transaction_handlers: vec![], - message_handlers, - runtime: Arc::new(vec![]), - link: "test".to_string().into(), - }, - context: Arc::new(None), - creation_block: None, - } - } - } - - impl MappingEventHandler { - fn with_origin(event_type: &str, origin: Option) -> MappingEventHandler { - MappingEventHandler { - event: event_type.to_string(), - origin, - handler: "handler".to_string(), - } - } - } -} diff --git a/chain/cosmos/src/lib.rs b/chain/cosmos/src/lib.rs deleted file mode 100644 index 5d0bf16d050..00000000000 --- a/chain/cosmos/src/lib.rs +++ /dev/null @@ -1,16 +0,0 @@ -mod adapter; -pub mod chain; -pub mod codec; -mod data_source; -mod protobuf; -pub mod runtime; -mod trigger; - -// ETHDEP: These concrete types should probably not be exposed. -pub use data_source::{DataSource, DataSourceTemplate}; - -pub use crate::adapter::TriggerFilter; -pub use crate::chain::Chain; - -pub use protobuf::pbcodec; -pub use protobuf::pbcodec::Block; diff --git a/chain/cosmos/src/protobuf/.gitignore b/chain/cosmos/src/protobuf/.gitignore deleted file mode 100644 index 96786948080..00000000000 --- a/chain/cosmos/src/protobuf/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -/google.protobuf.rs -/gogoproto.rs -/cosmos_proto.rs -/firehose.rs diff --git a/chain/cosmos/src/protobuf/mod.rs b/chain/cosmos/src/protobuf/mod.rs deleted file mode 100644 index f12336b2f4f..00000000000 --- a/chain/cosmos/src/protobuf/mod.rs +++ /dev/null @@ -1,8 +0,0 @@ -#[rustfmt::skip] -#[path = "sf.cosmos.r#type.v1.rs"] -pub mod pbcodec; - -pub use graph_runtime_wasm::asc_abi::class::{Array, Uint8Array}; - -pub use crate::runtime::abi::*; -pub use pbcodec::*; diff --git a/chain/cosmos/src/protobuf/sf.cosmos.r#type.v1.rs b/chain/cosmos/src/protobuf/sf.cosmos.r#type.v1.rs deleted file mode 100644 index a3938e2c9c1..00000000000 --- a/chain/cosmos/src/protobuf/sf.cosmos.r#type.v1.rs +++ /dev/null @@ -1,839 +0,0 @@ -// This file is @generated by prost-build. -#[graph_runtime_derive::generate_asc_type( - __required__{header:Header, - result_begin_block:ResponseBeginBlock, - result_end_block:ResponseEndBlock} -)] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type( - __required__{header:Header, - result_begin_block:ResponseBeginBlock, - result_end_block:ResponseEndBlock} -)] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Block { - #[prost(message, optional, tag = "1")] - pub header: ::core::option::Option
, - #[prost(message, optional, tag = "2")] - pub evidence: ::core::option::Option, - #[prost(message, optional, tag = "3")] - pub last_commit: ::core::option::Option, - #[prost(message, optional, tag = "4")] - pub result_begin_block: ::core::option::Option, - #[prost(message, optional, tag = "5")] - pub result_end_block: ::core::option::Option, - #[prost(message, repeated, tag = "7")] - pub transactions: ::prost::alloc::vec::Vec, - #[prost(message, repeated, tag = "8")] - pub validator_updates: ::prost::alloc::vec::Vec, -} -/// HeaderOnlyBlock is a standard \[Block\] structure where all other fields are -/// removed so that hydrating that object from a \[Block\] bytes payload will -/// drastically reduce allocated memory required to hold the full block. -/// -/// This can be used to unpack a \[Block\] when only the \[Header\] information -/// is required and greatly reduce required memory. -#[graph_runtime_derive::generate_asc_type(__required__{header:Header})] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type(__required__{header:Header})] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct HeaderOnlyBlock { - #[prost(message, optional, tag = "1")] - pub header: ::core::option::Option
, -} -#[graph_runtime_derive::generate_asc_type( - __required__{event:Event, - block:HeaderOnlyBlock} -)] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type( - __required__{event:Event, - block:HeaderOnlyBlock} -)] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct EventData { - #[prost(message, optional, tag = "1")] - pub event: ::core::option::Option, - #[prost(message, optional, tag = "2")] - pub block: ::core::option::Option, - #[prost(message, optional, tag = "3")] - pub tx: ::core::option::Option, -} -#[graph_runtime_derive::generate_asc_type( - __required__{tx:TxResult, - block:HeaderOnlyBlock} -)] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type( - __required__{tx:TxResult, - block:HeaderOnlyBlock} -)] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct TransactionData { - #[prost(message, optional, tag = "1")] - pub tx: ::core::option::Option, - #[prost(message, optional, tag = "2")] - pub block: ::core::option::Option, -} -#[graph_runtime_derive::generate_asc_type( - __required__{message:Any, - block:HeaderOnlyBlock, - tx:TransactionContext} -)] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type( - __required__{message:Any, - block:HeaderOnlyBlock, - tx:TransactionContext} -)] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct MessageData { - #[prost(message, optional, tag = "1")] - pub message: ::core::option::Option<::prost_types::Any>, - #[prost(message, optional, tag = "2")] - pub block: ::core::option::Option, - #[prost(message, optional, tag = "3")] - pub tx: ::core::option::Option, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct TransactionContext { - #[prost(bytes = "vec", tag = "1")] - pub hash: ::prost::alloc::vec::Vec, - #[prost(uint32, tag = "2")] - pub index: u32, - #[prost(uint32, tag = "3")] - pub code: u32, - #[prost(int64, tag = "4")] - pub gas_wanted: i64, - #[prost(int64, tag = "5")] - pub gas_used: i64, -} -#[graph_runtime_derive::generate_asc_type(__required__{last_block_id:BlockID})] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type(__required__{last_block_id:BlockID})] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Header { - #[prost(message, optional, tag = "1")] - pub version: ::core::option::Option, - #[prost(string, tag = "2")] - pub chain_id: ::prost::alloc::string::String, - #[prost(uint64, tag = "3")] - pub height: u64, - #[prost(message, optional, tag = "4")] - pub time: ::core::option::Option, - #[prost(message, optional, tag = "5")] - pub last_block_id: ::core::option::Option, - #[prost(bytes = "vec", tag = "6")] - pub last_commit_hash: ::prost::alloc::vec::Vec, - #[prost(bytes = "vec", tag = "7")] - pub data_hash: ::prost::alloc::vec::Vec, - #[prost(bytes = "vec", tag = "8")] - pub validators_hash: ::prost::alloc::vec::Vec, - #[prost(bytes = "vec", tag = "9")] - pub next_validators_hash: ::prost::alloc::vec::Vec, - #[prost(bytes = "vec", tag = "10")] - pub consensus_hash: ::prost::alloc::vec::Vec, - #[prost(bytes = "vec", tag = "11")] - pub app_hash: ::prost::alloc::vec::Vec, - #[prost(bytes = "vec", tag = "12")] - pub last_results_hash: ::prost::alloc::vec::Vec, - #[prost(bytes = "vec", tag = "13")] - pub evidence_hash: ::prost::alloc::vec::Vec, - #[prost(bytes = "vec", tag = "14")] - pub proposer_address: ::prost::alloc::vec::Vec, - #[prost(bytes = "vec", tag = "15")] - pub hash: ::prost::alloc::vec::Vec, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Consensus { - #[prost(uint64, tag = "1")] - pub block: u64, - #[prost(uint64, tag = "2")] - pub app: u64, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Timestamp { - #[prost(int64, tag = "1")] - pub seconds: i64, - #[prost(int32, tag = "2")] - pub nanos: i32, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct BlockId { - #[prost(bytes = "vec", tag = "1")] - pub hash: ::prost::alloc::vec::Vec, - #[prost(message, optional, tag = "2")] - pub part_set_header: ::core::option::Option, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct PartSetHeader { - #[prost(uint32, tag = "1")] - pub total: u32, - #[prost(bytes = "vec", tag = "2")] - pub hash: ::prost::alloc::vec::Vec, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct EvidenceList { - #[prost(message, repeated, tag = "1")] - pub evidence: ::prost::alloc::vec::Vec, -} -#[graph_runtime_derive::generate_asc_type( - sum{duplicate_vote_evidence:DuplicateVoteEvidence, - light_client_attack_evidence:LightClientAttackEvidence} -)] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type( - sum{duplicate_vote_evidence:DuplicateVoteEvidence, - light_client_attack_evidence:LightClientAttackEvidence} -)] -#[graph_runtime_derive::generate_array_type(Cosmos)] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Evidence { - #[prost(oneof = "evidence::Sum", tags = "1, 2")] - pub sum: ::core::option::Option, -} -/// Nested message and enum types in `Evidence`. -pub mod evidence { - #[allow(clippy::derive_partial_eq_without_eq)] - #[derive(Clone, PartialEq, ::prost::Oneof)] - pub enum Sum { - #[prost(message, tag = "1")] - DuplicateVoteEvidence(super::DuplicateVoteEvidence), - #[prost(message, tag = "2")] - LightClientAttackEvidence(super::LightClientAttackEvidence), - } -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct DuplicateVoteEvidence { - #[prost(message, optional, tag = "1")] - pub vote_a: ::core::option::Option, - #[prost(message, optional, tag = "2")] - pub vote_b: ::core::option::Option, - #[prost(int64, tag = "3")] - pub total_voting_power: i64, - #[prost(int64, tag = "4")] - pub validator_power: i64, - #[prost(message, optional, tag = "5")] - pub timestamp: ::core::option::Option, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct EventVote { - #[prost(enumeration = "SignedMsgType", tag = "1")] - pub event_vote_type: i32, - #[prost(uint64, tag = "2")] - pub height: u64, - #[prost(int32, tag = "3")] - pub round: i32, - #[prost(message, optional, tag = "4")] - pub block_id: ::core::option::Option, - #[prost(message, optional, tag = "5")] - pub timestamp: ::core::option::Option, - #[prost(bytes = "vec", tag = "6")] - pub validator_address: ::prost::alloc::vec::Vec, - #[prost(int32, tag = "7")] - pub validator_index: i32, - #[prost(bytes = "vec", tag = "8")] - pub signature: ::prost::alloc::vec::Vec, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct LightClientAttackEvidence { - #[prost(message, optional, tag = "1")] - pub conflicting_block: ::core::option::Option, - #[prost(int64, tag = "2")] - pub common_height: i64, - #[prost(message, repeated, tag = "3")] - pub byzantine_validators: ::prost::alloc::vec::Vec, - #[prost(int64, tag = "4")] - pub total_voting_power: i64, - #[prost(message, optional, tag = "5")] - pub timestamp: ::core::option::Option, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct LightBlock { - #[prost(message, optional, tag = "1")] - pub signed_header: ::core::option::Option, - #[prost(message, optional, tag = "2")] - pub validator_set: ::core::option::Option, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct SignedHeader { - #[prost(message, optional, tag = "1")] - pub header: ::core::option::Option
, - #[prost(message, optional, tag = "2")] - pub commit: ::core::option::Option, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Commit { - #[prost(int64, tag = "1")] - pub height: i64, - #[prost(int32, tag = "2")] - pub round: i32, - #[prost(message, optional, tag = "3")] - pub block_id: ::core::option::Option, - #[prost(message, repeated, tag = "4")] - pub signatures: ::prost::alloc::vec::Vec, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[graph_runtime_derive::generate_array_type(Cosmos)] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct CommitSig { - #[prost(enumeration = "BlockIdFlag", tag = "1")] - pub block_id_flag: i32, - #[prost(bytes = "vec", tag = "2")] - pub validator_address: ::prost::alloc::vec::Vec, - #[prost(message, optional, tag = "3")] - pub timestamp: ::core::option::Option, - #[prost(bytes = "vec", tag = "4")] - pub signature: ::prost::alloc::vec::Vec, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ValidatorSet { - #[prost(message, repeated, tag = "1")] - pub validators: ::prost::alloc::vec::Vec, - #[prost(message, optional, tag = "2")] - pub proposer: ::core::option::Option, - #[prost(int64, tag = "3")] - pub total_voting_power: i64, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[graph_runtime_derive::generate_array_type(Cosmos)] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Validator { - #[prost(bytes = "vec", tag = "1")] - pub address: ::prost::alloc::vec::Vec, - #[prost(message, optional, tag = "2")] - pub pub_key: ::core::option::Option, - #[prost(int64, tag = "3")] - pub voting_power: i64, - #[prost(int64, tag = "4")] - pub proposer_priority: i64, -} -#[graph_runtime_derive::generate_asc_type(sum{ed25519:Vec, secp256k1:Vec})] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type(sum{ed25519:Vec, secp256k1:Vec})] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct PublicKey { - #[prost(oneof = "public_key::Sum", tags = "1, 2")] - pub sum: ::core::option::Option, -} -/// Nested message and enum types in `PublicKey`. -pub mod public_key { - #[allow(clippy::derive_partial_eq_without_eq)] - #[derive(Clone, PartialEq, ::prost::Oneof)] - pub enum Sum { - #[prost(bytes, tag = "1")] - Ed25519(::prost::alloc::vec::Vec), - #[prost(bytes, tag = "2")] - Secp256k1(::prost::alloc::vec::Vec), - } -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ResponseBeginBlock { - #[prost(message, repeated, tag = "1")] - pub events: ::prost::alloc::vec::Vec, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[graph_runtime_derive::generate_array_type(Cosmos)] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Event { - #[prost(string, tag = "1")] - pub event_type: ::prost::alloc::string::String, - #[prost(message, repeated, tag = "2")] - pub attributes: ::prost::alloc::vec::Vec, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[graph_runtime_derive::generate_array_type(Cosmos)] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct EventAttribute { - #[prost(string, tag = "1")] - pub key: ::prost::alloc::string::String, - #[prost(string, tag = "2")] - pub value: ::prost::alloc::string::String, - #[prost(bool, tag = "3")] - pub index: bool, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ResponseEndBlock { - #[prost(message, repeated, tag = "1")] - pub validator_updates: ::prost::alloc::vec::Vec, - #[prost(message, optional, tag = "2")] - pub consensus_param_updates: ::core::option::Option, - #[prost(message, repeated, tag = "3")] - pub events: ::prost::alloc::vec::Vec, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[graph_runtime_derive::generate_array_type(Cosmos)] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ValidatorUpdate { - #[prost(bytes = "vec", tag = "1")] - pub address: ::prost::alloc::vec::Vec, - #[prost(message, optional, tag = "2")] - pub pub_key: ::core::option::Option, - #[prost(int64, tag = "3")] - pub power: i64, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ConsensusParams { - #[prost(message, optional, tag = "1")] - pub block: ::core::option::Option, - #[prost(message, optional, tag = "2")] - pub evidence: ::core::option::Option, - #[prost(message, optional, tag = "3")] - pub validator: ::core::option::Option, - #[prost(message, optional, tag = "4")] - pub version: ::core::option::Option, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct BlockParams { - #[prost(int64, tag = "1")] - pub max_bytes: i64, - #[prost(int64, tag = "2")] - pub max_gas: i64, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct EvidenceParams { - #[prost(int64, tag = "1")] - pub max_age_num_blocks: i64, - #[prost(message, optional, tag = "2")] - pub max_age_duration: ::core::option::Option, - #[prost(int64, tag = "3")] - pub max_bytes: i64, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Duration { - #[prost(int64, tag = "1")] - pub seconds: i64, - #[prost(int32, tag = "2")] - pub nanos: i32, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ValidatorParams { - #[prost(string, repeated, tag = "1")] - pub pub_key_types: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct VersionParams { - #[prost(uint64, tag = "1")] - pub app_version: u64, -} -#[graph_runtime_derive::generate_asc_type(__required__{tx:Tx, result:ResponseDeliverTx})] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type( - __required__{tx:Tx, - result:ResponseDeliverTx} -)] -#[graph_runtime_derive::generate_array_type(Cosmos)] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct TxResult { - #[prost(uint64, tag = "1")] - pub height: u64, - #[prost(uint32, tag = "2")] - pub index: u32, - #[prost(message, optional, tag = "3")] - pub tx: ::core::option::Option, - #[prost(message, optional, tag = "4")] - pub result: ::core::option::Option, - #[prost(bytes = "vec", tag = "5")] - pub hash: ::prost::alloc::vec::Vec, -} -#[graph_runtime_derive::generate_asc_type(__required__{body:TxBody})] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type(__required__{body:TxBody})] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Tx { - #[prost(message, optional, tag = "1")] - pub body: ::core::option::Option, - #[prost(message, optional, tag = "2")] - pub auth_info: ::core::option::Option, - #[prost(bytes = "vec", repeated, tag = "3")] - pub signatures: ::prost::alloc::vec::Vec<::prost::alloc::vec::Vec>, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct TxBody { - #[prost(message, repeated, tag = "1")] - pub messages: ::prost::alloc::vec::Vec<::prost_types::Any>, - #[prost(string, tag = "2")] - pub memo: ::prost::alloc::string::String, - #[prost(uint64, tag = "3")] - pub timeout_height: u64, - #[prost(message, repeated, tag = "1023")] - pub extension_options: ::prost::alloc::vec::Vec<::prost_types::Any>, - #[prost(message, repeated, tag = "2047")] - pub non_critical_extension_options: ::prost::alloc::vec::Vec<::prost_types::Any>, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[graph_runtime_derive::generate_array_type(Cosmos)] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Any { - #[prost(string, tag = "1")] - pub type_url: ::prost::alloc::string::String, - #[prost(bytes = "vec", tag = "2")] - pub value: ::prost::alloc::vec::Vec, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct AuthInfo { - #[prost(message, repeated, tag = "1")] - pub signer_infos: ::prost::alloc::vec::Vec, - #[prost(message, optional, tag = "2")] - pub fee: ::core::option::Option, - #[prost(message, optional, tag = "3")] - pub tip: ::core::option::Option, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[graph_runtime_derive::generate_array_type(Cosmos)] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct SignerInfo { - #[prost(message, optional, tag = "1")] - pub public_key: ::core::option::Option<::prost_types::Any>, - #[prost(message, optional, tag = "2")] - pub mode_info: ::core::option::Option, - #[prost(uint64, tag = "3")] - pub sequence: u64, -} -#[graph_runtime_derive::generate_asc_type( - sum{single:ModeInfoSingle, - multi:ModeInfoMulti} -)] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type( - sum{single:ModeInfoSingle, - multi:ModeInfoMulti} -)] -#[graph_runtime_derive::generate_array_type(Cosmos)] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ModeInfo { - #[prost(oneof = "mode_info::Sum", tags = "1, 2")] - pub sum: ::core::option::Option, -} -/// Nested message and enum types in `ModeInfo`. -pub mod mode_info { - #[allow(clippy::derive_partial_eq_without_eq)] - #[derive(Clone, PartialEq, ::prost::Oneof)] - pub enum Sum { - #[prost(message, tag = "1")] - Single(super::ModeInfoSingle), - #[prost(message, tag = "2")] - Multi(super::ModeInfoMulti), - } -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ModeInfoSingle { - #[prost(enumeration = "SignMode", tag = "1")] - pub mode: i32, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ModeInfoMulti { - #[prost(message, optional, tag = "1")] - pub bitarray: ::core::option::Option, - #[prost(message, repeated, tag = "2")] - pub mode_infos: ::prost::alloc::vec::Vec, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct CompactBitArray { - #[prost(uint32, tag = "1")] - pub extra_bits_stored: u32, - #[prost(bytes = "vec", tag = "2")] - pub elems: ::prost::alloc::vec::Vec, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Fee { - #[prost(message, repeated, tag = "1")] - pub amount: ::prost::alloc::vec::Vec, - #[prost(uint64, tag = "2")] - pub gas_limit: u64, - #[prost(string, tag = "3")] - pub payer: ::prost::alloc::string::String, - #[prost(string, tag = "4")] - pub granter: ::prost::alloc::string::String, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[graph_runtime_derive::generate_array_type(Cosmos)] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Coin { - #[prost(string, tag = "1")] - pub denom: ::prost::alloc::string::String, - #[prost(string, tag = "2")] - pub amount: ::prost::alloc::string::String, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Tip { - #[prost(message, repeated, tag = "1")] - pub amount: ::prost::alloc::vec::Vec, - #[prost(string, tag = "2")] - pub tipper: ::prost::alloc::string::String, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ResponseDeliverTx { - #[prost(uint32, tag = "1")] - pub code: u32, - #[prost(bytes = "vec", tag = "2")] - pub data: ::prost::alloc::vec::Vec, - #[prost(string, tag = "3")] - pub log: ::prost::alloc::string::String, - #[prost(string, tag = "4")] - pub info: ::prost::alloc::string::String, - #[prost(int64, tag = "5")] - pub gas_wanted: i64, - #[prost(int64, tag = "6")] - pub gas_used: i64, - #[prost(message, repeated, tag = "7")] - pub events: ::prost::alloc::vec::Vec, - #[prost(string, tag = "8")] - pub codespace: ::prost::alloc::string::String, -} -#[graph_runtime_derive::generate_asc_type()] -#[graph_runtime_derive::generate_network_type_id(Cosmos)] -#[graph_runtime_derive::generate_from_rust_type()] -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ValidatorSetUpdates { - #[prost(message, repeated, tag = "1")] - pub validator_updates: ::prost::alloc::vec::Vec, -} -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] -#[repr(i32)] -pub enum SignedMsgType { - Unknown = 0, - Prevote = 1, - Precommit = 2, - Proposal = 32, -} -impl SignedMsgType { - /// String value of the enum field names used in the ProtoBuf definition. - /// - /// The values are not transformed in any way and thus are considered stable - /// (if the ProtoBuf definition does not change) and safe for programmatic use. - pub fn as_str_name(&self) -> &'static str { - match self { - SignedMsgType::Unknown => "SIGNED_MSG_TYPE_UNKNOWN", - SignedMsgType::Prevote => "SIGNED_MSG_TYPE_PREVOTE", - SignedMsgType::Precommit => "SIGNED_MSG_TYPE_PRECOMMIT", - SignedMsgType::Proposal => "SIGNED_MSG_TYPE_PROPOSAL", - } - } - /// Creates an enum from field names used in the ProtoBuf definition. - pub fn from_str_name(value: &str) -> ::core::option::Option { - match value { - "SIGNED_MSG_TYPE_UNKNOWN" => Some(Self::Unknown), - "SIGNED_MSG_TYPE_PREVOTE" => Some(Self::Prevote), - "SIGNED_MSG_TYPE_PRECOMMIT" => Some(Self::Precommit), - "SIGNED_MSG_TYPE_PROPOSAL" => Some(Self::Proposal), - _ => None, - } - } -} -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] -#[repr(i32)] -pub enum BlockIdFlag { - Unknown = 0, - Absent = 1, - Commit = 2, - Nil = 3, -} -impl BlockIdFlag { - /// String value of the enum field names used in the ProtoBuf definition. - /// - /// The values are not transformed in any way and thus are considered stable - /// (if the ProtoBuf definition does not change) and safe for programmatic use. - pub fn as_str_name(&self) -> &'static str { - match self { - BlockIdFlag::Unknown => "BLOCK_ID_FLAG_UNKNOWN", - BlockIdFlag::Absent => "BLOCK_ID_FLAG_ABSENT", - BlockIdFlag::Commit => "BLOCK_ID_FLAG_COMMIT", - BlockIdFlag::Nil => "BLOCK_ID_FLAG_NIL", - } - } - /// Creates an enum from field names used in the ProtoBuf definition. - pub fn from_str_name(value: &str) -> ::core::option::Option { - match value { - "BLOCK_ID_FLAG_UNKNOWN" => Some(Self::Unknown), - "BLOCK_ID_FLAG_ABSENT" => Some(Self::Absent), - "BLOCK_ID_FLAG_COMMIT" => Some(Self::Commit), - "BLOCK_ID_FLAG_NIL" => Some(Self::Nil), - _ => None, - } - } -} -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] -#[repr(i32)] -pub enum SignMode { - Unspecified = 0, - Direct = 1, - Textual = 2, - LegacyAminoJson = 127, -} -impl SignMode { - /// String value of the enum field names used in the ProtoBuf definition. - /// - /// The values are not transformed in any way and thus are considered stable - /// (if the ProtoBuf definition does not change) and safe for programmatic use. - pub fn as_str_name(&self) -> &'static str { - match self { - SignMode::Unspecified => "SIGN_MODE_UNSPECIFIED", - SignMode::Direct => "SIGN_MODE_DIRECT", - SignMode::Textual => "SIGN_MODE_TEXTUAL", - SignMode::LegacyAminoJson => "SIGN_MODE_LEGACY_AMINO_JSON", - } - } - /// Creates an enum from field names used in the ProtoBuf definition. - pub fn from_str_name(value: &str) -> ::core::option::Option { - match value { - "SIGN_MODE_UNSPECIFIED" => Some(Self::Unspecified), - "SIGN_MODE_DIRECT" => Some(Self::Direct), - "SIGN_MODE_TEXTUAL" => Some(Self::Textual), - "SIGN_MODE_LEGACY_AMINO_JSON" => Some(Self::LegacyAminoJson), - _ => None, - } - } -} diff --git a/chain/cosmos/src/runtime/abi.rs b/chain/cosmos/src/runtime/abi.rs deleted file mode 100644 index af9260b63be..00000000000 --- a/chain/cosmos/src/runtime/abi.rs +++ /dev/null @@ -1,80 +0,0 @@ -use crate::protobuf::*; -use graph::runtime::HostExportError; -pub use graph::semver::Version; - -pub use graph::runtime::{ - asc_new, gas::GasCounter, AscHeap, AscIndexId, AscPtr, AscType, AscValue, - DeterministicHostError, IndexForAscTypeId, ToAscObj, -}; -/* -TODO: AscBytesArray seem to be generic to all chains, but AscIndexId pins it to Cosmos -****************** this can be moved to runtime graph/runtime/src/asc_heap.rs, but IndexForAscTypeId::CosmosBytesArray ****** -*/ -pub struct AscBytesArray(pub Array>); - -impl ToAscObj for Vec> { - fn to_asc_obj( - &self, - heap: &mut H, - gas: &GasCounter, - ) -> Result { - let content: Result, _> = self - .iter() - .map(|x| asc_new(heap, &graph_runtime_wasm::asc_abi::class::Bytes(x), gas)) - .collect(); - - Ok(AscBytesArray(Array::new(&content?, heap, gas)?)) - } -} - -//this can be moved to runtime -impl AscType for AscBytesArray { - fn to_asc_bytes(&self) -> Result, DeterministicHostError> { - self.0.to_asc_bytes() - } - - fn from_asc_bytes( - asc_obj: &[u8], - api_version: &Version, - ) -> Result { - Ok(Self(Array::from_asc_bytes(asc_obj, api_version)?)) - } -} - -//we will have to keep this chain specific (Inner/Outer) -impl AscIndexId for AscBytesArray { - const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::CosmosBytesArray; -} - -/************************************************************************** */ -// this can be moved to runtime - prost_types::Any -impl ToAscObj for prost_types::Any { - fn to_asc_obj( - &self, - heap: &mut H, - gas: &GasCounter, - ) -> Result { - Ok(AscAny { - type_url: asc_new(heap, &self.type_url, gas)?, - value: asc_new( - heap, - &graph_runtime_wasm::asc_abi::class::Bytes(&self.value), - gas, - )?, - ..Default::default() - }) - } -} - -//this can be moved to runtime - prost_types::Any -impl ToAscObj for Vec { - fn to_asc_obj( - &self, - heap: &mut H, - gas: &GasCounter, - ) -> Result { - let content: Result, _> = self.iter().map(|x| asc_new(heap, x, gas)).collect(); - - Ok(AscAnyArray(Array::new(&content?, heap, gas)?)) - } -} diff --git a/chain/cosmos/src/runtime/mod.rs b/chain/cosmos/src/runtime/mod.rs deleted file mode 100644 index 78d10d4d7d3..00000000000 --- a/chain/cosmos/src/runtime/mod.rs +++ /dev/null @@ -1,348 +0,0 @@ -pub mod abi; - -#[cfg(test)] -mod test { - use crate::protobuf::*; - - use graph::semver::Version; - - /// A macro that takes an ASC struct value definition and calls AscBytes methods to check that - /// memory layout is padded properly. - macro_rules! assert_asc_bytes { - ($struct_name:ident { - $($field:ident : $field_value:expr),+ - $(,)? // trailing - }) => { - let value = $struct_name { - $($field: $field_value),+ - }; - - // just call the function. it will panic on misalignments - let asc_bytes = value.to_asc_bytes().unwrap(); - - let value_004 = $struct_name::from_asc_bytes(&asc_bytes, &Version::new(0, 0, 4)).unwrap(); - let value_005 = $struct_name::from_asc_bytes(&asc_bytes, &Version::new(0, 0, 5)).unwrap(); - - // turn the values into bytes again to verify that they are the same as the original - // because these types usually don't implement PartialEq - assert_eq!( - asc_bytes, - value_004.to_asc_bytes().unwrap(), - "Expected {} v0.0.4 asc bytes to be the same", - stringify!($struct_name) - ); - assert_eq!( - asc_bytes, - value_005.to_asc_bytes().unwrap(), - "Expected {} v0.0.5 asc bytes to be the same", - stringify!($struct_name) - ); - }; - } - - #[test] - fn test_asc_type_alignment() { - // TODO: automatically generate these tests for each struct in derive(AscType) macro - - assert_asc_bytes!(AscBlock { - header: new_asc_ptr(), - evidence: new_asc_ptr(), - last_commit: new_asc_ptr(), - result_begin_block: new_asc_ptr(), - result_end_block: new_asc_ptr(), - transactions: new_asc_ptr(), - validator_updates: new_asc_ptr(), - }); - - assert_asc_bytes!(AscHeaderOnlyBlock { - header: new_asc_ptr(), - }); - - assert_asc_bytes!(AscEventData { - event: new_asc_ptr(), - block: new_asc_ptr(), - tx: new_asc_ptr(), - }); - - assert_asc_bytes!(AscTransactionData { - tx: new_asc_ptr(), - block: new_asc_ptr(), - }); - - assert_asc_bytes!(AscMessageData { - message: new_asc_ptr(), - block: new_asc_ptr(), - tx: new_asc_ptr(), - }); - - assert_asc_bytes!(AscTransactionContext { - hash: new_asc_ptr(), - index: 20, - code: 20, - gas_wanted: 20, - gas_used: 20, - }); - - assert_asc_bytes!(AscHeader { - version: new_asc_ptr(), - chain_id: new_asc_ptr(), - height: 20, - time: new_asc_ptr(), - last_block_id: new_asc_ptr(), - last_commit_hash: new_asc_ptr(), - data_hash: new_asc_ptr(), - validators_hash: new_asc_ptr(), - next_validators_hash: new_asc_ptr(), - consensus_hash: new_asc_ptr(), - app_hash: new_asc_ptr(), - last_results_hash: new_asc_ptr(), - evidence_hash: new_asc_ptr(), - proposer_address: new_asc_ptr(), - hash: new_asc_ptr(), - }); - - assert_asc_bytes!(AscConsensus { block: 0, app: 0 }); - - assert_asc_bytes!(AscTimestamp { - seconds: 20, - nanos: 20, - }); - - assert_asc_bytes!(AscBlockId { - hash: new_asc_ptr(), - part_set_header: new_asc_ptr(), - }); - - assert_asc_bytes!(AscPartSetHeader { - total: 20, - hash: new_asc_ptr(), - }); - - assert_asc_bytes!(AscEvidenceList { - evidence: new_asc_ptr(), - }); - - assert_asc_bytes!(AscEvidence { - duplicate_vote_evidence: new_asc_ptr(), - light_client_attack_evidence: new_asc_ptr(), - }); - - assert_asc_bytes!(AscDuplicateVoteEvidence { - vote_a: new_asc_ptr(), - vote_b: new_asc_ptr(), - total_voting_power: 20, - validator_power: 20, - timestamp: new_asc_ptr(), - }); - - assert_asc_bytes!(AscEventVote { - event_vote_type: 20, - height: 20, - round: 20, - block_id: new_asc_ptr(), - timestamp: new_asc_ptr(), - validator_address: new_asc_ptr(), - validator_index: 20, - signature: new_asc_ptr(), - }); - - assert_asc_bytes!(AscLightClientAttackEvidence { - conflicting_block: new_asc_ptr(), - common_height: 20, - total_voting_power: 20, - byzantine_validators: new_asc_ptr(), - timestamp: new_asc_ptr(), - }); - - assert_asc_bytes!(AscLightBlock { - signed_header: new_asc_ptr(), - validator_set: new_asc_ptr(), - }); - - assert_asc_bytes!(AscSignedHeader { - header: new_asc_ptr(), - commit: new_asc_ptr(), - }); - - assert_asc_bytes!(AscCommit { - height: 20, - round: 20, - block_id: new_asc_ptr(), - signatures: new_asc_ptr(), - }); - - assert_asc_bytes!(AscCommitSig { - block_id_flag: 20, - validator_address: new_asc_ptr(), - timestamp: new_asc_ptr(), - signature: new_asc_ptr(), - }); - - assert_asc_bytes!(AscValidatorSet { - validators: new_asc_ptr(), - proposer: new_asc_ptr(), - total_voting_power: 20, - }); - - assert_asc_bytes!(AscValidator { - address: new_asc_ptr(), - pub_key: new_asc_ptr(), - voting_power: 20, - proposer_priority: 20, - }); - - assert_asc_bytes!(AscPublicKey { - ed25519: new_asc_ptr(), - secp256k1: new_asc_ptr(), - }); - - assert_asc_bytes!(AscResponseBeginBlock { - events: new_asc_ptr(), - }); - - assert_asc_bytes!(AscEvent { - event_type: new_asc_ptr(), - attributes: new_asc_ptr(), - }); - - assert_asc_bytes!(AscEventAttribute { - key: new_asc_ptr(), - value: new_asc_ptr(), - index: true, - }); - - assert_asc_bytes!(AscResponseEndBlock { - validator_updates: new_asc_ptr(), - consensus_param_updates: new_asc_ptr(), - events: new_asc_ptr(), - }); - - assert_asc_bytes!(AscValidatorUpdate { - address: new_asc_ptr(), - pub_key: new_asc_ptr(), - power: 20, - }); - - assert_asc_bytes!(AscConsensusParams { - block: new_asc_ptr(), - evidence: new_asc_ptr(), - validator: new_asc_ptr(), - version: new_asc_ptr(), - }); - - assert_asc_bytes!(AscBlockParams { - max_bytes: 20, - max_gas: 20, - }); - - assert_asc_bytes!(AscEvidenceParams { - max_age_num_blocks: 20, - max_age_duration: new_asc_ptr(), - max_bytes: 20, - }); - - assert_asc_bytes!(AscDuration { - seconds: 20, - nanos: 20, - }); - - assert_asc_bytes!(AscValidatorParams { - pub_key_types: new_asc_ptr(), - }); - - assert_asc_bytes!(AscVersionParams { app_version: 20 }); - - assert_asc_bytes!(AscTxResult { - height: 20, - index: 20, - tx: new_asc_ptr(), - result: new_asc_ptr(), - hash: new_asc_ptr(), - }); - - assert_asc_bytes!(AscTx { - body: new_asc_ptr(), - auth_info: new_asc_ptr(), - signatures: new_asc_ptr(), - }); - - assert_asc_bytes!(AscTxBody { - messages: new_asc_ptr(), - memo: new_asc_ptr(), - timeout_height: 20, - extension_options: new_asc_ptr(), - non_critical_extension_options: new_asc_ptr(), - }); - - assert_asc_bytes!(AscAny { - type_url: new_asc_ptr(), - value: new_asc_ptr(), - }); - - assert_asc_bytes!(AscAuthInfo { - signer_infos: new_asc_ptr(), - fee: new_asc_ptr(), - tip: new_asc_ptr(), - }); - - assert_asc_bytes!(AscSignerInfo { - public_key: new_asc_ptr(), - mode_info: new_asc_ptr(), - sequence: 20, - }); - - assert_asc_bytes!(AscModeInfo { - single: new_asc_ptr(), - multi: new_asc_ptr(), - }); - - assert_asc_bytes!(AscModeInfoSingle { mode: 20 }); - - assert_asc_bytes!(AscModeInfoMulti { - bitarray: new_asc_ptr(), - mode_infos: new_asc_ptr(), - }); - - assert_asc_bytes!(AscCompactBitArray { - extra_bits_stored: 20, - elems: new_asc_ptr(), - }); - - assert_asc_bytes!(AscFee { - amount: new_asc_ptr(), - gas_limit: 20, - payer: new_asc_ptr(), - granter: new_asc_ptr(), - }); - - assert_asc_bytes!(AscCoin { - denom: new_asc_ptr(), - amount: new_asc_ptr(), - }); - - assert_asc_bytes!(AscTip { - amount: new_asc_ptr(), - tipper: new_asc_ptr(), - }); - - assert_asc_bytes!(AscResponseDeliverTx { - code: 20, - data: new_asc_ptr(), - log: new_asc_ptr(), - info: new_asc_ptr(), - gas_wanted: 20, - gas_used: 20, - events: new_asc_ptr(), - codespace: new_asc_ptr(), - }); - - assert_asc_bytes!(AscValidatorSetUpdates { - validator_updates: new_asc_ptr(), - }); - } - - // non-null AscPtr - fn new_asc_ptr() -> AscPtr { - AscPtr::new(12) - } -} diff --git a/chain/cosmos/src/trigger.rs b/chain/cosmos/src/trigger.rs deleted file mode 100644 index 9700a75bf76..00000000000 --- a/chain/cosmos/src/trigger.rs +++ /dev/null @@ -1,364 +0,0 @@ -use std::{cmp::Ordering, sync::Arc}; - -use graph::blockchain::{Block, BlockHash, MappingTriggerTrait, TriggerData}; -use graph::derive::CheapClone; -use graph::prelude::{BlockNumber, Error}; -use graph::runtime::HostExportError; -use graph::runtime::{asc_new, gas::GasCounter, AscHeap, AscPtr}; -use graph_runtime_wasm::module::ToAscPtr; - -use crate::codec; -use crate::data_source::EventOrigin; - -// Logging the block is too verbose, so this strips the block from the trigger for Debug. -impl std::fmt::Debug for CosmosTrigger { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - #[allow(unused)] - #[derive(Debug)] - pub enum MappingTriggerWithoutBlock<'e> { - Block, - Event { - event_type: &'e str, - origin: EventOrigin, - }, - Transaction, - Message, - } - - let trigger_without_block = match self { - CosmosTrigger::Block(_) => MappingTriggerWithoutBlock::Block, - CosmosTrigger::Event { event_data, origin } => MappingTriggerWithoutBlock::Event { - event_type: &event_data.event().map_err(|_| std::fmt::Error)?.event_type, - origin: *origin, - }, - CosmosTrigger::Transaction(_) => MappingTriggerWithoutBlock::Transaction, - CosmosTrigger::Message(_) => MappingTriggerWithoutBlock::Message, - }; - - write!(f, "{:?}", trigger_without_block) - } -} - -impl ToAscPtr for CosmosTrigger { - fn to_asc_ptr( - self, - heap: &mut H, - gas: &GasCounter, - ) -> Result, HostExportError> { - Ok(match self { - CosmosTrigger::Block(block) => asc_new(heap, block.as_ref(), gas)?.erase(), - CosmosTrigger::Event { event_data, .. } => { - asc_new(heap, event_data.as_ref(), gas)?.erase() - } - CosmosTrigger::Transaction(transaction_data) => { - asc_new(heap, transaction_data.as_ref(), gas)?.erase() - } - CosmosTrigger::Message(message_data) => { - asc_new(heap, message_data.as_ref(), gas)?.erase() - } - }) - } -} - -#[derive(Clone, CheapClone)] -pub enum CosmosTrigger { - Block(Arc), - Event { - event_data: Arc, - origin: EventOrigin, - }, - Transaction(Arc), - Message(Arc), -} - -impl PartialEq for CosmosTrigger { - fn eq(&self, other: &Self) -> bool { - match (self, other) { - (Self::Block(a_ptr), Self::Block(b_ptr)) => a_ptr == b_ptr, - ( - Self::Event { - event_data: a_event_data, - origin: a_origin, - }, - Self::Event { - event_data: b_event_data, - origin: b_origin, - }, - ) => { - if let (Ok(a_event), Ok(b_event)) = (a_event_data.event(), b_event_data.event()) { - let mut attributes_a = a_event.attributes.clone(); - attributes_a.sort_by(|a, b| a.key.cmp(&b.key)); - - let mut attributes_b = b_event.attributes.clone(); - attributes_b.sort_by(|a, b| a.key.cmp(&b.key)); - - a_event.event_type == b_event.event_type - && a_origin == b_origin - && attributes_a == attributes_b - } else { - false - } - } - (Self::Transaction(a_ptr), Self::Transaction(b_ptr)) => a_ptr == b_ptr, - (Self::Message(a_ptr), Self::Message(b_ptr)) => a_ptr == b_ptr, - _ => false, - } - } -} - -impl Eq for CosmosTrigger {} - -impl CosmosTrigger { - pub(crate) fn with_event( - event: codec::Event, - block: codec::HeaderOnlyBlock, - tx_context: Option, - origin: EventOrigin, - ) -> CosmosTrigger { - CosmosTrigger::Event { - event_data: Arc::new(codec::EventData { - event: Some(event), - block: Some(block), - tx: tx_context, - }), - origin, - } - } - - pub(crate) fn with_transaction( - tx_result: codec::TxResult, - block: codec::HeaderOnlyBlock, - ) -> CosmosTrigger { - CosmosTrigger::Transaction(Arc::new(codec::TransactionData { - tx: Some(tx_result), - block: Some(block), - })) - } - - pub(crate) fn with_message( - message: ::prost_types::Any, - block: codec::HeaderOnlyBlock, - tx_context: codec::TransactionContext, - ) -> CosmosTrigger { - CosmosTrigger::Message(Arc::new(codec::MessageData { - message: Some(message), - block: Some(block), - tx: Some(tx_context), - })) - } - - pub fn block_number(&self) -> Result { - match self { - CosmosTrigger::Block(block) => Ok(block.number()), - CosmosTrigger::Event { event_data, .. } => event_data.block().map(|b| b.number()), - CosmosTrigger::Transaction(transaction_data) => { - transaction_data.block().map(|b| b.number()) - } - CosmosTrigger::Message(message_data) => message_data.block().map(|b| b.number()), - } - } - - pub fn block_hash(&self) -> Result { - match self { - CosmosTrigger::Block(block) => Ok(block.hash()), - CosmosTrigger::Event { event_data, .. } => event_data.block().map(|b| b.hash()), - CosmosTrigger::Transaction(transaction_data) => { - transaction_data.block().map(|b| b.hash()) - } - CosmosTrigger::Message(message_data) => message_data.block().map(|b| b.hash()), - } - } - - fn error_context(&self) -> std::string::String { - match self { - CosmosTrigger::Block(..) => { - if let (Ok(block_number), Ok(block_hash)) = (self.block_number(), self.block_hash()) - { - format!("block #{block_number}, hash {block_hash}") - } else { - "block".to_string() - } - } - CosmosTrigger::Event { event_data, origin } => { - if let (Ok(event), Ok(block_number), Ok(block_hash)) = - (event_data.event(), self.block_number(), self.block_hash()) - { - format!( - "event type {}, origin: {:?}, block #{block_number}, hash {block_hash}", - event.event_type, origin, - ) - } else { - "event".to_string() - } - } - CosmosTrigger::Transaction(transaction_data) => { - if let (Ok(block_number), Ok(block_hash), Ok(response_deliver_tx)) = ( - self.block_number(), - self.block_hash(), - transaction_data.response_deliver_tx(), - ) { - format!( - "block #{block_number}, hash {block_hash}, transaction log: {}", - response_deliver_tx.log - ) - } else { - "transaction".to_string() - } - } - CosmosTrigger::Message(message_data) => { - if let (Ok(message), Ok(block_number), Ok(block_hash)) = ( - message_data.message(), - self.block_number(), - self.block_hash(), - ) { - format!( - "message type {}, block #{block_number}, hash {block_hash}", - message.type_url, - ) - } else { - "message".to_string() - } - } - } - } -} - -impl Ord for CosmosTrigger { - fn cmp(&self, other: &Self) -> Ordering { - match (self, other) { - // Events have no intrinsic ordering information, so we keep the order in - // which they are included in the `events` field - (Self::Event { .. }, Self::Event { .. }) => Ordering::Equal, - - // Keep the order when comparing two message triggers - (Self::Message(..), Self::Message(..)) => Ordering::Equal, - - // Transactions are ordered by their index inside the block - (Self::Transaction(a), Self::Transaction(b)) => { - if let (Ok(a_tx_result), Ok(b_tx_result)) = (a.tx_result(), b.tx_result()) { - a_tx_result.index.cmp(&b_tx_result.index) - } else { - Ordering::Equal - } - } - - // Keep the order when comparing two block triggers - (Self::Block(..), Self::Block(..)) => Ordering::Equal, - - // Event triggers always come first - (Self::Event { .. }, _) => Ordering::Greater, - (_, Self::Event { .. }) => Ordering::Less, - - // Block triggers always come last - (Self::Block(..), _) => Ordering::Less, - (_, Self::Block(..)) => Ordering::Greater, - - // Message triggers before Transaction triggers - (Self::Message(..), Self::Transaction(..)) => Ordering::Greater, - (Self::Transaction(..), Self::Message(..)) => Ordering::Less, - } - } -} - -impl PartialOrd for CosmosTrigger { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -impl TriggerData for CosmosTrigger { - fn error_context(&self) -> String { - self.error_context() - } - - fn address_match(&self) -> Option<&[u8]> { - None - } -} - -impl MappingTriggerTrait for CosmosTrigger { - fn error_context(&self) -> String { - self.error_context() - } -} -#[cfg(test)] -mod tests { - use crate::codec::TxResult; - - use super::*; - - #[test] - fn test_cosmos_trigger_ordering() { - let event_trigger = CosmosTrigger::Event { - event_data: Arc::::new(codec::EventData { - ..Default::default() - }), - origin: EventOrigin::BeginBlock, - }; - let other_event_trigger = CosmosTrigger::Event { - event_data: Arc::::new(codec::EventData { - ..Default::default() - }), - origin: EventOrigin::BeginBlock, - }; - let message_trigger = - CosmosTrigger::Message(Arc::::new(codec::MessageData { - ..Default::default() - })); - let other_message_trigger = - CosmosTrigger::Message(Arc::::new(codec::MessageData { - ..Default::default() - })); - let transaction_trigger = CosmosTrigger::Transaction(Arc::::new( - codec::TransactionData { - block: None, - tx: Some(TxResult { - index: 1, - ..Default::default() - }), - }, - )); - let other_transaction_trigger = CosmosTrigger::Transaction( - Arc::::new(codec::TransactionData { - block: None, - tx: Some(TxResult { - index: 2, - ..Default::default() - }), - }), - ); - let block_trigger = CosmosTrigger::Block(Arc::::new(codec::Block { - ..Default::default() - })); - let other_block_trigger = CosmosTrigger::Block(Arc::::new(codec::Block { - ..Default::default() - })); - - assert_eq!(event_trigger.cmp(&block_trigger), Ordering::Greater); - assert_eq!(event_trigger.cmp(&transaction_trigger), Ordering::Greater); - assert_eq!(event_trigger.cmp(&message_trigger), Ordering::Greater); - assert_eq!(event_trigger.cmp(&other_event_trigger), Ordering::Equal); - - assert_eq!(message_trigger.cmp(&block_trigger), Ordering::Greater); - assert_eq!(message_trigger.cmp(&transaction_trigger), Ordering::Greater); - assert_eq!(message_trigger.cmp(&other_message_trigger), Ordering::Equal); - assert_eq!(message_trigger.cmp(&event_trigger), Ordering::Less); - - assert_eq!(transaction_trigger.cmp(&block_trigger), Ordering::Greater); - assert_eq!( - transaction_trigger.cmp(&other_transaction_trigger), - Ordering::Less - ); - assert_eq!( - other_transaction_trigger.cmp(&transaction_trigger), - Ordering::Greater - ); - assert_eq!(transaction_trigger.cmp(&message_trigger), Ordering::Less); - assert_eq!(transaction_trigger.cmp(&event_trigger), Ordering::Less); - - assert_eq!(block_trigger.cmp(&other_block_trigger), Ordering::Equal); - assert_eq!(block_trigger.cmp(&transaction_trigger), Ordering::Less); - assert_eq!(block_trigger.cmp(&message_trigger), Ordering::Less); - assert_eq!(block_trigger.cmp(&event_trigger), Ordering::Less); - } -} diff --git a/core/Cargo.toml b/core/Cargo.toml index 4533b5d8880..06c3d7d9862 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -13,7 +13,6 @@ graph = { path = "../graph" } graph-chain-arweave = { path = "../chain/arweave" } graph-chain-ethereum = { path = "../chain/ethereum" } graph-chain-near = { path = "../chain/near" } -graph-chain-cosmos = { path = "../chain/cosmos" } graph-chain-substreams = { path = "../chain/substreams" } graph-runtime-wasm = { path = "../runtime/wasm" } serde_yaml = { workspace = true } diff --git a/core/src/subgraph/instance_manager.rs b/core/src/subgraph/instance_manager.rs index 2d54a90417c..8c2b76e5b6c 100644 --- a/core/src/subgraph/instance_manager.rs +++ b/core/src/subgraph/instance_manager.rs @@ -124,21 +124,6 @@ impl SubgraphInstanceManagerTrait for SubgraphInstanceManager< self.start_subgraph_inner(logger, loc, runner).await } - BlockchainKind::Cosmos => { - let runner = instance_manager - .build_subgraph_runner::( - logger.clone(), - self.env_vars.cheap_clone(), - loc.clone(), - manifest, - stop_block, - Box::new(SubgraphTriggerProcessor {}), - deployment_status_metric, - ) - .await?; - - self.start_subgraph_inner(logger, loc, runner).await - } BlockchainKind::Substreams => { let runner = instance_manager .build_subgraph_runner::( diff --git a/core/src/subgraph/registrar.rs b/core/src/subgraph/registrar.rs index 41f7697c529..3a712b6daa9 100644 --- a/core/src/subgraph/registrar.rs +++ b/core/src/subgraph/registrar.rs @@ -367,24 +367,6 @@ where ) .await? } - BlockchainKind::Cosmos => { - create_subgraph_version::( - &logger, - self.store.clone(), - self.chains.cheap_clone(), - name.clone(), - hash.cheap_clone(), - start_block_override, - graft_block_override, - raw, - node_id, - debug_fork, - self.version_switching_mode, - &self.resolver, - history_blocks, - ) - .await? - } BlockchainKind::Substreams => { create_subgraph_version::( &logger, diff --git a/graph/build.rs b/graph/build.rs index 4ad3c03a459..53bbe421ab9 100644 --- a/graph/build.rs +++ b/graph/build.rs @@ -7,7 +7,6 @@ fn main() { "proto/firehose.proto", "proto/ethereum/transforms.proto", "proto/near/transforms.proto", - "proto/cosmos/transforms.proto", ], &["proto"], ) diff --git a/graph/src/blockchain/mod.rs b/graph/src/blockchain/mod.rs index 2e3c60e88b0..25a7d147540 100644 --- a/graph/src/blockchain/mod.rs +++ b/graph/src/blockchain/mod.rs @@ -565,9 +565,6 @@ pub enum BlockchainKind { /// NEAR chains (Mainnet, Testnet) or chains that are compatible Near, - /// Cosmos chains - Cosmos, - Substreams, } @@ -577,7 +574,6 @@ impl fmt::Display for BlockchainKind { BlockchainKind::Arweave => "arweave", BlockchainKind::Ethereum => "ethereum", BlockchainKind::Near => "near", - BlockchainKind::Cosmos => "cosmos", BlockchainKind::Substreams => "substreams", }; write!(f, "{}", value) @@ -592,7 +588,6 @@ impl FromStr for BlockchainKind { "arweave" => Ok(BlockchainKind::Arweave), "ethereum" => Ok(BlockchainKind::Ethereum), "near" => Ok(BlockchainKind::Near), - "cosmos" => Ok(BlockchainKind::Cosmos), "substreams" => Ok(BlockchainKind::Substreams), "subgraph" => Ok(BlockchainKind::Ethereum), // TODO(krishna): We should detect the blockchain kind from the source subgraph _ => Err(anyhow!("unknown blockchain kind {}", s)), diff --git a/graph/src/firehose/codec.rs b/graph/src/firehose/codec.rs index 5537dba153b..3768f3acf45 100644 --- a/graph/src/firehose/codec.rs +++ b/graph/src/firehose/codec.rs @@ -10,11 +10,6 @@ mod pbethereum; #[path = "sf.near.transform.v1.rs"] mod pbnear; -#[rustfmt::skip] -#[path = "sf.cosmos.transform.v1.rs"] -mod pbcosmos; - -pub use pbcosmos::*; pub use pbethereum::*; pub use pbfirehose::*; pub use pbnear::*; diff --git a/graph/src/firehose/sf.cosmos.transform.v1.rs b/graph/src/firehose/sf.cosmos.transform.v1.rs deleted file mode 100644 index 5bde2c0e996..00000000000 --- a/graph/src/firehose/sf.cosmos.transform.v1.rs +++ /dev/null @@ -1,7 +0,0 @@ -// This file is @generated by prost-build. -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct EventTypeFilter { - #[prost(string, repeated, tag = "1")] - pub event_types: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, -} diff --git a/graph/src/runtime/mod.rs b/graph/src/runtime/mod.rs index 5622d37c100..6732a57429e 100644 --- a/graph/src/runtime/mod.rs +++ b/graph/src/runtime/mod.rs @@ -268,77 +268,7 @@ pub enum IndexForAscTypeId { // ... // LastEthereumType = 1499, - // Reserved discriminant space for Cosmos type IDs: [1,500, 2,499] - CosmosAny = 1500, - CosmosAnyArray = 1501, - CosmosBytesArray = 1502, - CosmosCoinArray = 1503, - CosmosCommitSigArray = 1504, - CosmosEventArray = 1505, - CosmosEventAttributeArray = 1506, - CosmosEvidenceArray = 1507, - CosmosModeInfoArray = 1508, - CosmosSignerInfoArray = 1509, - CosmosTxResultArray = 1510, - CosmosValidatorArray = 1511, - CosmosValidatorUpdateArray = 1512, - CosmosAuthInfo = 1513, - CosmosBlock = 1514, - CosmosBlockId = 1515, - CosmosBlockIdFlagEnum = 1516, - CosmosBlockParams = 1517, - CosmosCoin = 1518, - CosmosCommit = 1519, - CosmosCommitSig = 1520, - CosmosCompactBitArray = 1521, - CosmosConsensus = 1522, - CosmosConsensusParams = 1523, - CosmosDuplicateVoteEvidence = 1524, - CosmosDuration = 1525, - CosmosEvent = 1526, - CosmosEventAttribute = 1527, - CosmosEventData = 1528, - CosmosEventVote = 1529, - CosmosEvidence = 1530, - CosmosEvidenceList = 1531, - CosmosEvidenceParams = 1532, - CosmosFee = 1533, - CosmosHeader = 1534, - CosmosHeaderOnlyBlock = 1535, - CosmosLightBlock = 1536, - CosmosLightClientAttackEvidence = 1537, - CosmosModeInfo = 1538, - CosmosModeInfoMulti = 1539, - CosmosModeInfoSingle = 1540, - CosmosPartSetHeader = 1541, - CosmosPublicKey = 1542, - CosmosResponseBeginBlock = 1543, - CosmosResponseDeliverTx = 1544, - CosmosResponseEndBlock = 1545, - CosmosSignModeEnum = 1546, - CosmosSignedHeader = 1547, - CosmosSignedMsgTypeEnum = 1548, - CosmosSignerInfo = 1549, - CosmosTimestamp = 1550, - CosmosTip = 1551, - CosmosTransactionData = 1552, - CosmosTx = 1553, - CosmosTxBody = 1554, - CosmosTxResult = 1555, - CosmosValidator = 1556, - CosmosValidatorParams = 1557, - CosmosValidatorSet = 1558, - CosmosValidatorSetUpdates = 1559, - CosmosValidatorUpdate = 1560, - CosmosVersionParams = 1561, - CosmosMessageData = 1562, - CosmosTransactionContext = 1563, - // Continue to add more Cosmos type IDs here. - // e.g.: - // NextCosmosType = 1564, - // AnotherCosmosType = 1565, - // ... - // LastCosmosType = 2499, + // Discriminant space [1,500, 2,499] was reserved for Cosmos, which has been removed // Arweave types ArweaveBlock = 2500, diff --git a/node/Cargo.toml b/node/Cargo.toml index 812ab5fc3e0..ee6411fc87c 100644 --- a/node/Cargo.toml +++ b/node/Cargo.toml @@ -25,7 +25,6 @@ graph-core = { path = "../core" } graph-chain-arweave = { path = "../chain/arweave" } graph-chain-ethereum = { path = "../chain/ethereum" } graph-chain-near = { path = "../chain/near" } -graph-chain-cosmos = { path = "../chain/cosmos" } graph-chain-substreams = { path = "../chain/substreams" } graph-graphql = { path = "../graphql" } graph-server-http = { path = "../server/http" } diff --git a/node/src/chain.rs b/node/src/chain.rs index beeb366ad61..00785d11876 100644 --- a/node/src/chain.rs +++ b/node/src/chain.rs @@ -506,33 +506,6 @@ pub async fn networks_as_chains( ) .await; } - BlockchainKind::Cosmos => { - let firehose_endpoints = networks.firehose_endpoints(chain_id.clone()); - blockchain_map.insert::( - chain_id.clone(), - Arc::new( - BasicBlockchainBuilder { - logger_factory: logger_factory.clone(), - name: chain_id.clone(), - chain_store: chain_store.cheap_clone(), - firehose_endpoints, - metrics_registry: metrics_registry.clone(), - } - .build(config) - .await, - ), - ); - add_substreams::( - networks, - config, - chain_id.clone(), - blockchain_map, - logger_factory.clone(), - chain_store, - metrics_registry.clone(), - ) - .await; - } BlockchainKind::Substreams => { let substreams_endpoints = networks.substreams_endpoints(chain_id.clone()); blockchain_map.insert::( diff --git a/node/src/network_setup.rs b/node/src/network_setup.rs index 4a8b4cedca1..1ebe2b5109c 100644 --- a/node/src/network_setup.rs +++ b/node/src/network_setup.rs @@ -338,9 +338,6 @@ impl Networks { BlockchainKind::Near => { block_ingestor::(logger, id, chain, &mut res).await? } - BlockchainKind::Cosmos => { - block_ingestor::(logger, id, chain, &mut res).await? - } BlockchainKind::Substreams => {} } } diff --git a/server/index-node/Cargo.toml b/server/index-node/Cargo.toml index 74048cc5e0a..72b7ff869f7 100644 --- a/server/index-node/Cargo.toml +++ b/server/index-node/Cargo.toml @@ -10,6 +10,5 @@ graph-graphql = { path = "../../graphql" } graph-chain-arweave = { path = "../../chain/arweave" } graph-chain-ethereum = { path = "../../chain/ethereum" } graph-chain-near = { path = "../../chain/near" } -graph-chain-cosmos = { path = "../../chain/cosmos" } graph-chain-substreams = { path = "../../chain/substreams" } git-testament = "0.2.5" diff --git a/server/index-node/src/resolver.rs b/server/index-node/src/resolver.rs index 6603d296509..a60e5d35fd9 100644 --- a/server/index-node/src/resolver.rs +++ b/server/index-node/src/resolver.rs @@ -523,23 +523,6 @@ impl IndexNodeResolver { ) .await? } - BlockchainKind::Cosmos => { - let unvalidated_subgraph_manifest = - UnvalidatedSubgraphManifest::::resolve( - deployment_hash.clone(), - raw_yaml, - &self.link_resolver, - &self.logger, - max_spec_version, - ) - .await?; - - Self::validate_and_extract_features( - &self.store.subgraph_store(), - unvalidated_subgraph_manifest, - ) - .await? - } BlockchainKind::Near => { let unvalidated_subgraph_manifest = UnvalidatedSubgraphManifest::::resolve( @@ -698,7 +681,6 @@ impl IndexNodeResolver { // so this seems like the next best thing. try_resolve_for_chain!(graph_chain_ethereum::Chain); try_resolve_for_chain!(graph_chain_arweave::Chain); - try_resolve_for_chain!(graph_chain_cosmos::Chain); try_resolve_for_chain!(graph_chain_near::Chain); // If you're adding support for a new chain and this `match` clause just @@ -710,7 +692,6 @@ impl IndexNodeResolver { BlockchainKind::Substreams | BlockchainKind::Arweave | BlockchainKind::Ethereum - | BlockchainKind::Cosmos | BlockchainKind::Near => (), } diff --git a/substreams/substreams-trigger-filter/build.rs b/substreams/substreams-trigger-filter/build.rs index f29e8ecf091..3bb9182d874 100644 --- a/substreams/substreams-trigger-filter/build.rs +++ b/substreams/substreams-trigger-filter/build.rs @@ -11,7 +11,6 @@ fn main() { // "graph_chain_ethereum::codec::pbcodec", // ) // .extern_path(".sf.arweave.type.v1", "graph_chain_arweave::codec::pbcodec") - // .extern_path(".sf.cosmos.type.v1", "graph_chain_cosmos::codec") .out_dir("src/pb") .compile(&["proto/receipts.proto"], &["proto"]) .expect("Failed to compile Substreams entity proto(s)"); From 0356d93845ce69d88a176fc8903d7f6c3ffbc33c Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Tue, 18 Feb 2025 09:38:43 -0800 Subject: [PATCH 043/150] runtime: Remove derive macros only used by Cosmos --- runtime/derive/src/generate_array_type.rs | 78 ------ runtime/derive/src/generate_asc_type.rs | 172 ------------- runtime/derive/src/generate_from_rust_type.rs | 228 ------------------ .../derive/src/generate_network_type_id.rs | 54 ----- runtime/derive/src/lib.rs | 122 +--------- runtime/test/src/test_padding.rs | 218 ----------------- 6 files changed, 1 insertion(+), 871 deletions(-) delete mode 100644 runtime/derive/src/generate_array_type.rs delete mode 100644 runtime/derive/src/generate_asc_type.rs delete mode 100644 runtime/derive/src/generate_from_rust_type.rs delete mode 100644 runtime/derive/src/generate_network_type_id.rs diff --git a/runtime/derive/src/generate_array_type.rs b/runtime/derive/src/generate_array_type.rs deleted file mode 100644 index 1e674c182c7..00000000000 --- a/runtime/derive/src/generate_array_type.rs +++ /dev/null @@ -1,78 +0,0 @@ -use proc_macro::TokenStream; -use proc_macro2::{Ident, Span}; -use quote::quote; -use syn::{self, parse_macro_input, ItemStruct}; - -pub fn generate_array_type(metadata: TokenStream, input: TokenStream) -> TokenStream { - let item_struct = parse_macro_input!(input as ItemStruct); - let name = item_struct.ident.clone(); - - let asc_name = Ident::new(&format!("Asc{}", name), Span::call_site()); - let asc_name_array = Ident::new(&format!("Asc{}Array", name), Span::call_site()); - - let args = { - let mut args = Vec::new(); - let parser = syn::meta::parser(|meta| { - if let Some(ident) = meta.path.get_ident() { - args.push(ident.to_string()); - } - Ok(()) - }); - parse_macro_input!(metadata with parser); - args - }; - - assert!( - !args.is_empty(), - "arguments not found! generate_array_type()" - ); - - let no_asc_name = if name.to_string().to_uppercase().starts_with("ASC") { - name.to_string()[3..].to_owned() - } else { - name.to_string() - }; - - let index_asc_type_id_array = format!("{}{}Array", args[0], no_asc_name) - .parse::() - .unwrap(); - - quote! { - #item_struct - - #[automatically_derived] - pub struct #asc_name_array(pub graph_runtime_wasm::asc_abi::class::Array>); - - impl graph::runtime::ToAscObj<#asc_name_array> for Vec<#name> { - fn to_asc_obj( - &self, - heap: &mut H, - gas: &graph::runtime::gas::GasCounter, - ) -> Result<#asc_name_array, graph::runtime::HostExportError> { - let content: Result, _> = self.iter().map(|x| graph::runtime::asc_new(heap, x, gas)).collect(); - - Ok(#asc_name_array(graph_runtime_wasm::asc_abi::class::Array::new(&content?, heap, gas)?)) - } - } - - impl graph::runtime::AscType for #asc_name_array { - fn to_asc_bytes(&self) -> Result, graph::runtime::DeterministicHostError> { - self.0.to_asc_bytes() - } - - fn from_asc_bytes( - asc_obj: &[u8], - api_version: &graph::semver::Version, - ) -> Result { - Ok(Self(graph_runtime_wasm::asc_abi::class::Array::from_asc_bytes(asc_obj, api_version)?)) - } - } - - #[automatically_derived] - impl graph::runtime::AscIndexId for #asc_name_array { - const INDEX_ASC_TYPE_ID: graph::runtime::IndexForAscTypeId = graph::runtime::IndexForAscTypeId::#index_asc_type_id_array ; - } - - } - .into() -} diff --git a/runtime/derive/src/generate_asc_type.rs b/runtime/derive/src/generate_asc_type.rs deleted file mode 100644 index 0d79f08482c..00000000000 --- a/runtime/derive/src/generate_asc_type.rs +++ /dev/null @@ -1,172 +0,0 @@ -use proc_macro::TokenStream; -use proc_macro2::{Ident, Span}; -use quote::quote; -use syn::{self, parse_macro_input, Field, ItemStruct}; - -pub fn generate_asc_type(metadata: TokenStream, input: TokenStream) -> TokenStream { - let item_struct = parse_macro_input!(input as ItemStruct); - let args = parse_macro_input!(metadata as super::Args); - - let name = item_struct.ident.clone(); - let asc_name = Ident::new(&format!("Asc{}", name), Span::call_site()); - - let enum_names = args - .vars - .iter() - .filter(|f| f.ident != super::REQUIRED_IDENT_NAME) - .map(|f| f.ident.to_string()) - .collect::>(); - - //struct's fields -> need to skip enum fields - let mut fields = item_struct - .fields - .iter() - .filter(|f| !enum_names.contains(&f.ident.as_ref().unwrap().to_string())) - .collect::>(); - - //extend fields list with enum's variants - args.vars - .iter() - .filter(|f| f.ident != super::REQUIRED_IDENT_NAME) - .flat_map(|f| f.fields.named.iter()) - .for_each(|f| fields.push(f)); - - let m_fields: Vec = fields - .iter() - .map(|f| { - let fld_name = f.ident.clone().unwrap(); - let typ = field_type_map(field_type(f)); - let fld_type = typ.parse::().unwrap(); - - quote! { - pub #fld_name : #fld_type , - } - }) - .collect(); - - let expanded = quote! { - - #item_struct - - #[automatically_derived] - - #[repr(C)] - #[derive(graph_runtime_derive::AscType)] - #[derive(Debug, Default)] - pub struct #asc_name { - #(#m_fields)* - } - }; - - expanded.into() -} - -fn is_scalar(nm: &str) -> bool { - match nm { - "i8" | "u8" => true, - "i16" | "u16" => true, - "i32" | "u32" => true, - "i64" | "u64" => true, - "usize" | "isize" => true, - "bool" => true, - _ => false, - } -} - -fn field_type_map(tp: String) -> String { - if is_scalar(&tp) { - tp - } else { - match tp.as_ref() { - "String" => "graph_runtime_wasm::asc_abi::class::AscString".into(), - _ => tp.clone(), - } - } -} - -fn field_type(fld: &syn::Field) -> String { - if let syn::Type::Path(tp) = &fld.ty { - if let Some(ps) = tp.path.segments.last() { - let name = ps.ident.to_string(); - //TODO - this must be optimized - match name.as_ref() { - "Vec" => match &ps.arguments { - syn::PathArguments::AngleBracketed(v) => { - if let syn::GenericArgument::Type(syn::Type::Path(p)) = &v.args[0] { - let nm = path_to_string(&p.path); - - match nm.as_ref(){ - "u8" => "graph::runtime::AscPtr".to_owned(), - "Vec" => "graph::runtime::AscPtr".to_owned(), - "String" => "graph::runtime::AscPtr>>".to_owned(), - _ => format!("graph::runtime::AscPtr", path_to_string(&p.path)) - } - } else { - name - } - } - - syn::PathArguments::None => name, - syn::PathArguments::Parenthesized(_v) => { - panic!("syn::PathArguments::Parenthesized is not implemented") - } - }, - "Option" => match &ps.arguments { - syn::PathArguments::AngleBracketed(v) => { - if let syn::GenericArgument::Type(syn::Type::Path(p)) = &v.args[0] { - let tp_nm = path_to_string(&p.path); - if is_scalar(&tp_nm) { - format!("Option<{}>", tp_nm) - } else { - format!("graph::runtime::AscPtr", tp_nm) - } - } else { - name - } - } - - syn::PathArguments::None => name, - syn::PathArguments::Parenthesized(_v) => { - panic!("syn::PathArguments::Parenthesized is not implemented") - } - }, - "String" => { - //format!("graph::runtime::AscPtr", name) - "graph::runtime::AscPtr" - .to_owned() - } - - _ => { - if is_scalar(&name) { - name - } else { - format!("graph::runtime::AscPtr", name) - } - } - } - } else { - "N/A".into() - } - } else { - "N/A".into() - } -} - -//recursive -fn path_to_string(path: &syn::Path) -> String { - if let Some(ps) = path.segments.last() { - let nm = ps.ident.to_string(); - - if let syn::PathArguments::AngleBracketed(v) = &ps.arguments { - if let syn::GenericArgument::Type(syn::Type::Path(p)) = &v.args[0] { - format!("{}<{}>", nm, path_to_string(&p.path)) - } else { - nm - } - } else { - nm - } - } else { - panic!("path_to_string - can't get last segment!") - } -} diff --git a/runtime/derive/src/generate_from_rust_type.rs b/runtime/derive/src/generate_from_rust_type.rs deleted file mode 100644 index 6e24ad78c8c..00000000000 --- a/runtime/derive/src/generate_from_rust_type.rs +++ /dev/null @@ -1,228 +0,0 @@ -use proc_macro::TokenStream; -use proc_macro2::{Ident, Span}; -use quote::quote; -use syn::{self, parse_macro_input, Field, ItemStruct}; - -pub fn generate_from_rust_type(metadata: TokenStream, input: TokenStream) -> TokenStream { - let item_struct = parse_macro_input!(input as ItemStruct); - let args = parse_macro_input!(metadata as super::Args); - - let enum_names = args - .vars - .iter() - .filter(|f| f.ident != super::REQUIRED_IDENT_NAME) - .map(|f| f.ident.to_string()) - .collect::>(); - - let required_flds = args - .vars - .iter() - .filter(|f| f.ident == super::REQUIRED_IDENT_NAME) - .flat_map(|f| f.fields.named.iter()) - .map(|f| f.ident.as_ref().unwrap().to_string()) - .collect::>(); - - //struct's standard fields - let fields = item_struct - .fields - .iter() - .filter(|f| { - let nm = f.ident.as_ref().unwrap().to_string(); - !enum_names.contains(&nm) && !nm.starts_with('_') - }) - .collect::>(); - - //struct's enum fields - let enum_fields = item_struct - .fields - .iter() - .filter(|f| enum_names.contains(&f.ident.as_ref().unwrap().to_string())) - .collect::>(); - - //module name - let mod_name = Ident::new( - &format!("__{}__", item_struct.ident.to_string().to_lowercase()), - item_struct.ident.span(), - ); - - let name = item_struct.ident.clone(); - let asc_name = Ident::new(&format!("Asc{}", name), Span::call_site()); - - //generate enum fields validator - let enum_validation = enum_fields.iter().map(|f|{ - let fld_name = f.ident.as_ref().unwrap(); //empty, maybe call it "sum"? - let type_nm = format!("\"{}\"", name).parse::().unwrap(); - let fld_nm = format!("\"{}\"", fld_name).parse::().unwrap(); - - quote! { - let #fld_name = self.#fld_name.as_ref() - .ok_or_else(|| graph::runtime::HostExportError::from(graph::runtime::DeterministicHostError::from(anyhow::anyhow!("{} missing {}", #type_nm, #fld_nm))))?; - } - }); - - let mut methods:Vec = - fields.iter().map(|f| { - let fld_name = f.ident.as_ref().unwrap(); - let self_ref = - if is_byte_array(f){ - quote! { graph_runtime_wasm::asc_abi::class::Bytes(&self.#fld_name) } - }else{ - quote!{ self.#fld_name } - }; - - let is_required = is_required(f, &required_flds); - - let setter = - if is_nullable(f) { - if is_required{ - let type_nm = format!("\"{}\"", name).parse::().unwrap(); - let fld_nm = format!("\"{}\"", fld_name).parse::().unwrap(); - - quote! { - #fld_name: graph::runtime::asc_new_or_missing(heap, &#self_ref, gas, #type_nm, #fld_nm)?, - } - }else{ - quote! { - #fld_name: graph::runtime::asc_new_or_null(heap, &#self_ref, gas)?, - } - } - } else if is_scalar(&field_type(f)){ - quote!{ - #fld_name: #self_ref, - } - }else{ - quote! { - #fld_name: graph::runtime::asc_new(heap, &#self_ref, gas)?, - } - }; - setter - }) - .collect(); - - for var in args.vars { - let var_nm = var.ident.to_string(); - if var_nm == super::REQUIRED_IDENT_NAME { - continue; - } - - let mut c = var_nm.chars(); - let var_type_name = c.next().unwrap().to_uppercase().collect::() + c.as_str(); - - var.fields.named.iter().map(|f|{ - - let fld_nm = f.ident.as_ref().unwrap(); - let var_nm = var.ident.clone(); - - use heck::{ToUpperCamelCase, ToSnakeCase}; - - let varian_type_name = fld_nm.to_string().to_upper_camel_case(); - let mod_name = item_struct.ident.to_string().to_snake_case(); - let varian_type_name = format!("{}::{}::{}",mod_name, var_type_name, varian_type_name).parse::().unwrap(); - - if is_byte_array(f){ - quote! { - #fld_nm: if let #varian_type_name(v) = #var_nm {graph::runtime::asc_new(heap, &graph_runtime_wasm::asc_abi::class::Bytes(v), gas)? } else {graph::runtime::AscPtr::null()}, - } - }else{ - quote! { - #fld_nm: if let #varian_type_name(v) = #var_nm {graph::runtime::asc_new(heap, v, gas)? } else {graph::runtime::AscPtr::null()}, - } - } - }) - .for_each(|ts| methods.push(ts)); - } - - let expanded = quote! { - #item_struct - - #[automatically_derived] - mod #mod_name{ - use super::*; - - use crate::protobuf::*; - - impl graph::runtime::ToAscObj<#asc_name> for #name { - - #[allow(unused_variables)] - fn to_asc_obj( - &self, - heap: &mut H, - gas: &graph::runtime::gas::GasCounter, - ) -> Result<#asc_name, graph::runtime::HostExportError> { - - #(#enum_validation)* - - Ok( - #asc_name { - #(#methods)* - ..Default::default() - } - ) - } - } - } // -------- end of mod - - - }; - - expanded.into() -} - -fn is_scalar(fld: &str) -> bool { - match fld { - "i8" | "u8" => true, - "i16" | "u16" => true, - "i32" | "u32" => true, - "i64" | "u64" => true, - "usize" | "isize" => true, - "bool" => true, - _ => false, - } -} - -fn field_type(fld: &syn::Field) -> String { - if let syn::Type::Path(tp) = &fld.ty { - if let Some(ps) = tp.path.segments.last() { - ps.ident.to_string() - } else { - "N/A".into() - } - } else { - "N/A".into() - } -} - -fn is_required(fld: &syn::Field, req_list: &[String]) -> bool { - let fld_name = fld.ident.as_ref().unwrap().to_string(); - req_list.iter().any(|r| r == &fld_name) -} - -fn is_nullable(fld: &syn::Field) -> bool { - if let syn::Type::Path(tp) = &fld.ty { - if let Some(last) = tp.path.segments.last() { - return last.ident == "Option"; - } - } - false -} - -fn is_byte_array(fld: &syn::Field) -> bool { - if let syn::Type::Path(tp) = &fld.ty { - if let Some(last) = tp.path.segments.last() { - if last.ident == "Vec" { - if let syn::PathArguments::AngleBracketed(ref v) = last.arguments { - if let Some(last) = v.args.last() { - if let syn::GenericArgument::Type(t) = last { - if let syn::Type::Path(p) = t { - if let Some(a) = p.path.segments.last() { - return a.ident == "u8"; - } - } - } - } - } - } - } - } - false -} diff --git a/runtime/derive/src/generate_network_type_id.rs b/runtime/derive/src/generate_network_type_id.rs deleted file mode 100644 index 15a586fa6f1..00000000000 --- a/runtime/derive/src/generate_network_type_id.rs +++ /dev/null @@ -1,54 +0,0 @@ -use proc_macro::TokenStream; -use proc_macro2::{Ident, Span}; -use quote::quote; -use syn::{self, parse_macro_input, ItemStruct}; - -pub fn generate_network_type_id(metadata: TokenStream, input: TokenStream) -> TokenStream { - let item_struct = parse_macro_input!(input as ItemStruct); - let name = item_struct.ident.clone(); - - let asc_name = if name.to_string().to_uppercase().starts_with("ASC") { - name.clone() - } else { - Ident::new(&format!("Asc{}", name), Span::call_site()) - }; - - let no_asc_name = if name.to_string().to_uppercase().starts_with("ASC") { - name.to_string()[3..].to_owned() - } else { - name.to_string() - }; - - let args = { - let mut args = Vec::new(); - let parser = syn::meta::parser(|meta| { - if let Some(ident) = meta.path.get_ident() { - args.push(ident.to_string()); - } - Ok(()) - }); - parse_macro_input!(metadata with parser); - args - }; - - assert!( - !args.is_empty(), - "arguments not found! generate_network_type_id()" - ); - - //type_id variant name - let index_asc_type_id = format!("{}{}", args[0], no_asc_name) - .parse::() - .unwrap(); - - let expanded = quote! { - #item_struct - - #[automatically_derived] - impl graph::runtime::AscIndexId for #asc_name { - const INDEX_ASC_TYPE_ID: graph::runtime::IndexForAscTypeId = graph::runtime::IndexForAscTypeId::#index_asc_type_id ; - } - }; - - expanded.into() -} diff --git a/runtime/derive/src/lib.rs b/runtime/derive/src/lib.rs index 3974ea343b5..6238797ce50 100644 --- a/runtime/derive/src/lib.rs +++ b/runtime/derive/src/lib.rs @@ -4,127 +4,7 @@ extern crate proc_macro; use proc_macro::TokenStream; use quote::quote; -use syn::{ - parse::{Parse, ParseStream}, - Fields, FieldsNamed, Ident, Item, ItemEnum, ItemStruct, Token, -}; - -const REQUIRED_IDENT_NAME: &str = "__required__"; - -struct Args { - vars: Vec, -} - -struct ArgsField { - ident: Ident, - fields: FieldsNamed, -} - -impl Parse for Args { - fn parse(input: ParseStream) -> syn::Result { - let mut idents = Vec::::new(); - - while input.peek(syn::Ident) { - let ident = input.call(Ident::parse)?; - idents.push(ArgsField { - ident, - fields: input.call(FieldsNamed::parse)?, - }); - let _: Option = input.parse()?; - } - - Ok(Args { vars: idents }) - } -} - -//generates graph::runtime::ToAscObj implementation for the type -//takes optional optional list of required fields '__required__{name:TypeName}' and enumerations field decraration with types, i.e. sum{single: ModeInfoSingle,multi: ModeInfoMulti} -//intended use is in build.rs with tonic_build's type_attribute(<...>, <...>) to generate type implementation of graph::runtime::ToAscObj -//Annotation example: -//#[graph_runtime_derive::generate_from_rust_type(...)] -// pub struct MyMessageType { -// .. -// } -//the above annotation will produce following implementation -// impl graph::runtime::ToAscObj for MyMessageType { -// ... -// } -mod generate_from_rust_type; -#[proc_macro_attribute] -pub fn generate_from_rust_type(args: TokenStream, input: TokenStream) -> TokenStream { - generate_from_rust_type::generate_from_rust_type(args, input) -} - -//generates graph::runtime::AscIndexId implementation for the type -//takes required network name attribute to form variant name graph::runtime::IndexForAscTypeId::+ -//Annotation example: -//#[graph_runtime_derive::generate_network_type_id(Cosmos)] -// pub struct MyMessageType { -// .. -// } -//the above annotation will produce following implementation -// impl graph::runtime::AscIndexId for AscMyMessageType { -// const INDEX_ASC_TYPE_ID: graph::runtime::IndexForAscTypeId = graph::runtime::IndexForAscTypeId::CosmosMyMessageType ; -// } - -mod generate_network_type_id; -#[proc_macro_attribute] -pub fn generate_network_type_id(args: TokenStream, input: TokenStream) -> TokenStream { - generate_network_type_id::generate_network_type_id(args, input) -} - -//generates AscType for Type. Takes optional list of non-optional field+type -//Annotation example: -//#[graph_runtime_derive::generate_asc_type(non-optional-field-name: non-optional-field-type,...)] -// pub struct MyMessageType { -// .. -// } -//the above annotation will produce following implementation -// #[repr(C)] -// #[derive(graph_runtime_derive::AscType)] -// #[derive(Debug, Default)] -// pub struct AscMyMessageType { -// ... -// } -// -//Note: this macro makes heavy reliance on types to be available via crate::protobuf (network chain crate root/src/protobuf/lib.rs) -//please see usage exmple in chain::cosmos crate... lib.rs imports generates protobuf bindings, as well as any other needed types -mod generate_asc_type; -#[proc_macro_attribute] -pub fn generate_asc_type(args: TokenStream, input: TokenStream) -> TokenStream { - generate_asc_type::generate_asc_type(args, input) -} - -//generates array type for a type. -//Annotation example: -// #[graph_runtime_derive::generate_array_type(>)] -// pub struct MyMessageType { -// .. -// } -//the above annoation will generate code for MyMessageType type -//Example: -// pub struct AscMyMessageTypeArray(pub graph_runtime_wasm::asc_abi::class::Array>) -//where "AscMyMessageTypeArray" is an array type for "AscMyMessageType" (AscMyMessageType is generated by asc_type derive macro above) -//Macro, also, will generate code for the following 3 trait implementations -//1. graph::runtime::ToAscObj trait -//Example: -// impl graph::runtime::ToAscObj for Vec { -// ... -// } -//2. graph::runtime::AscType -//Example: -// impl graph::runtime::AscType for AscMyMessageTypeArray { -// ... -// } -//3. graph::runtime::AscIndexId (adding expected >Array (CosmosMyMessageTypeArray) variant to graph::runtime::IndexForAscTypeId is manual step) -//impl graph::runtime::AscIndexId for MyMessageTypeArray { -// const INDEX_ASC_TYPE_ID: graph::runtime::IndexForAscTypeId = graph::runtime::IndexForAscTypeId::CosmosMyMessageTypeArray ; -//} -mod generate_array_type; -#[proc_macro_attribute] -pub fn generate_array_type(args: TokenStream, input: TokenStream) -> TokenStream { - generate_array_type::generate_array_type(args, input) -} +use syn::{Fields, Item, ItemEnum, ItemStruct}; #[proc_macro_derive(AscType)] pub fn asc_type_derive(input: TokenStream) -> TokenStream { diff --git a/runtime/test/src/test_padding.rs b/runtime/test/src/test_padding.rs index ae244be67b3..a68f27f8c61 100644 --- a/runtime/test/src/test_padding.rs +++ b/runtime/test/src/test_padding.rs @@ -17,56 +17,6 @@ fn rnd_sub_graph_name(size: usize) -> String { } pub mod data { - #[graph_runtime_derive::generate_asc_type()] - #[graph_runtime_derive::generate_network_type_id(UnitTestNetwork)] - #[graph_runtime_derive::generate_from_rust_type()] - #[graph_runtime_derive::generate_array_type(UnitTestNetwork)] - #[derive(Debug, PartialEq)] - pub struct UnitTestTypeBool { - pub str_pref: String, - pub under_test: bool, - pub str_suff: String, - pub large: i64, - pub tail: bool, - } - - #[graph_runtime_derive::generate_asc_type()] - #[graph_runtime_derive::generate_network_type_id(UnitTestNetwork)] - #[graph_runtime_derive::generate_from_rust_type()] - #[graph_runtime_derive::generate_array_type(UnitTestNetwork)] - #[derive(Debug, PartialEq)] - pub struct UnitTestTypeI8 { - pub str_pref: String, - pub under_test: i8, - pub str_suff: String, - pub large: i64, - pub tail: bool, - } - #[graph_runtime_derive::generate_asc_type()] - #[graph_runtime_derive::generate_network_type_id(UnitTestNetwork)] - #[graph_runtime_derive::generate_from_rust_type()] - #[graph_runtime_derive::generate_array_type(UnitTestNetwork)] - #[derive(Debug, PartialEq)] - pub struct UnitTestTypeU16 { - pub str_pref: String, - pub under_test: u16, - pub str_suff: String, - pub large: i64, - pub tail: bool, - } - #[graph_runtime_derive::generate_asc_type()] - #[graph_runtime_derive::generate_network_type_id(UnitTestNetwork)] - #[graph_runtime_derive::generate_from_rust_type()] - #[graph_runtime_derive::generate_array_type(UnitTestNetwork)] - #[derive(Debug, PartialEq)] - pub struct UnitTestTypeU32 { - pub str_pref: String, - pub under_test: u32, - pub str_suff: String, - pub large: i64, - pub tail: bool, - } - pub struct Bad { pub nonce: u64, pub str_suff: String, @@ -212,46 +162,6 @@ async fn test_v4_manual_padding_should_fail() { manual_padding_should_fail(super::test::API_VERSION_0_0_4).await } -#[tokio::test] -async fn test_v5_bool_padding_ok() { - bool_padding_ok(super::test::API_VERSION_0_0_5).await -} - -#[tokio::test] -async fn test_v4_bool_padding_ok() { - bool_padding_ok(super::test::API_VERSION_0_0_4).await -} - -#[tokio::test] -async fn test_v5_i8_padding_ok() { - i8_padding_ok(super::test::API_VERSION_0_0_5).await -} - -#[tokio::test] -async fn test_v4_i8_padding_ok() { - i8_padding_ok(super::test::API_VERSION_0_0_4).await -} - -#[tokio::test] -async fn test_v5_u16_padding_ok() { - u16_padding_ok(super::test::API_VERSION_0_0_5).await -} - -#[tokio::test] -async fn test_v4_u16_padding_ok() { - u16_padding_ok(super::test::API_VERSION_0_0_4).await -} - -#[tokio::test] -async fn test_v5_u32_padding_ok() { - u32_padding_ok(super::test::API_VERSION_0_0_5).await -} - -#[tokio::test] -async fn test_v4_u32_padding_ok() { - u32_padding_ok(super::test::API_VERSION_0_0_4).await -} - async fn manual_padding_should_fail(api_version: semver::Version) { let mut instance = super::test::test_module( &rnd_sub_graph_name(12), @@ -314,131 +224,3 @@ async fn manual_padding_manualy_fixed_ok(api_version: semver::Version) { assert!(res.is_ok(), "{:?}", res.err()); } - -async fn bool_padding_ok(api_version: semver::Version) { - let mut instance = super::test::test_module( - &rnd_sub_graph_name(12), - super::common::mock_data_source( - &super::test::wasm_file_path(WASM_FILE_NAME, api_version.clone()), - api_version.clone(), - ), - api_version, - ) - .await; - - let parm = protobuf::UnitTestTypeBool { - str_pref: "pref".into(), - under_test: true, - str_suff: "suff".into(), - large: i64::MAX, - tail: true, - }; - - let new_obj = instance.asc_new(&parm).unwrap(); - - let func = instance - .get_func("test_padding_bool") - .typed(&mut instance.store.as_context_mut()) - .unwrap() - .clone(); - - let res: Result<(), _> = func.call(&mut instance.store.as_context_mut(), new_obj.wasm_ptr()); - - assert!(res.is_ok(), "{:?}", res.err()); -} - -async fn i8_padding_ok(api_version: semver::Version) { - let mut instance = super::test::test_module( - &rnd_sub_graph_name(12), - super::common::mock_data_source( - &super::test::wasm_file_path(WASM_FILE_NAME, api_version.clone()), - api_version.clone(), - ), - api_version, - ) - .await; - - let parm = protobuf::UnitTestTypeI8 { - str_pref: "pref".into(), - under_test: i8::MAX, - str_suff: "suff".into(), - large: i64::MAX, - tail: true, - }; - - let new_obj = instance.asc_new(&parm).unwrap(); - - let func = instance - .get_func("test_padding_i8") - .typed(&mut instance.store.as_context_mut()) - .unwrap() - .clone(); - - let res: Result<(), _> = func.call(&mut instance.store.as_context_mut(), new_obj.wasm_ptr()); - - assert!(res.is_ok(), "{:?}", res.err()); -} - -async fn u16_padding_ok(api_version: semver::Version) { - let mut instance = super::test::test_module( - &rnd_sub_graph_name(12), - super::common::mock_data_source( - &super::test::wasm_file_path(WASM_FILE_NAME, api_version.clone()), - api_version.clone(), - ), - api_version, - ) - .await; - - let parm = protobuf::UnitTestTypeU16 { - str_pref: "pref".into(), - under_test: i16::MAX as u16, - str_suff: "suff".into(), - large: i64::MAX, - tail: true, - }; - - let new_obj = instance.asc_new(&parm).unwrap(); - - let func = instance - .get_func("test_padding_i16") - .typed(&mut instance.store.as_context_mut()) - .unwrap() - .clone(); - - let res: Result<(), _> = func.call(&mut instance.store.as_context_mut(), new_obj.wasm_ptr()); - - assert!(res.is_ok(), "{:?}", res.err()); -} - -async fn u32_padding_ok(api_version: semver::Version) { - let mut instance = super::test::test_module( - &rnd_sub_graph_name(12), - super::common::mock_data_source( - &super::test::wasm_file_path(WASM_FILE_NAME, api_version.clone()), - api_version.clone(), - ), - api_version, - ) - .await; - - let parm = protobuf::UnitTestTypeU32 { - str_pref: "pref".into(), - under_test: i32::MAX as u32, - str_suff: "suff".into(), - large: i64::MAX, - tail: true, - }; - - let new_obj = instance.asc_new(&parm).unwrap(); - - let func = instance - .get_func("test_padding_i32") - .typed(&mut instance.store.as_context_mut()) - .unwrap() - .clone(); - - let res: Result<(), _> = func.call(&mut instance.store.as_context_mut(), new_obj.wasm_ptr()); - - assert!(res.is_ok(), "{:?}", res.err()); -} From 1fcf5a5b8ef8ba9862f5c750c7cd61650b787e8d Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Tue, 18 Feb 2025 10:20:56 -0800 Subject: [PATCH 044/150] Cargo.toml: Spell out workspace members instead of using * imports --- Cargo.toml | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index c19d6939f01..b3cd092114f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,7 +4,11 @@ members = [ "core", "core/graphman", "core/graphman_store", - "chain/*", + "chain/arweave", + "chain/common", + "chain/ethereum", + "chain/near", + "chain/substreams", "graphql", "node", "runtime/derive", @@ -43,7 +47,15 @@ bs58 = "0.5.1" chrono = "0.4.38" clap = { version = "4.5.4", features = ["derive", "env"] } derivative = "2.2.0" -diesel = { version = "2.2.4", features = ["postgres", "serde_json", "numeric", "r2d2", "chrono", "uuid", "i-implement-a-third-party-backend-and-opt-into-breaking-changes"] } +diesel = { version = "2.2.4", features = [ + "postgres", + "serde_json", + "numeric", + "r2d2", + "chrono", + "uuid", + "i-implement-a-third-party-backend-and-opt-into-breaking-changes", +] } diesel-derive-enum = { version = "2.1.0", features = ["postgres"] } diesel-dynamic-schema = { version = "0.2.1", features = ["postgres"] } diesel_derives = "2.1.4" @@ -65,7 +77,10 @@ serde_derive = "1.0.125" serde_json = { version = "1.0", features = ["arbitrary_precision"] } serde_regex = "1.1.0" serde_yaml = "0.9.21" -slog = { version = "2.7.0", features = ["release_max_level_trace", "max_level_trace"] } +slog = { version = "2.7.0", features = [ + "release_max_level_trace", + "max_level_trace", +] } sqlparser = "0.46.0" strum = { version = "0.26", features = ["derive"] } syn = { version = "2.0.87", features = ["full"] } From 91168d603f4e144bd8364ea2913d16101e1e8082 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Sat, 1 Feb 2025 16:51:36 -0800 Subject: [PATCH 045/150] graph, store: Use a much larger fetch_size for fdw Changing the fetch_size to 10,000 from its default of 100 will have a dramatic effect on the time copy operations take. The fetch_size can be controlled with the environment variable GRAPH_STORE_FDW_FETCH_SIZE --- graph/src/env/store.rs | 6 ++++++ store/postgres/src/connection_pool.rs | 14 ++++++++++++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/graph/src/env/store.rs b/graph/src/env/store.rs index 148e5641088..3b4e50ec87d 100644 --- a/graph/src/env/store.rs +++ b/graph/src/env/store.rs @@ -126,6 +126,9 @@ pub struct EnvVarsStore { /// used to work around Postgres errors complaining 'number of /// parameters must be between 0 and 65535' when inserting entities pub insert_extra_cols: usize, + /// The number of rows to fetch from the foreign data wrapper in one go, + /// this will be set as the option 'fetch_size' on all foreign servers + pub fdw_fetch_size: usize, } // This does not print any values avoid accidentally leaking any sensitive env vars @@ -175,6 +178,7 @@ impl From for EnvVarsStore { disable_block_cache_for_lookup: x.disable_block_cache_for_lookup, last_rollup_from_poi: x.last_rollup_from_poi, insert_extra_cols: x.insert_extra_cols, + fdw_fetch_size: x.fdw_fetch_size, } } } @@ -238,6 +242,8 @@ pub struct InnerStore { last_rollup_from_poi: bool, #[envconfig(from = "GRAPH_STORE_INSERT_EXTRA_COLS", default = "0")] insert_extra_cols: usize, + #[envconfig(from = "GRAPH_STORE_FDW_FETCH_SIZE", default = "10000")] + fdw_fetch_size: usize, } #[derive(Clone, Copy, Debug)] diff --git a/store/postgres/src/connection_pool.rs b/store/postgres/src/connection_pool.rs index ae8ab3e71b6..374a1adc5ab 100644 --- a/store/postgres/src/connection_pool.rs +++ b/store/postgres/src/connection_pool.rs @@ -150,7 +150,11 @@ impl ForeignServer { "\ create server \"{name}\" foreign data wrapper postgres_fdw - options (host '{remote_host}', port '{remote_port}', dbname '{remote_db}', updatable 'false'); + options (host '{remote_host}', \ + port '{remote_port}', \ + dbname '{remote_db}', \ + fetch_size '{fetch_size}', \ + updatable 'false'); create user mapping for current_user server \"{name}\" options (user '{remote_user}', password '{remote_password}');", @@ -160,6 +164,7 @@ impl ForeignServer { remote_db = self.dbname, remote_user = self.user, remote_password = self.password, + fetch_size = ENV_VARS.store.fdw_fetch_size, ); Ok(conn.batch_execute(&query)?) } @@ -178,17 +183,22 @@ impl ForeignServer { let query = format!( "\ alter server \"{name}\" - options (set host '{remote_host}', {set_port} port '{remote_port}', set dbname '{remote_db}'); + options (set host '{remote_host}', \ + {set_port} port '{remote_port}', \ + set dbname '{remote_db}, \ + {set_fetch_size} fetch_size '{fetch_size}'); alter user mapping for current_user server \"{name}\" options (set user '{remote_user}', set password '{remote_password}');", name = self.name, remote_host = self.host, set_port = set_or_add("port"), + set_fetch_size = set_or_add("fetch_size"), remote_port = self.port, remote_db = self.dbname, remote_user = self.user, remote_password = self.password, + fetch_size = ENV_VARS.store.fdw_fetch_size, ); Ok(conn.batch_execute(&query)?) } From a633ae2bd4fbf7ab48f80952a6a33de9c96b5c62 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 5 Mar 2025 19:16:17 +0000 Subject: [PATCH 046/150] build(deps): bump substreams from 0.5.20 to 0.6.0 (#5683) * build(deps): bump substreams from 0.5.20 to 0.6.0 Bumps [substreams](https://github.com/streamingfast/substreams-rs) from 0.5.20 to 0.6.0. - [Release notes](https://github.com/streamingfast/substreams-rs/releases) - [Changelog](https://github.com/streamingfast/substreams-rs/blob/develop/CHANGELOG.md) - [Commits](https://github.com/streamingfast/substreams-rs/compare/v0.5.20...v0.6.0) --- updated-dependencies: - dependency-name: substreams dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * update tonic and prost --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Filipe Azevedo --- Cargo.lock | 683 +++++----- Cargo.toml | 17 +- chain/arweave/build.rs | 2 +- .../src/protobuf/sf.arweave.r#type.v1.rs | 5 - chain/ethereum/build.rs | 2 +- .../src/protobuf/sf.ethereum.r#type.v2.rs | 122 +- chain/near/build.rs | 2 +- chain/near/src/protobuf/receipts.v1.rs | 1 - chain/substreams/build.rs | 2 +- .../src/protobuf/substreams.entity.v1.rs | 14 +- docs/implementation/add-chain.md | 2 +- graph/build.rs | 6 +- graph/src/firehose/endpoints.rs | 2 +- .../src/firehose/sf.ethereum.transform.v1.rs | 11 +- graph/src/firehose/sf.firehose.v2.rs | 276 ++-- graph/src/firehose/sf.near.transform.v1.rs | 2 - graph/src/substreams/sf.substreams.v1.rs | 46 +- graph/src/substreams_rpc/sf.firehose.v2.rs | 242 ++-- .../substreams_rpc/sf.substreams.rpc.v2.rs | 196 ++- .../substreams-trigger-filter/Cargo.toml | 10 +- substreams/substreams-trigger-filter/build.rs | 7 +- .../src/pb/receipts.v1.rs | 1 - .../src/pb/sf.near.type.v1.rs | 1181 ----------------- 23 files changed, 821 insertions(+), 2011 deletions(-) delete mode 100644 substreams/substreams-trigger-filter/src/pb/sf.near.type.v1.rs diff --git a/Cargo.lock b/Cargo.lock index 384e357019a..c49377cce75 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -45,7 +45,7 @@ dependencies = [ "cfg-if 1.0.0", "once_cell", "version_check", - "zerocopy", + "zerocopy 0.7.35", ] [[package]] @@ -210,26 +210,25 @@ dependencies = [ "serde_urlencoded", "static_assertions_next", "tempfile", - "thiserror", + "thiserror 1.0.61", "uuid", ] [[package]] name = "async-graphql-axum" -version = "7.0.11" +version = "7.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9aa80e171205c6d562057fd5a49167c8fbe61f7db2bed6540f6d4f2234d7ff2" +checksum = "6bf2882c816094fef6e39d381b8e9b710e5943e7bdef5198496441d5083164fa" dependencies = [ "async-graphql", - "async-trait", - "axum 0.7.5", + "axum 0.8.1", "bytes", "futures-util", "serde_json", "tokio", "tokio-stream", "tokio-util 0.7.11", - "tower-service 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "tower-service 0.3.3", ] [[package]] @@ -246,7 +245,7 @@ dependencies = [ "quote", "strum", "syn 2.0.87", - "thiserror", + "thiserror 1.0.61", ] [[package]] @@ -337,42 +336,41 @@ checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" [[package]] name = "axum" -version = "0.6.20" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b829e4e32b91e643de6eafe82b1d90675f5874230191a4ffbc1b336dec4d6bf" +checksum = "3a6c9af12842a67734c9a2e355436e5d03b22383ed60cf13cd0c18fbfe3dcbcf" dependencies = [ "async-trait", - "axum-core 0.3.4", - "bitflags 1.3.2", + "axum-core 0.4.3", "bytes", "futures-util", - "http 0.2.12", - "http-body 0.4.6", - "hyper 0.14.29", + "http 1.1.0", + "http-body 1.0.0", + "http-body-util", "itoa", - "matchit", + "matchit 0.7.3", "memchr", "mime", "percent-encoding", "pin-project-lite", "rustversion", "serde", - "sync_wrapper 0.1.2", + "sync_wrapper 1.0.1", "tower 0.4.13 (registry+https://github.com/rust-lang/crates.io-index)", - "tower-layer 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", - "tower-service 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "tower-layer 0.3.3", + "tower-service 0.3.3", ] [[package]] name = "axum" -version = "0.7.5" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a6c9af12842a67734c9a2e355436e5d03b22383ed60cf13cd0c18fbfe3dcbcf" +checksum = "6d6fd624c75e18b3b4c6b9caf42b1afe24437daaee904069137d8bab077be8b8" dependencies = [ - "async-trait", - "axum-core 0.4.3", - "base64 0.21.7", + "axum-core 0.5.0", + "base64 0.22.1", "bytes", + "form_urlencoded", "futures-util", "http 1.1.0", "http-body 1.0.0", @@ -380,7 +378,7 @@ dependencies = [ "hyper 1.6.0", "hyper-util", "itoa", - "matchit", + "matchit 0.8.4", "memchr", "mime", "percent-encoding", @@ -394,36 +392,38 @@ dependencies = [ "sync_wrapper 1.0.1", "tokio", "tokio-tungstenite", - "tower 0.4.13 (registry+https://github.com/rust-lang/crates.io-index)", - "tower-layer 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", - "tower-service 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "tower 0.5.2", + "tower-layer 0.3.3", + "tower-service 0.3.3", "tracing", ] [[package]] name = "axum-core" -version = "0.3.4" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c" +checksum = "a15c63fd72d41492dc4f497196f5da1fb04fb7529e631d73630d1b491e47a2e3" dependencies = [ "async-trait", "bytes", "futures-util", - "http 0.2.12", - "http-body 0.4.6", + "http 1.1.0", + "http-body 1.0.0", + "http-body-util", "mime", + "pin-project-lite", "rustversion", - "tower-layer 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", - "tower-service 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "sync_wrapper 0.1.2", + "tower-layer 0.3.3", + "tower-service 0.3.3", ] [[package]] name = "axum-core" -version = "0.4.3" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a15c63fd72d41492dc4f497196f5da1fb04fb7529e631d73630d1b491e47a2e3" +checksum = "df1362f362fd16024ae199c1970ce98f9661bf5ef94b9808fee734bc3698b733" dependencies = [ - "async-trait", "bytes", "futures-util", "http 1.1.0", @@ -432,9 +432,9 @@ dependencies = [ "mime", "pin-project-lite", "rustversion", - "sync_wrapper 0.1.2", - "tower-layer 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", - "tower-service 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "sync_wrapper 1.0.1", + "tower-layer 0.3.3", + "tower-service 0.3.3", "tracing", ] @@ -642,9 +642,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.6.0" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" +checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" dependencies = [ "serde", ] @@ -802,11 +802,21 @@ dependencies = [ "libc", ] +[[package]] +name = "core-foundation" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b55271e5c8c478ad3f38ad24ef34923091e0548492a266d19b3c0b4d82574c63" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation-sys" -version = "0.8.6" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "core2" @@ -1450,7 +1460,7 @@ dependencies = [ "serde", "serde_json", "sha3", - "thiserror", + "thiserror 1.0.61", "uint 0.9.5", ] @@ -1527,7 +1537,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cfcf0ed7fe52a17a03854ec54a9f76d6d84508d1c0e66bc1793301c73fc8493c" dependencies = [ "byteorder", - "rand", + "rand 0.8.5", "rustc-hex", "static_assertions", ] @@ -1726,7 +1736,19 @@ checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" dependencies = [ "cfg-if 1.0.0", "libc", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", +] + +[[package]] +name = "getrandom" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43a49c392881ce6d5c3b8cb70f98717b7c07aabbdff06687b9030dbfbe2725f8" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "wasi 0.13.3+wasi-0.2.2", + "windows-targets 0.52.6", ] [[package]] @@ -1829,9 +1851,9 @@ dependencies = [ "petgraph", "priority-queue", "prometheus", - "prost 0.12.6", - "prost-types 0.12.6", - "rand", + "prost", + "prost-types", + "rand 0.8.5", "regex", "reqwest", "semver", @@ -1849,7 +1871,7 @@ dependencies = [ "stable-hash 0.3.4", "stable-hash 0.4.4", "strum_macros", - "thiserror", + "thiserror 1.0.61", "tiny-keccak 1.5.0", "tokio", "tokio-retry", @@ -1872,8 +1894,8 @@ dependencies = [ "graph", "graph-runtime-derive", "graph-runtime-wasm", - "prost 0.12.6", - "prost-types 0.12.6", + "prost", + "prost-types", "serde", "sha2", "tonic-build", @@ -1902,8 +1924,8 @@ dependencies = [ "hex", "itertools 0.13.0", "jsonrpc-core", - "prost 0.12.6", - "prost-types 0.12.6", + "prost", + "prost-types", "semver", "serde", "tiny-keccak 1.5.0", @@ -1920,8 +1942,8 @@ dependencies = [ "graph", "graph-runtime-derive", "graph-runtime-wasm", - "prost 0.12.6", - "prost-types 0.12.6", + "prost", + "prost-types", "serde", "tonic-build", "trigger-filters", @@ -1937,8 +1959,8 @@ dependencies = [ "graph-runtime-wasm", "hex", "lazy_static", - "prost 0.12.6", - "prost-types 0.12.6", + "prost", + "prost-types", "semver", "serde", "tokio", @@ -2033,7 +2055,7 @@ dependencies = [ "graph-chain-ethereum", "graph-runtime-derive", "graph-runtime-wasm", - "rand", + "rand 0.8.5", "semver", "test-store", "wasmtime", @@ -2129,7 +2151,7 @@ dependencies = [ "postgres", "postgres-openssl", "pretty_assertions", - "rand", + "rand 0.8.5", "serde", "serde_json", "stable-hash 0.3.4", @@ -2181,7 +2203,7 @@ dependencies = [ "graph-store-postgres", "graphman-store", "itertools 0.13.0", - "thiserror", + "thiserror 1.0.61", "tokio", ] @@ -2192,7 +2214,7 @@ dependencies = [ "anyhow", "async-graphql", "async-graphql-axum", - "axum 0.7.5", + "axum 0.8.1", "chrono", "diesel", "graph", @@ -2205,7 +2227,7 @@ dependencies = [ "serde_json", "slog", "test-store", - "thiserror", + "thiserror 1.0.61", "tokio", "tower-http", ] @@ -2227,7 +2249,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2ebc8013b4426d5b81a4364c419a95ed0b404af2b82e2457de52d9348f0e474" dependencies = [ "combine", - "thiserror", + "thiserror 1.0.61", ] [[package]] @@ -2292,7 +2314,7 @@ dependencies = [ "pest_derive", "serde", "serde_json", - "thiserror", + "thiserror 1.0.61", ] [[package]] @@ -2500,7 +2522,7 @@ dependencies = [ "pin-project-lite", "socket2", "tokio", - "tower-service 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "tower-service 0.3.3", "tracing", "want", ] @@ -2536,24 +2558,25 @@ dependencies = [ "http 1.1.0", "hyper 1.6.0", "hyper-util", - "rustls 0.23.10", - "rustls-native-certs", + "rustls", + "rustls-native-certs 0.7.1", "rustls-pki-types", "tokio", - "tokio-rustls 0.26.0", - "tower-service 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-rustls", + "tower-service 0.3.3", ] [[package]] name = "hyper-timeout" -version = "0.4.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" +checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" dependencies = [ - "hyper 0.14.29", + "hyper 1.6.0", + "hyper-util", "pin-project-lite", "tokio", - "tokio-io-timeout", + "tower-service 0.3.3", ] [[package]] @@ -2569,14 +2592,14 @@ dependencies = [ "native-tls", "tokio", "tokio-native-tls", - "tower-service 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "tower-service 0.3.3", ] [[package]] name = "hyper-util" -version = "0.1.6" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ab92f4f49ee4fb4f997c784b7a2e0fa70050211e0b6a287f898c3c9785ca956" +checksum = "df2dcfbe0677734ab2f3ffa7fa7bfd4706bfdc1ef393f2ee30184aed67e631b4" dependencies = [ "bytes", "futures-channel", @@ -2587,8 +2610,7 @@ dependencies = [ "pin-project-lite", "socket2", "tokio", - "tower 0.4.13 (registry+https://github.com/rust-lang/crates.io-index)", - "tower-service 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "tower-service 0.3.3", "tracing", ] @@ -2623,7 +2645,7 @@ checksum = "d1fcc7f316b2c079dde77564a1360639c1a956a23fa96122732e416cb10717bb" dependencies = [ "cfg-if 1.0.0", "num-traits", - "rand", + "rand 0.8.5", "static_assertions", ] @@ -2873,11 +2895,11 @@ dependencies = [ "jsonrpsee-types", "lazy_static", "parking_lot", - "rand", + "rand 0.8.5", "rustc-hash 1.1.0", "serde", "serde_json", - "thiserror", + "thiserror 1.0.61", "tokio", "tracing", "unicase", @@ -2911,7 +2933,7 @@ dependencies = [ "beef", "serde", "serde_json", - "thiserror", + "thiserror 1.0.61", "tracing", ] @@ -3007,6 +3029,12 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + [[package]] name = "maybe-owned" version = "0.3.4" @@ -3100,7 +3128,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" dependencies = [ "libc", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", "windows-sys 0.48.0", ] @@ -3142,12 +3170,6 @@ dependencies = [ "unsigned-varint 0.7.2", ] -[[package]] -name = "multimap" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" - [[package]] name = "multimap" version = "0.10.0" @@ -3166,7 +3188,7 @@ dependencies = [ "openssl-probe", "openssl-sys", "schannel", - "security-framework", + "security-framework 2.11.0", "security-framework-sys", "tempfile", ] @@ -3271,7 +3293,7 @@ dependencies = [ "parking_lot", "percent-encoding", "quick-xml", - "rand", + "rand 0.8.5", "reqwest", "ring", "rustls-pemfile", @@ -3429,7 +3451,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd53dff83f26735fdc1ca837098ccf133605d794cdae66acfc2bfac3ec809d95" dependencies = [ "memchr", - "thiserror", + "thiserror 1.0.61", "ucd-trie", ] @@ -3573,7 +3595,7 @@ dependencies = [ "hmac", "md-5", "memchr", - "rand", + "rand 0.8.5", "sha2", "stringprep", ] @@ -3620,16 +3642,6 @@ dependencies = [ "yansi", ] -[[package]] -name = "prettyplease" -version = "0.1.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c8646e95016a7a6c4adea95bafa8a16baab64b583356217f2c85db4a39d9a86" -dependencies = [ - "proc-macro2", - "syn 1.0.109", -] - [[package]] name = "prettyplease" version = "0.2.20" @@ -3707,67 +3719,34 @@ dependencies = [ "parking_lot", "protobuf 2.28.0", "reqwest", - "thiserror", + "thiserror 1.0.61", ] [[package]] name = "prost" -version = "0.11.9" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd" +checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" dependencies = [ "bytes", - "prost-derive 0.11.9", -] - -[[package]] -name = "prost" -version = "0.12.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "deb1435c188b76130da55f17a466d252ff7b1418b2ad3e037d127b94e3411f29" -dependencies = [ - "bytes", - "prost-derive 0.12.6", + "prost-derive", ] [[package]] name = "prost-build" -version = "0.11.9" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "119533552c9a7ffacc21e099c24a0ac8bb19c2a2a3f363de84cd9b844feab270" +checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" dependencies = [ - "bytes", - "heck 0.4.1", - "itertools 0.10.5", - "lazy_static", - "log", - "multimap 0.8.3", - "petgraph", - "prettyplease 0.1.25", - "prost 0.11.9", - "prost-types 0.11.9", - "regex", - "syn 1.0.109", - "tempfile", - "which", -] - -[[package]] -name = "prost-build" -version = "0.12.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22505a5c94da8e3b7c2996394d1c933236c4d743e81a410bcca4e6989fc066a4" -dependencies = [ - "bytes", "heck 0.5.0", - "itertools 0.10.5", + "itertools 0.13.0", "log", - "multimap 0.10.0", + "multimap", "once_cell", "petgraph", - "prettyplease 0.2.20", - "prost 0.12.6", - "prost-types 0.12.6", + "prettyplease", + "prost", + "prost-types", "regex", "syn 2.0.87", "tempfile", @@ -3775,25 +3754,12 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.11.9" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4" +checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools 0.10.5", - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "prost-derive" -version = "0.12.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81bddcdb20abf9501610992b6759a4c888aef7d1a7247ef75e2404275ac24af1" -dependencies = [ - "anyhow", - "itertools 0.10.5", + "itertools 0.13.0", "proc-macro2", "quote", "syn 2.0.87", @@ -3801,20 +3767,11 @@ dependencies = [ [[package]] name = "prost-types" -version = "0.11.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "213622a1460818959ac1181aaeb2dc9c7f63df720db7d788b3e24eacd1983e13" -dependencies = [ - "prost 0.11.9", -] - -[[package]] -name = "prost-types" -version = "0.12.6" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9091c90b0a32608e984ff2fa4091273cbdd755d54935c51d520887f4a1dbd5b0" +checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" dependencies = [ - "prost 0.12.6", + "prost", ] [[package]] @@ -3831,7 +3788,7 @@ checksum = "a3a7c64d9bf75b1b8d981124c14c179074e8caa7dfe7b6a12e6222ddcd0c8f72" dependencies = [ "once_cell", "protobuf-support", - "thiserror", + "thiserror 1.0.61", ] [[package]] @@ -3846,7 +3803,7 @@ dependencies = [ "protobuf 3.7.1", "protobuf-support", "tempfile", - "thiserror", + "thiserror 1.0.61", "which", ] @@ -3856,7 +3813,7 @@ version = "3.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b088fd20b938a875ea00843b6faf48579462630015c3788d397ad6a786663252" dependencies = [ - "thiserror", + "thiserror 1.0.61", ] [[package]] @@ -3889,8 +3846,8 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash 1.1.0", - "rustls 0.23.10", - "thiserror", + "rustls", + "thiserror 1.0.61", "tokio", "tracing", ] @@ -3902,12 +3859,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fadfaed2cd7f389d0161bb73eeb07b7b78f8691047a6f3e73caaeae55310a4a6" dependencies = [ "bytes", - "rand", + "rand 0.8.5", "ring", "rustc-hash 2.0.0", - "rustls 0.23.10", + "rustls", "slab", - "thiserror", + "thiserror 1.0.61", "tinyvec", "tracing", ] @@ -3958,8 +3915,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", - "rand_chacha", - "rand_core", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3779b94aeb87e8bd4e834cee3650289ee9e0d5677f976ecdb6d219e5f4f6cd94" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.3", + "zerocopy 0.8.21", ] [[package]] @@ -3969,7 +3937,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.3", ] [[package]] @@ -3978,7 +3956,16 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom", + "getrandom 0.2.15", +] + +[[package]] +name = "rand_core" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" +dependencies = [ + "getrandom 0.3.1", ] [[package]] @@ -4031,9 +4018,9 @@ version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd283d9651eeda4b2a83a43c1c91b266c40fd76ecd39a50a8c630ae69dc72891" dependencies = [ - "getrandom", + "getrandom 0.2.15", "libredox", - "thiserror", + "thiserror 1.0.61", ] [[package]] @@ -4108,8 +4095,8 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.10", - "rustls-native-certs", + "rustls", + "rustls-native-certs 0.7.1", "rustls-pemfile", "rustls-pki-types", "serde", @@ -4119,9 +4106,9 @@ dependencies = [ "system-configuration", "tokio", "tokio-native-tls", - "tokio-rustls 0.26.0", + "tokio-rustls", "tokio-util 0.7.11", - "tower-service 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "tower-service 0.3.3", "url", "wasm-bindgen", "wasm-bindgen-futures", @@ -4138,7 +4125,7 @@ checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" dependencies = [ "cc", "cfg-if 1.0.0", - "getrandom", + "getrandom 0.2.15", "libc", "spin", "untrusted", @@ -4203,11 +4190,12 @@ dependencies = [ [[package]] name = "rustls" -version = "0.22.4" +version = "0.23.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf4ef73721ac7bcd79b2b315da7779d8fc09718c6b3d2d1b2d94850eb8c18432" +checksum = "05cff451f60db80f490f3c182b77c35260baace73209e9cdbbe526bfe3a4d402" dependencies = [ "log", + "once_cell", "ring", "rustls-pki-types", "rustls-webpki", @@ -4216,30 +4204,28 @@ dependencies = [ ] [[package]] -name = "rustls" -version = "0.23.10" +name = "rustls-native-certs" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05cff451f60db80f490f3c182b77c35260baace73209e9cdbbe526bfe3a4d402" +checksum = "a88d6d420651b496bdd98684116959239430022a115c1240e6c3993be0b15fba" dependencies = [ - "once_cell", - "ring", + "openssl-probe", + "rustls-pemfile", "rustls-pki-types", - "rustls-webpki", - "subtle", - "zeroize", + "schannel", + "security-framework 2.11.0", ] [[package]] name = "rustls-native-certs" -version = "0.7.1" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a88d6d420651b496bdd98684116959239430022a115c1240e6c3993be0b15fba" +checksum = "7fcff2dd52b58a8d98a70243663a0d234c4e2b79235637849d15913394a247d3" dependencies = [ "openssl-probe", - "rustls-pemfile", "rustls-pki-types", "schannel", - "security-framework", + "security-framework 3.2.0", ] [[package]] @@ -4254,9 +4240,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.7.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "976295e77ce332211c0d24d92c0e83e50f5c5f046d11082cea19f3df13a3562d" +checksum = "917ce264624a4b4db1c364dcc35bfca9ded014d0a958cd47ad3e960e988ea51c" [[package]] name = "rustls-webpki" @@ -4339,7 +4325,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c627723fd09706bacdb5cf41499e95098555af3c3c29d014dc3c458ef6be11c0" dependencies = [ "bitflags 2.6.0", - "core-foundation", + "core-foundation 0.9.4", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271720403f46ca04f7ba6f55d438f8bd878d6b8ca0a1046e8228c4145bcbb316" +dependencies = [ + "bitflags 2.6.0", + "core-foundation 0.10.0", "core-foundation-sys", "libc", "security-framework-sys", @@ -4347,9 +4346,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.11.0" +version = "2.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "317936bbbd05227752583946b9e66d7ce3b489f84e11a94a510b4437fef407d7" +checksum = "49db231d56a190491cb4aeda9527f1ad45345af50b0851622a7adb8c03b01c32" dependencies = [ "core-foundation-sys", "libc", @@ -4692,7 +4691,7 @@ dependencies = [ "futures 0.3.30", "httparse", "log", - "rand", + "rand 0.8.5", "sha-1", ] @@ -4804,9 +4803,9 @@ dependencies = [ [[package]] name = "substreams" -version = "0.5.20" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "392f77309a4e36d7839d0552a38557b53894200aba239f3d0725ec167ebf4297" +checksum = "5bb63116b90d4c174114fb237a8916dd995c939874f7576333990a44d78b642a" dependencies = [ "anyhow", "bigdecimal 0.3.1", @@ -4818,22 +4817,22 @@ dependencies = [ "pad", "pest", "pest_derive", - "prost 0.11.9", - "prost-build 0.11.9", - "prost-types 0.11.9", + "prost", + "prost-build", + "prost-types", "substreams-macro", - "thiserror", + "thiserror 1.0.61", ] [[package]] name = "substreams-entity-change" -version = "1.3.2" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2c7fca123abff659d15ed30da5b605fa954a29e912c94260c488d0d18f9107d" +checksum = "0587b8d5dd7bffb0415d544c31e742c4cabdb81bbe9a3abfffff125185e4e9e8" dependencies = [ "base64 0.13.1", - "prost 0.11.9", - "prost-types 0.11.9", + "prost", + "prost-types", "substreams", ] @@ -4843,26 +4842,28 @@ version = "0.36.0" [[package]] name = "substreams-macro" -version = "0.5.20" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ccc7137347f05d26c7007dced97b4caef67a13b3d422789d969fe6e4cd8cc4a" +checksum = "f36f36e9da94db29f49daf3ab6b47b529b57c43fc5d58bc35b160aaad1a7233f" dependencies = [ "proc-macro2", "quote", "syn 1.0.109", - "thiserror", + "thiserror 1.0.61", ] [[package]] name = "substreams-near-core" -version = "0.10.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9922f437e6cb86b62cfd8bdede93937def710616ac2825ffff06b8770bbd06df" +checksum = "01ef8a763c5a5604b16f4898ab75d39494ef785c457aaca1fd7761b299f40fbf" dependencies = [ "bs58 0.4.0", - "prost 0.11.9", - "prost-build 0.11.9", - "prost-types 0.11.9", + "getrandom 0.2.15", + "hex", + "prost", + "prost-build", + "prost-types", ] [[package]] @@ -4870,7 +4871,7 @@ name = "substreams-trigger-filter" version = "0.36.0" dependencies = [ "hex", - "prost 0.11.9", + "prost", "substreams", "substreams-entity-change", "substreams-near-core", @@ -4925,7 +4926,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" dependencies = [ "bitflags 1.3.2", - "core-foundation", + "core-foundation 0.9.4", "system-configuration-sys", ] @@ -5013,7 +5014,7 @@ dependencies = [ "hex-literal 0.4.1", "lazy_static", "pretty_assertions", - "prost-types 0.12.6", + "prost-types", ] [[package]] @@ -5022,7 +5023,16 @@ version = "1.0.61" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c546c80d6be4bc6a00c0f01730c08df82eaa7a7a61f11d656526506112cc1709" dependencies = [ - "thiserror-impl", + "thiserror-impl 1.0.61", +] + +[[package]] +name = "thiserror" +version = "2.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708" +dependencies = [ + "thiserror-impl 2.0.12", ] [[package]] @@ -5036,6 +5046,17 @@ dependencies = [ "syn 2.0.87", ] +[[package]] +name = "thiserror-impl" +version = "2.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + [[package]] name = "thread_local" version = "1.1.8" @@ -5129,16 +5150,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "tokio-io-timeout" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf" -dependencies = [ - "pin-project-lite", - "tokio", -] - [[package]] name = "tokio-macros" version = "2.3.0" @@ -5191,7 +5202,7 @@ dependencies = [ "pin-project-lite", "postgres-protocol", "postgres-types", - "rand", + "rand 0.8.5", "socket2", "tokio", "tokio-util 0.7.11", @@ -5205,18 +5216,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f57eb36ecbe0fc510036adff84824dd3c24bb781e21bfa67b69d556aa85214f" dependencies = [ "pin-project", - "rand", - "tokio", -] - -[[package]] -name = "tokio-rustls" -version = "0.25.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "775e0c0f0adb3a2f22a00c4745d728b479985fc15ee7ca6a2608388c5569860f" -dependencies = [ - "rustls 0.22.4", - "rustls-pki-types", + "rand 0.8.5", "tokio", ] @@ -5226,16 +5226,16 @@ version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" dependencies = [ - "rustls 0.23.10", + "rustls", "rustls-pki-types", "tokio", ] [[package]] name = "tokio-stream" -version = "0.1.15" +version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "267ac89e0bec6e691e5813911606935d77c476ff49024f98abcea3e7b15e37af" +checksum = "eca58d7bba4a75707817a2c44174253f9236b2d5fbd055602e9d5c07c139a047" dependencies = [ "futures-core", "pin-project-lite", @@ -5258,9 +5258,9 @@ dependencies = [ [[package]] name = "tokio-tungstenite" -version = "0.21.0" +version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c83b561d025642014097b66e6c1bb422783339e0909e4429cde4749d1990bc38" +checksum = "7a9daff607c6d2bf6c16fd681ccb7eecc83e4e2cdc1ca067ffaadfca5de7f084" dependencies = [ "futures-util", "log", @@ -5353,45 +5353,48 @@ dependencies = [ [[package]] name = "tonic" -version = "0.11.0" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76c4eb7a4e9ef9d4763600161f12f5070b92a578e1b634db88a6887844c91a13" +checksum = "877c5b330756d856ffcc4553ab34a5684481ade925ecc54bcd1bf02b1d0d4d52" dependencies = [ "async-stream", "async-trait", - "axum 0.6.20", - "base64 0.21.7", + "axum 0.7.5", + "base64 0.22.1", "bytes", "flate2", - "h2 0.3.26", - "http 0.2.12", - "http-body 0.4.6", - "hyper 0.14.29", + "h2 0.4.5", + "http 1.1.0", + "http-body 1.0.0", + "http-body-util", + "hyper 1.6.0", "hyper-timeout", + "hyper-util", "percent-encoding", "pin-project", - "prost 0.12.6", - "rustls-native-certs", + "prost", + "rustls-native-certs 0.8.1", "rustls-pemfile", - "rustls-pki-types", + "socket2", "tokio", - "tokio-rustls 0.25.0", + "tokio-rustls", "tokio-stream", "tower 0.4.13 (registry+https://github.com/rust-lang/crates.io-index)", - "tower-layer 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", - "tower-service 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "tower-layer 0.3.3", + "tower-service 0.3.3", "tracing", ] [[package]] name = "tonic-build" -version = "0.11.0" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be4ef6dd70a610078cb4e338a0f79d06bc759ff1b22d2120c2ff02ae264ba9c2" +checksum = "9557ce109ea773b399c9b9e5dca39294110b74f1f342cb347a80d1fce8c26a11" dependencies = [ - "prettyplease 0.2.20", + "prettyplease", "proc-macro2", - "prost-build 0.12.6", + "prost-build", + "prost-types", "quote", "syn 2.0.87", ] @@ -5407,12 +5410,12 @@ dependencies = [ "indexmap 1.9.3", "pin-project", "pin-project-lite", - "rand", + "rand 0.8.5", "slab", "tokio", "tokio-util 0.7.11", - "tower-layer 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", - "tower-service 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "tower-layer 0.3.3", + "tower-service 0.3.3", "tracing", ] @@ -5430,8 +5433,24 @@ dependencies = [ "sync_wrapper 0.1.2", "tokio", "tokio-util 0.7.11", - "tower-layer 0.3.2 (git+https://github.com/tower-rs/tower.git)", - "tower-service 0.3.2 (git+https://github.com/tower-rs/tower.git)", + "tower-layer 0.3.2", + "tower-service 0.3.2", + "tracing", +] + +[[package]] +name = "tower" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper 1.0.1", + "tokio", + "tower-layer 0.3.3", + "tower-service 0.3.3", "tracing", ] @@ -5447,31 +5466,31 @@ dependencies = [ "http-body 1.0.0", "http-body-util", "pin-project-lite", - "tower-layer 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", - "tower-service 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "tower-layer 0.3.3", + "tower-service 0.3.3", ] [[package]] name = "tower-layer" version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" +source = "git+https://github.com/tower-rs/tower.git#39adf5c509a1b2141f679654d8317524ca96b58b" [[package]] name = "tower-layer" -version = "0.3.2" -source = "git+https://github.com/tower-rs/tower.git#39adf5c509a1b2141f679654d8317524ca96b58b" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" [[package]] name = "tower-service" version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" +source = "git+https://github.com/tower-rs/tower.git#39adf5c509a1b2141f679654d8317524ca96b58b" [[package]] name = "tower-service" -version = "0.3.2" -source = "git+https://github.com/tower-rs/tower.git#39adf5c509a1b2141f679654d8317524ca96b58b" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tower-test" @@ -5482,8 +5501,8 @@ dependencies = [ "pin-project-lite", "tokio", "tokio-test", - "tower-layer 0.3.2 (git+https://github.com/tower-rs/tower.git)", - "tower-service 0.3.2 (git+https://github.com/tower-rs/tower.git)", + "tower-layer 0.3.2", + "tower-service 0.3.2", ] [[package]] @@ -5543,20 +5562,18 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "tungstenite" -version = "0.21.0" +version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ef1a641ea34f399a848dea702823bbecfb4c486f911735368f1f137cb8257e1" +checksum = "4793cb5e56680ecbb1d843515b23b6de9a75eb04b66643e256a396d43be33c13" dependencies = [ - "byteorder", "bytes", "data-encoding", "http 1.1.0", "httparse", "log", - "rand", + "rand 0.9.0", "sha1", - "thiserror", - "url", + "thiserror 2.0.12", "utf-8", ] @@ -5706,7 +5723,7 @@ version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5de17fd2f7da591098415cff336e12965a28061ddace43b59cb3c430179c9439" dependencies = [ - "getrandom", + "getrandom 0.2.15", "serde", ] @@ -5753,6 +5770,15 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasi" +version = "0.13.3+wasi-0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26816d2e1a4a36a2940b96c5296ce403917633dff8f3440e9b236ed6f6bacad2" +dependencies = [ + "wit-bindgen-rt", +] + [[package]] name = "wasite" version = "0.1.0" @@ -5990,7 +6016,7 @@ dependencies = [ "log", "object 0.32.2", "target-lexicon", - "thiserror", + "thiserror 1.0.61", "wasmparser 0.116.1", "wasmtime-cranelift-shared", "wasmtime-environ", @@ -6028,7 +6054,7 @@ dependencies = [ "serde", "serde_derive", "target-lexicon", - "thiserror", + "thiserror 1.0.61", "wasmparser 0.116.1", "wasmtime-types", ] @@ -6114,7 +6140,7 @@ dependencies = [ "memfd", "memoffset", "paste", - "rand", + "rand 0.8.5", "rustix", "sptr", "wasm-encoder 0.36.2", @@ -6136,7 +6162,7 @@ dependencies = [ "cranelift-entity", "serde", "serde_derive", - "thiserror", + "thiserror 1.0.61", "wasmparser 0.116.1", ] @@ -6243,7 +6269,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f6d8d1636b2627fe63518d5a9b38a569405d9c9bc665c43c9c341de57227ebb" dependencies = [ "native-tls", - "thiserror", + "thiserror 1.0.61", "tokio", "url", ] @@ -6502,6 +6528,15 @@ dependencies = [ "url", ] +[[package]] +name = "wit-bindgen-rt" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3268f3d866458b787f390cf61f4bbb563b922d091359f9608842999eaee3943c" +dependencies = [ + "bitflags 2.6.0", +] + [[package]] name = "wit-parser" version = "0.13.2" @@ -6546,7 +6581,16 @@ version = "0.7.35" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" dependencies = [ - "zerocopy-derive", + "zerocopy-derive 0.7.35", +] + +[[package]] +name = "zerocopy" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcf01143b2dd5d134f11f545cf9f1431b13b749695cb33bcce051e7568f99478" +dependencies = [ + "zerocopy-derive 0.8.21", ] [[package]] @@ -6560,6 +6604,17 @@ dependencies = [ "syn 2.0.87", ] +[[package]] +name = "zerocopy-derive" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712c8386f4f4299382c9abee219bee7084f78fb939d88b6840fcc1320d5f6da2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + [[package]] name = "zeroize" version = "1.8.1" diff --git a/Cargo.toml b/Cargo.toml index b3cd092114f..0cf70f16736 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -41,10 +41,10 @@ license = "MIT OR Apache-2.0" [workspace.dependencies] anyhow = "1.0" async-graphql = { version = "7.0.15", features = ["chrono", "uuid"] } -async-graphql-axum = "7.0.11" -axum = "0.7.5" -bs58 = "0.5.1" +async-graphql-axum = "7.0.15" +axum = "0.8.1" chrono = "0.4.38" +bs58 = "0.5.1" clap = { version = "4.5.4", features = ["derive", "env"] } derivative = "2.2.0" diesel = { version = "2.2.4", features = [ @@ -68,8 +68,8 @@ graphman = { path = "./core/graphman" } graphman-store = { path = "./core/graphman_store" } itertools = "0.13.0" lazy_static = "1.5.0" -prost = "0.12.6" -prost-types = "0.12.6" +prost = "0.13" +prost-types = "0.13" regex = "1.5.4" reqwest = "0.12.5" serde = { version = "1.0.126", features = ["rc"] } @@ -87,11 +87,14 @@ syn = { version = "2.0.87", features = ["full"] } test-store = { path = "./store/test-store" } thiserror = "1.0.25" tokio = { version = "1.38.0", features = ["full"] } -tonic = { version = "0.11.0", features = ["tls-roots", "gzip"] } -tonic-build = { version = "0.11.0", features = ["prost"] } +tonic = { version = "0.12.3", features = ["tls-roots", "gzip"] } +tonic-build = { version = "0.12.3", features = ["prost"] } tower-http = { version = "0.5.2", features = ["cors"] } wasmparser = "0.118.1" wasmtime = "15.0.1" +substreams = "=0.6.0" +substreams-entity-change = "2" +substreams-near-core = "=0.10.2" # Incremental compilation on Rust 1.58 causes an ICE on build. As soon as graph node builds again, these can be removed. [profile.test] diff --git a/chain/arweave/build.rs b/chain/arweave/build.rs index ea8153e7bd1..c0abc5fb8c9 100644 --- a/chain/arweave/build.rs +++ b/chain/arweave/build.rs @@ -2,6 +2,6 @@ fn main() { println!("cargo:rerun-if-changed=proto"); tonic_build::configure() .out_dir("src/protobuf") - .compile(&["proto/arweave.proto"], &["proto"]) + .compile_protos(&["proto/arweave.proto"], &["proto"]) .expect("Failed to compile Firehose Arweave proto(s)"); } diff --git a/chain/arweave/src/protobuf/sf.arweave.r#type.v1.rs b/chain/arweave/src/protobuf/sf.arweave.r#type.v1.rs index 39f83444cae..17f44498491 100644 --- a/chain/arweave/src/protobuf/sf.arweave.r#type.v1.rs +++ b/chain/arweave/src/protobuf/sf.arweave.r#type.v1.rs @@ -1,11 +1,9 @@ // This file is @generated by prost-build. -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct BigInt { #[prost(bytes = "vec", tag = "1")] pub bytes: ::prost::alloc::vec::Vec, } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct Block { /// Firehose block version (unrelated to Arweave block version) @@ -75,7 +73,6 @@ pub struct Block { pub poa: ::core::option::Option, } /// A succinct proof of access to a recall byte found in a TX -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct ProofOfAccess { /// The recall byte option chosen; global offset of index byte @@ -94,7 +91,6 @@ pub struct ProofOfAccess { #[prost(bytes = "vec", tag = "4")] pub chunk: ::prost::alloc::vec::Vec, } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct Transaction { /// 1 or 2 for v1 or v2 transactions. More allowable in the future @@ -137,7 +133,6 @@ pub struct Transaction { #[prost(message, optional, tag = "12")] pub reward: ::core::option::Option, } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct Tag { #[prost(bytes = "vec", tag = "1")] diff --git a/chain/ethereum/build.rs b/chain/ethereum/build.rs index 8ccae67aa92..227a50914a6 100644 --- a/chain/ethereum/build.rs +++ b/chain/ethereum/build.rs @@ -3,6 +3,6 @@ fn main() { tonic_build::configure() .out_dir("src/protobuf") - .compile(&["proto/ethereum.proto"], &["proto"]) + .compile_protos(&["proto/ethereum.proto"], &["proto"]) .expect("Failed to compile Firehose Ethereum proto(s)"); } diff --git a/chain/ethereum/src/protobuf/sf.ethereum.r#type.v2.rs b/chain/ethereum/src/protobuf/sf.ethereum.r#type.v2.rs index 3f0cc728f39..4ab8d0a1324 100644 --- a/chain/ethereum/src/protobuf/sf.ethereum.r#type.v2.rs +++ b/chain/ethereum/src/protobuf/sf.ethereum.r#type.v2.rs @@ -1,5 +1,4 @@ // This file is @generated by prost-build. -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct Block { #[prost(int32, tag = "1")] @@ -32,7 +31,6 @@ pub struct Block { /// /// WARN: this is a client-side optimization pattern and should be moved in the /// consuming code. -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct HeaderOnlyBlock { #[prost(message, optional, tag = "5")] @@ -41,7 +39,6 @@ pub struct HeaderOnlyBlock { /// BlockWithRefs is a lightweight block, with traces and transactions /// purged from the `block` within, and only. It is used in transports /// to pass block data around. -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct BlockWithRefs { #[prost(string, tag = "1")] @@ -53,19 +50,16 @@ pub struct BlockWithRefs { #[prost(bool, tag = "4")] pub irreversible: bool, } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct TransactionRefs { #[prost(bytes = "vec", repeated, tag = "1")] pub hashes: ::prost::alloc::vec::Vec<::prost::alloc::vec::Vec>, } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct UnclesHeaders { #[prost(message, repeated, tag = "1")] pub uncles: ::prost::alloc::vec::Vec, } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct BlockRef { #[prost(bytes = "vec", tag = "1")] @@ -73,7 +67,6 @@ pub struct BlockRef { #[prost(uint64, tag = "2")] pub number: u64, } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct BlockHeader { #[prost(bytes = "vec", tag = "1")] @@ -163,13 +156,11 @@ pub struct BlockHeader { #[prost(message, optional, tag = "18")] pub base_fee_per_gas: ::core::option::Option, } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct BigInt { #[prost(bytes = "vec", tag = "1")] pub bytes: ::prost::alloc::vec::Vec, } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct TransactionTrace { /// consensus @@ -290,9 +281,9 @@ pub mod transaction_trace { /// (if the ProtoBuf definition does not change) and safe for programmatic use. pub fn as_str_name(&self) -> &'static str { match self { - Type::TrxTypeLegacy => "TRX_TYPE_LEGACY", - Type::TrxTypeAccessList => "TRX_TYPE_ACCESS_LIST", - Type::TrxTypeDynamicFee => "TRX_TYPE_DYNAMIC_FEE", + Self::TrxTypeLegacy => "TRX_TYPE_LEGACY", + Self::TrxTypeAccessList => "TRX_TYPE_ACCESS_LIST", + Self::TrxTypeDynamicFee => "TRX_TYPE_DYNAMIC_FEE", } } /// Creates an enum from field names used in the ProtoBuf definition. @@ -308,7 +299,6 @@ pub mod transaction_trace { } /// AccessTuple represents a list of storage keys for a given contract's address and is used /// for AccessList construction. -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct AccessTuple { #[prost(bytes = "vec", tag = "1")] @@ -317,7 +307,6 @@ pub struct AccessTuple { pub storage_keys: ::prost::alloc::vec::Vec<::prost::alloc::vec::Vec>, } /// TransactionTraceWithBlockRef -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct TransactionTraceWithBlockRef { #[prost(message, optional, tag = "1")] @@ -325,7 +314,6 @@ pub struct TransactionTraceWithBlockRef { #[prost(message, optional, tag = "2")] pub block_ref: ::core::option::Option, } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct TransactionReceipt { /// State root is an intermediate state_root hash, computed in-between transactions to make @@ -350,7 +338,6 @@ pub struct TransactionReceipt { #[prost(message, repeated, tag = "4")] pub logs: ::prost::alloc::vec::Vec, } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct Log { #[prost(bytes = "vec", tag = "1")] @@ -384,7 +371,6 @@ pub struct Log { #[prost(uint64, tag = "7")] pub ordinal: u64, } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct Call { #[prost(uint32, tag = "1")] @@ -477,7 +463,6 @@ pub struct Call { #[prost(message, repeated, tag = "33")] pub account_creations: ::prost::alloc::vec::Vec, } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct StorageChange { #[prost(bytes = "vec", tag = "1")] @@ -491,7 +476,6 @@ pub struct StorageChange { #[prost(uint64, tag = "5")] pub ordinal: u64, } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct BalanceChange { #[prost(bytes = "vec", tag = "1")] @@ -550,22 +534,22 @@ pub mod balance_change { /// (if the ProtoBuf definition does not change) and safe for programmatic use. pub fn as_str_name(&self) -> &'static str { match self { - Reason::Unknown => "REASON_UNKNOWN", - Reason::RewardMineUncle => "REASON_REWARD_MINE_UNCLE", - Reason::RewardMineBlock => "REASON_REWARD_MINE_BLOCK", - Reason::DaoRefundContract => "REASON_DAO_REFUND_CONTRACT", - Reason::DaoAdjustBalance => "REASON_DAO_ADJUST_BALANCE", - Reason::Transfer => "REASON_TRANSFER", - Reason::GenesisBalance => "REASON_GENESIS_BALANCE", - Reason::GasBuy => "REASON_GAS_BUY", - Reason::RewardTransactionFee => "REASON_REWARD_TRANSACTION_FEE", - Reason::RewardFeeReset => "REASON_REWARD_FEE_RESET", - Reason::GasRefund => "REASON_GAS_REFUND", - Reason::TouchAccount => "REASON_TOUCH_ACCOUNT", - Reason::SuicideRefund => "REASON_SUICIDE_REFUND", - Reason::SuicideWithdraw => "REASON_SUICIDE_WITHDRAW", - Reason::CallBalanceOverride => "REASON_CALL_BALANCE_OVERRIDE", - Reason::Burn => "REASON_BURN", + Self::Unknown => "REASON_UNKNOWN", + Self::RewardMineUncle => "REASON_REWARD_MINE_UNCLE", + Self::RewardMineBlock => "REASON_REWARD_MINE_BLOCK", + Self::DaoRefundContract => "REASON_DAO_REFUND_CONTRACT", + Self::DaoAdjustBalance => "REASON_DAO_ADJUST_BALANCE", + Self::Transfer => "REASON_TRANSFER", + Self::GenesisBalance => "REASON_GENESIS_BALANCE", + Self::GasBuy => "REASON_GAS_BUY", + Self::RewardTransactionFee => "REASON_REWARD_TRANSACTION_FEE", + Self::RewardFeeReset => "REASON_REWARD_FEE_RESET", + Self::GasRefund => "REASON_GAS_REFUND", + Self::TouchAccount => "REASON_TOUCH_ACCOUNT", + Self::SuicideRefund => "REASON_SUICIDE_REFUND", + Self::SuicideWithdraw => "REASON_SUICIDE_WITHDRAW", + Self::CallBalanceOverride => "REASON_CALL_BALANCE_OVERRIDE", + Self::Burn => "REASON_BURN", } } /// Creates an enum from field names used in the ProtoBuf definition. @@ -592,7 +576,6 @@ pub mod balance_change { } } } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct NonceChange { #[prost(bytes = "vec", tag = "1")] @@ -604,7 +587,6 @@ pub struct NonceChange { #[prost(uint64, tag = "4")] pub ordinal: u64, } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct AccountCreation { #[prost(bytes = "vec", tag = "1")] @@ -612,7 +594,6 @@ pub struct AccountCreation { #[prost(uint64, tag = "2")] pub ordinal: u64, } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct CodeChange { #[prost(bytes = "vec", tag = "1")] @@ -634,8 +615,7 @@ pub struct CodeChange { /// /// Hence, we only index some of them, those that are costly like all the calls /// one, log events, return data, etc. -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, ::prost::Message)] pub struct GasChange { #[prost(uint64, tag = "1")] pub old_value: u64, @@ -696,27 +676,27 @@ pub mod gas_change { /// (if the ProtoBuf definition does not change) and safe for programmatic use. pub fn as_str_name(&self) -> &'static str { match self { - Reason::Unknown => "REASON_UNKNOWN", - Reason::Call => "REASON_CALL", - Reason::CallCode => "REASON_CALL_CODE", - Reason::CallDataCopy => "REASON_CALL_DATA_COPY", - Reason::CodeCopy => "REASON_CODE_COPY", - Reason::CodeStorage => "REASON_CODE_STORAGE", - Reason::ContractCreation => "REASON_CONTRACT_CREATION", - Reason::ContractCreation2 => "REASON_CONTRACT_CREATION2", - Reason::DelegateCall => "REASON_DELEGATE_CALL", - Reason::EventLog => "REASON_EVENT_LOG", - Reason::ExtCodeCopy => "REASON_EXT_CODE_COPY", - Reason::FailedExecution => "REASON_FAILED_EXECUTION", - Reason::IntrinsicGas => "REASON_INTRINSIC_GAS", - Reason::PrecompiledContract => "REASON_PRECOMPILED_CONTRACT", - Reason::RefundAfterExecution => "REASON_REFUND_AFTER_EXECUTION", - Reason::Return => "REASON_RETURN", - Reason::ReturnDataCopy => "REASON_RETURN_DATA_COPY", - Reason::Revert => "REASON_REVERT", - Reason::SelfDestruct => "REASON_SELF_DESTRUCT", - Reason::StaticCall => "REASON_STATIC_CALL", - Reason::StateColdAccess => "REASON_STATE_COLD_ACCESS", + Self::Unknown => "REASON_UNKNOWN", + Self::Call => "REASON_CALL", + Self::CallCode => "REASON_CALL_CODE", + Self::CallDataCopy => "REASON_CALL_DATA_COPY", + Self::CodeCopy => "REASON_CODE_COPY", + Self::CodeStorage => "REASON_CODE_STORAGE", + Self::ContractCreation => "REASON_CONTRACT_CREATION", + Self::ContractCreation2 => "REASON_CONTRACT_CREATION2", + Self::DelegateCall => "REASON_DELEGATE_CALL", + Self::EventLog => "REASON_EVENT_LOG", + Self::ExtCodeCopy => "REASON_EXT_CODE_COPY", + Self::FailedExecution => "REASON_FAILED_EXECUTION", + Self::IntrinsicGas => "REASON_INTRINSIC_GAS", + Self::PrecompiledContract => "REASON_PRECOMPILED_CONTRACT", + Self::RefundAfterExecution => "REASON_REFUND_AFTER_EXECUTION", + Self::Return => "REASON_RETURN", + Self::ReturnDataCopy => "REASON_RETURN_DATA_COPY", + Self::Revert => "REASON_REVERT", + Self::SelfDestruct => "REASON_SELF_DESTRUCT", + Self::StaticCall => "REASON_STATIC_CALL", + Self::StateColdAccess => "REASON_STATE_COLD_ACCESS", } } /// Creates an enum from field names used in the ProtoBuf definition. @@ -763,10 +743,10 @@ impl TransactionTraceStatus { /// (if the ProtoBuf definition does not change) and safe for programmatic use. pub fn as_str_name(&self) -> &'static str { match self { - TransactionTraceStatus::Unknown => "UNKNOWN", - TransactionTraceStatus::Succeeded => "SUCCEEDED", - TransactionTraceStatus::Failed => "FAILED", - TransactionTraceStatus::Reverted => "REVERTED", + Self::Unknown => "UNKNOWN", + Self::Succeeded => "SUCCEEDED", + Self::Failed => "FAILED", + Self::Reverted => "REVERTED", } } /// Creates an enum from field names used in the ProtoBuf definition. @@ -799,12 +779,12 @@ impl CallType { /// (if the ProtoBuf definition does not change) and safe for programmatic use. pub fn as_str_name(&self) -> &'static str { match self { - CallType::Unspecified => "UNSPECIFIED", - CallType::Call => "CALL", - CallType::Callcode => "CALLCODE", - CallType::Delegate => "DELEGATE", - CallType::Static => "STATIC", - CallType::Create => "CREATE", + Self::Unspecified => "UNSPECIFIED", + Self::Call => "CALL", + Self::Callcode => "CALLCODE", + Self::Delegate => "DELEGATE", + Self::Static => "STATIC", + Self::Create => "CREATE", } } /// Creates an enum from field names used in the ProtoBuf definition. diff --git a/chain/near/build.rs b/chain/near/build.rs index 611f861baf2..0bb50d10b27 100644 --- a/chain/near/build.rs +++ b/chain/near/build.rs @@ -3,7 +3,7 @@ fn main() { tonic_build::configure() .out_dir("src/protobuf") .extern_path(".sf.near.codec.v1", "crate::codec::pbcodec") - .compile( + .compile_protos( &["proto/near.proto", "proto/substreams-triggers.proto"], &["proto"], ) diff --git a/chain/near/src/protobuf/receipts.v1.rs b/chain/near/src/protobuf/receipts.v1.rs index 00d3e2fe004..2b844103e9a 100644 --- a/chain/near/src/protobuf/receipts.v1.rs +++ b/chain/near/src/protobuf/receipts.v1.rs @@ -1,5 +1,4 @@ // This file is @generated by prost-build. -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct BlockAndReceipts { #[prost(message, optional, tag = "1")] diff --git a/chain/substreams/build.rs b/chain/substreams/build.rs index 8cccc11fe3a..330a01a8c68 100644 --- a/chain/substreams/build.rs +++ b/chain/substreams/build.rs @@ -3,6 +3,6 @@ fn main() { tonic_build::configure() .protoc_arg("--experimental_allow_proto3_optional") .out_dir("src/protobuf") - .compile(&["proto/codec.proto"], &["proto"]) + .compile_protos(&["proto/codec.proto"], &["proto"]) .expect("Failed to compile Substreams entity proto(s)"); } diff --git a/chain/substreams/src/protobuf/substreams.entity.v1.rs b/chain/substreams/src/protobuf/substreams.entity.v1.rs index 372748908d7..4077f281ad7 100644 --- a/chain/substreams/src/protobuf/substreams.entity.v1.rs +++ b/chain/substreams/src/protobuf/substreams.entity.v1.rs @@ -1,11 +1,9 @@ // This file is @generated by prost-build. -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct EntityChanges { #[prost(message, repeated, tag = "5")] pub entity_changes: ::prost::alloc::vec::Vec, } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct EntityChange { #[prost(string, tag = "1")] @@ -47,10 +45,10 @@ pub mod entity_change { /// (if the ProtoBuf definition does not change) and safe for programmatic use. pub fn as_str_name(&self) -> &'static str { match self { - Operation::Unset => "UNSET", - Operation::Create => "CREATE", - Operation::Update => "UPDATE", - Operation::Delete => "DELETE", + Self::Unset => "UNSET", + Self::Create => "CREATE", + Self::Update => "UPDATE", + Self::Delete => "DELETE", } } /// Creates an enum from field names used in the ProtoBuf definition. @@ -65,7 +63,6 @@ pub mod entity_change { } } } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct Value { #[prost(oneof = "value::Typed", tags = "1, 2, 3, 4, 5, 6, 7, 10")] @@ -73,7 +70,6 @@ pub struct Value { } /// Nested message and enum types in `Value`. pub mod value { - #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Oneof)] pub enum Typed { #[prost(int32, tag = "1")] @@ -95,13 +91,11 @@ pub mod value { Array(super::Array), } } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct Array { #[prost(message, repeated, tag = "1")] pub value: ::prost::alloc::vec::Vec, } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct Field { #[prost(string, tag = "1")] diff --git a/docs/implementation/add-chain.md b/docs/implementation/add-chain.md index f4af3371f25..718c89bf24a 100644 --- a/docs/implementation/add-chain.md +++ b/docs/implementation/add-chain.md @@ -45,7 +45,7 @@ fn main() { println!("cargo:rerun-if-changed=proto"); tonic_build::configure() .out_dir("src/protobuf") - .compile(&["proto/codec.proto"], &["proto"]) + .compile_protos(&["proto/codec.proto"], &["proto"]) .expect("Failed to compile Firehose CoolChain proto(s)"); } ``` diff --git a/graph/build.rs b/graph/build.rs index 53bbe421ab9..d67e110edf4 100644 --- a/graph/build.rs +++ b/graph/build.rs @@ -2,7 +2,7 @@ fn main() { println!("cargo:rerun-if-changed=proto"); tonic_build::configure() .out_dir("src/firehose") - .compile( + .compile_protos( &[ "proto/firehose.proto", "proto/ethereum/transforms.proto", @@ -15,7 +15,7 @@ fn main() { tonic_build::configure() .protoc_arg("--experimental_allow_proto3_optional") .out_dir("src/substreams") - .compile(&["proto/substreams.proto"], &["proto"]) + .compile_protos(&["proto/substreams.proto"], &["proto"]) .expect("Failed to compile Substreams proto(s)"); tonic_build::configure() @@ -23,6 +23,6 @@ fn main() { .extern_path(".sf.substreams.v1", "crate::substreams") .extern_path(".sf.firehose.v2", "crate::firehose") .out_dir("src/substreams_rpc") - .compile(&["proto/substreams-rpc.proto"], &["proto"]) + .compile_protos(&["proto/substreams-rpc.proto"], &["proto"]) .expect("Failed to compile Substreams RPC proto(s)"); } diff --git a/graph/src/firehose/endpoints.rs b/graph/src/firehose/endpoints.rs index 00b87ea21a4..825f3ddbd20 100644 --- a/graph/src/firehose/endpoints.rs +++ b/graph/src/firehose/endpoints.rs @@ -15,7 +15,7 @@ use crate::{ }; use async_trait::async_trait; use futures03::StreamExt; -use http0::uri::{Scheme, Uri}; +use http::uri::{Scheme, Uri}; use itertools::Itertools; use slog::{error, info, trace, Logger}; use std::{collections::HashMap, fmt::Display, ops::ControlFlow, sync::Arc, time::Duration}; diff --git a/graph/src/firehose/sf.ethereum.transform.v1.rs b/graph/src/firehose/sf.ethereum.transform.v1.rs index 1f313e956e0..8f80ce08ea3 100644 --- a/graph/src/firehose/sf.ethereum.transform.v1.rs +++ b/graph/src/firehose/sf.ethereum.transform.v1.rs @@ -17,7 +17,6 @@ /// the "block index" is always produced after the merged-blocks files /// are produced. Therefore, the "live" blocks are never filtered out. /// -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct CombinedFilter { #[prost(message, repeated, tag = "1")] @@ -30,7 +29,6 @@ pub struct CombinedFilter { pub send_all_block_headers: bool, } /// MultiLogFilter concatenates the results of each LogFilter (inclusive OR) -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct MultiLogFilter { #[prost(message, repeated, tag = "1")] @@ -41,7 +39,6 @@ pub struct MultiLogFilter { /// * the event signature (topic.0) is one of the provided event_signatures -- OR event_signatures is empty -- /// /// a LogFilter with both empty addresses and event_signatures lists is invalid and will fail. -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct LogFilter { #[prost(bytes = "vec", repeated, tag = "1")] @@ -51,7 +48,6 @@ pub struct LogFilter { pub event_signatures: ::prost::alloc::vec::Vec<::prost::alloc::vec::Vec>, } /// MultiCallToFilter concatenates the results of each CallToFilter (inclusive OR) -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct MultiCallToFilter { #[prost(message, repeated, tag = "1")] @@ -62,7 +58,6 @@ pub struct MultiCallToFilter { /// * the method signature (in 4-bytes format) is one of the provided signatures -- OR signatures is empty -- /// /// a CallToFilter with both empty addresses and signatures lists is invalid and will fail. -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct CallToFilter { #[prost(bytes = "vec", repeated, tag = "1")] @@ -72,8 +67,7 @@ pub struct CallToFilter { } /// Deprecated: LightBlock is deprecated, replaced by HeaderOnly, note however that the new transform /// does not have any transactions traces returned, so it's not a direct replacement. -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, ::prost::Message)] pub struct LightBlock {} /// HeaderOnly returns only the block's header and few top-level core information for the block. Useful /// for cases where no transactions information is required at all. @@ -91,6 +85,5 @@ pub struct LightBlock {} /// ``` /// /// Everything else will be empty. -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, ::prost::Message)] pub struct HeaderOnly {} diff --git a/graph/src/firehose/sf.firehose.v2.rs b/graph/src/firehose/sf.firehose.v2.rs index b0980b35531..bca61385c71 100644 --- a/graph/src/firehose/sf.firehose.v2.rs +++ b/graph/src/firehose/sf.firehose.v2.rs @@ -1,5 +1,4 @@ // This file is @generated by prost-build. -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct SingleBlockRequest { #[prost(message, repeated, tag = "6")] @@ -10,14 +9,12 @@ pub struct SingleBlockRequest { /// Nested message and enum types in `SingleBlockRequest`. pub mod single_block_request { /// Get the current known canonical version of a block at with this number - #[allow(clippy::derive_partial_eq_without_eq)] - #[derive(Clone, PartialEq, ::prost::Message)] + #[derive(Clone, Copy, PartialEq, ::prost::Message)] pub struct BlockNumber { #[prost(uint64, tag = "1")] pub num: u64, } /// Get the current block with specific hash and number - #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct BlockHashAndNumber { #[prost(uint64, tag = "1")] @@ -26,13 +23,11 @@ pub mod single_block_request { pub hash: ::prost::alloc::string::String, } /// Get the block that generated a specific cursor - #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct Cursor { #[prost(string, tag = "1")] pub cursor: ::prost::alloc::string::String, } - #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Oneof)] pub enum Reference { #[prost(message, tag = "3")] @@ -43,13 +38,11 @@ pub mod single_block_request { Cursor(Cursor), } } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct SingleBlockResponse { #[prost(message, optional, tag = "1")] pub block: ::core::option::Option<::prost_types::Any>, } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct Request { /// Controls where the stream of blocks will start. @@ -90,7 +83,6 @@ pub struct Request { #[prost(message, repeated, tag = "10")] pub transforms: ::prost::alloc::vec::Vec<::prost_types::Any>, } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct Response { /// Chain specific block payload, ex: @@ -104,10 +96,8 @@ pub struct Response { #[prost(string, tag = "10")] pub cursor: ::prost::alloc::string::String, } -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, ::prost::Message)] pub struct InfoRequest {} -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct InfoResponse { /// Canonical chain name from (ex: matic, mainnet ...). @@ -160,12 +150,12 @@ pub mod info_response { /// (if the ProtoBuf definition does not change) and safe for programmatic use. pub fn as_str_name(&self) -> &'static str { match self { - BlockIdEncoding::Unset => "BLOCK_ID_ENCODING_UNSET", - BlockIdEncoding::Hex => "BLOCK_ID_ENCODING_HEX", - BlockIdEncoding::BlockIdEncoding0xHex => "BLOCK_ID_ENCODING_0X_HEX", - BlockIdEncoding::Base58 => "BLOCK_ID_ENCODING_BASE58", - BlockIdEncoding::Base64 => "BLOCK_ID_ENCODING_BASE64", - BlockIdEncoding::Base64url => "BLOCK_ID_ENCODING_BASE64URL", + Self::Unset => "BLOCK_ID_ENCODING_UNSET", + Self::Hex => "BLOCK_ID_ENCODING_HEX", + Self::BlockIdEncoding0xHex => "BLOCK_ID_ENCODING_0X_HEX", + Self::Base58 => "BLOCK_ID_ENCODING_BASE58", + Self::Base64 => "BLOCK_ID_ENCODING_BASE64", + Self::Base64url => "BLOCK_ID_ENCODING_BASE64URL", } } /// Creates an enum from field names used in the ProtoBuf definition. @@ -201,10 +191,10 @@ impl ForkStep { /// (if the ProtoBuf definition does not change) and safe for programmatic use. pub fn as_str_name(&self) -> &'static str { match self { - ForkStep::StepUnset => "STEP_UNSET", - ForkStep::StepNew => "STEP_NEW", - ForkStep::StepUndo => "STEP_UNDO", - ForkStep::StepFinal => "STEP_FINAL", + Self::StepUnset => "STEP_UNSET", + Self::StepNew => "STEP_NEW", + Self::StepUndo => "STEP_UNDO", + Self::StepFinal => "STEP_FINAL", } } /// Creates an enum from field names used in the ProtoBuf definition. @@ -220,7 +210,13 @@ impl ForkStep { } /// Generated client implementations. pub mod stream_client { - #![allow(unused_variables, dead_code, missing_docs, clippy::let_unit_value)] + #![allow( + unused_variables, + dead_code, + missing_docs, + clippy::wildcard_imports, + clippy::let_unit_value, + )] use tonic::codegen::*; use tonic::codegen::http::Uri; #[derive(Debug, Clone)] @@ -242,8 +238,8 @@ pub mod stream_client { where T: tonic::client::GrpcService, T::Error: Into, - T::ResponseBody: Body + Send + 'static, - ::Error: Into + Send, + T::ResponseBody: Body + std::marker::Send + 'static, + ::Error: Into + std::marker::Send, { pub fn new(inner: T) -> Self { let inner = tonic::client::Grpc::new(inner); @@ -268,7 +264,7 @@ pub mod stream_client { >, , - >>::Error: Into + Send + Sync, + >>::Error: Into + std::marker::Send + std::marker::Sync, { StreamClient::new(InterceptedService::new(inner, interceptor)) } @@ -314,8 +310,7 @@ pub mod stream_client { .ready() .await .map_err(|e| { - tonic::Status::new( - tonic::Code::Unknown, + tonic::Status::unknown( format!("Service was not ready: {}", e.into()), ) })?; @@ -332,7 +327,13 @@ pub mod stream_client { } /// Generated client implementations. pub mod fetch_client { - #![allow(unused_variables, dead_code, missing_docs, clippy::let_unit_value)] + #![allow( + unused_variables, + dead_code, + missing_docs, + clippy::wildcard_imports, + clippy::let_unit_value, + )] use tonic::codegen::*; use tonic::codegen::http::Uri; #[derive(Debug, Clone)] @@ -354,8 +355,8 @@ pub mod fetch_client { where T: tonic::client::GrpcService, T::Error: Into, - T::ResponseBody: Body + Send + 'static, - ::Error: Into + Send, + T::ResponseBody: Body + std::marker::Send + 'static, + ::Error: Into + std::marker::Send, { pub fn new(inner: T) -> Self { let inner = tonic::client::Grpc::new(inner); @@ -380,7 +381,7 @@ pub mod fetch_client { >, , - >>::Error: Into + Send + Sync, + >>::Error: Into + std::marker::Send + std::marker::Sync, { FetchClient::new(InterceptedService::new(inner, interceptor)) } @@ -426,8 +427,7 @@ pub mod fetch_client { .ready() .await .map_err(|e| { - tonic::Status::new( - tonic::Code::Unknown, + tonic::Status::unknown( format!("Service was not ready: {}", e.into()), ) })?; @@ -444,7 +444,13 @@ pub mod fetch_client { } /// Generated client implementations. pub mod endpoint_info_client { - #![allow(unused_variables, dead_code, missing_docs, clippy::let_unit_value)] + #![allow( + unused_variables, + dead_code, + missing_docs, + clippy::wildcard_imports, + clippy::let_unit_value, + )] use tonic::codegen::*; use tonic::codegen::http::Uri; #[derive(Debug, Clone)] @@ -466,8 +472,8 @@ pub mod endpoint_info_client { where T: tonic::client::GrpcService, T::Error: Into, - T::ResponseBody: Body + Send + 'static, - ::Error: Into + Send, + T::ResponseBody: Body + std::marker::Send + 'static, + ::Error: Into + std::marker::Send, { pub fn new(inner: T) -> Self { let inner = tonic::client::Grpc::new(inner); @@ -492,7 +498,7 @@ pub mod endpoint_info_client { >, , - >>::Error: Into + Send + Sync, + >>::Error: Into + std::marker::Send + std::marker::Sync, { EndpointInfoClient::new(InterceptedService::new(inner, interceptor)) } @@ -535,8 +541,7 @@ pub mod endpoint_info_client { .ready() .await .map_err(|e| { - tonic::Status::new( - tonic::Code::Unknown, + tonic::Status::unknown( format!("Service was not ready: {}", e.into()), ) })?; @@ -553,16 +558,22 @@ pub mod endpoint_info_client { } /// Generated server implementations. pub mod stream_server { - #![allow(unused_variables, dead_code, missing_docs, clippy::let_unit_value)] + #![allow( + unused_variables, + dead_code, + missing_docs, + clippy::wildcard_imports, + clippy::let_unit_value, + )] use tonic::codegen::*; /// Generated trait containing gRPC methods that should be implemented for use with StreamServer. #[async_trait] - pub trait Stream: Send + Sync + 'static { + pub trait Stream: std::marker::Send + std::marker::Sync + 'static { /// Server streaming response type for the Blocks method. type BlocksStream: tonic::codegen::tokio_stream::Stream< Item = std::result::Result, > - + Send + + std::marker::Send + 'static; async fn blocks( &self, @@ -570,20 +581,18 @@ pub mod stream_server { ) -> std::result::Result, tonic::Status>; } #[derive(Debug)] - pub struct StreamServer { - inner: _Inner, + pub struct StreamServer { + inner: Arc, accept_compression_encodings: EnabledCompressionEncodings, send_compression_encodings: EnabledCompressionEncodings, max_decoding_message_size: Option, max_encoding_message_size: Option, } - struct _Inner(Arc); - impl StreamServer { + impl StreamServer { pub fn new(inner: T) -> Self { Self::from_arc(Arc::new(inner)) } pub fn from_arc(inner: Arc) -> Self { - let inner = _Inner(inner); Self { inner, accept_compression_encodings: Default::default(), @@ -633,8 +642,8 @@ pub mod stream_server { impl tonic::codegen::Service> for StreamServer where T: Stream, - B: Body + Send + 'static, - B::Error: Into + Send + 'static, + B: Body + std::marker::Send + 'static, + B::Error: Into + std::marker::Send + 'static, { type Response = http::Response; type Error = std::convert::Infallible; @@ -646,7 +655,6 @@ pub mod stream_server { Poll::Ready(Ok(())) } fn call(&mut self, req: http::Request) -> Self::Future { - let inner = self.inner.clone(); match req.uri().path() { "/sf.firehose.v2.Stream/Blocks" => { #[allow(non_camel_case_types)] @@ -676,7 +684,6 @@ pub mod stream_server { let max_encoding_message_size = self.max_encoding_message_size; let inner = self.inner.clone(); let fut = async move { - let inner = inner.0; let method = BlocksSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) @@ -695,20 +702,25 @@ pub mod stream_server { } _ => { Box::pin(async move { - Ok( - http::Response::builder() - .status(200) - .header("grpc-status", "12") - .header("content-type", "application/grpc") - .body(empty_body()) - .unwrap(), - ) + let mut response = http::Response::new(empty_body()); + let headers = response.headers_mut(); + headers + .insert( + tonic::Status::GRPC_STATUS, + (tonic::Code::Unimplemented as i32).into(), + ); + headers + .insert( + http::header::CONTENT_TYPE, + tonic::metadata::GRPC_CONTENT_TYPE, + ); + Ok(response) }) } } } } - impl Clone for StreamServer { + impl Clone for StreamServer { fn clone(&self) -> Self { let inner = self.inner.clone(); Self { @@ -720,27 +732,25 @@ pub mod stream_server { } } } - impl Clone for _Inner { - fn clone(&self) -> Self { - Self(Arc::clone(&self.0)) - } - } - impl std::fmt::Debug for _Inner { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{:?}", self.0) - } - } - impl tonic::server::NamedService for StreamServer { - const NAME: &'static str = "sf.firehose.v2.Stream"; + /// Generated gRPC service name + pub const SERVICE_NAME: &str = "sf.firehose.v2.Stream"; + impl tonic::server::NamedService for StreamServer { + const NAME: &'static str = SERVICE_NAME; } } /// Generated server implementations. pub mod fetch_server { - #![allow(unused_variables, dead_code, missing_docs, clippy::let_unit_value)] + #![allow( + unused_variables, + dead_code, + missing_docs, + clippy::wildcard_imports, + clippy::let_unit_value, + )] use tonic::codegen::*; /// Generated trait containing gRPC methods that should be implemented for use with FetchServer. #[async_trait] - pub trait Fetch: Send + Sync + 'static { + pub trait Fetch: std::marker::Send + std::marker::Sync + 'static { async fn block( &self, request: tonic::Request, @@ -750,20 +760,18 @@ pub mod fetch_server { >; } #[derive(Debug)] - pub struct FetchServer { - inner: _Inner, + pub struct FetchServer { + inner: Arc, accept_compression_encodings: EnabledCompressionEncodings, send_compression_encodings: EnabledCompressionEncodings, max_decoding_message_size: Option, max_encoding_message_size: Option, } - struct _Inner(Arc); - impl FetchServer { + impl FetchServer { pub fn new(inner: T) -> Self { Self::from_arc(Arc::new(inner)) } pub fn from_arc(inner: Arc) -> Self { - let inner = _Inner(inner); Self { inner, accept_compression_encodings: Default::default(), @@ -813,8 +821,8 @@ pub mod fetch_server { impl tonic::codegen::Service> for FetchServer where T: Fetch, - B: Body + Send + 'static, - B::Error: Into + Send + 'static, + B: Body + std::marker::Send + 'static, + B::Error: Into + std::marker::Send + 'static, { type Response = http::Response; type Error = std::convert::Infallible; @@ -826,7 +834,6 @@ pub mod fetch_server { Poll::Ready(Ok(())) } fn call(&mut self, req: http::Request) -> Self::Future { - let inner = self.inner.clone(); match req.uri().path() { "/sf.firehose.v2.Fetch/Block" => { #[allow(non_camel_case_types)] @@ -855,7 +862,6 @@ pub mod fetch_server { let max_encoding_message_size = self.max_encoding_message_size; let inner = self.inner.clone(); let fut = async move { - let inner = inner.0; let method = BlockSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) @@ -874,20 +880,25 @@ pub mod fetch_server { } _ => { Box::pin(async move { - Ok( - http::Response::builder() - .status(200) - .header("grpc-status", "12") - .header("content-type", "application/grpc") - .body(empty_body()) - .unwrap(), - ) + let mut response = http::Response::new(empty_body()); + let headers = response.headers_mut(); + headers + .insert( + tonic::Status::GRPC_STATUS, + (tonic::Code::Unimplemented as i32).into(), + ); + headers + .insert( + http::header::CONTENT_TYPE, + tonic::metadata::GRPC_CONTENT_TYPE, + ); + Ok(response) }) } } } } - impl Clone for FetchServer { + impl Clone for FetchServer { fn clone(&self) -> Self { let inner = self.inner.clone(); Self { @@ -899,47 +910,43 @@ pub mod fetch_server { } } } - impl Clone for _Inner { - fn clone(&self) -> Self { - Self(Arc::clone(&self.0)) - } - } - impl std::fmt::Debug for _Inner { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{:?}", self.0) - } - } - impl tonic::server::NamedService for FetchServer { - const NAME: &'static str = "sf.firehose.v2.Fetch"; + /// Generated gRPC service name + pub const SERVICE_NAME: &str = "sf.firehose.v2.Fetch"; + impl tonic::server::NamedService for FetchServer { + const NAME: &'static str = SERVICE_NAME; } } /// Generated server implementations. pub mod endpoint_info_server { - #![allow(unused_variables, dead_code, missing_docs, clippy::let_unit_value)] + #![allow( + unused_variables, + dead_code, + missing_docs, + clippy::wildcard_imports, + clippy::let_unit_value, + )] use tonic::codegen::*; /// Generated trait containing gRPC methods that should be implemented for use with EndpointInfoServer. #[async_trait] - pub trait EndpointInfo: Send + Sync + 'static { + pub trait EndpointInfo: std::marker::Send + std::marker::Sync + 'static { async fn info( &self, request: tonic::Request, ) -> std::result::Result, tonic::Status>; } #[derive(Debug)] - pub struct EndpointInfoServer { - inner: _Inner, + pub struct EndpointInfoServer { + inner: Arc, accept_compression_encodings: EnabledCompressionEncodings, send_compression_encodings: EnabledCompressionEncodings, max_decoding_message_size: Option, max_encoding_message_size: Option, } - struct _Inner(Arc); - impl EndpointInfoServer { + impl EndpointInfoServer { pub fn new(inner: T) -> Self { Self::from_arc(Arc::new(inner)) } pub fn from_arc(inner: Arc) -> Self { - let inner = _Inner(inner); Self { inner, accept_compression_encodings: Default::default(), @@ -989,8 +996,8 @@ pub mod endpoint_info_server { impl tonic::codegen::Service> for EndpointInfoServer where T: EndpointInfo, - B: Body + Send + 'static, - B::Error: Into + Send + 'static, + B: Body + std::marker::Send + 'static, + B::Error: Into + std::marker::Send + 'static, { type Response = http::Response; type Error = std::convert::Infallible; @@ -1002,7 +1009,6 @@ pub mod endpoint_info_server { Poll::Ready(Ok(())) } fn call(&mut self, req: http::Request) -> Self::Future { - let inner = self.inner.clone(); match req.uri().path() { "/sf.firehose.v2.EndpointInfo/Info" => { #[allow(non_camel_case_types)] @@ -1031,7 +1037,6 @@ pub mod endpoint_info_server { let max_encoding_message_size = self.max_encoding_message_size; let inner = self.inner.clone(); let fut = async move { - let inner = inner.0; let method = InfoSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) @@ -1050,20 +1055,25 @@ pub mod endpoint_info_server { } _ => { Box::pin(async move { - Ok( - http::Response::builder() - .status(200) - .header("grpc-status", "12") - .header("content-type", "application/grpc") - .body(empty_body()) - .unwrap(), - ) + let mut response = http::Response::new(empty_body()); + let headers = response.headers_mut(); + headers + .insert( + tonic::Status::GRPC_STATUS, + (tonic::Code::Unimplemented as i32).into(), + ); + headers + .insert( + http::header::CONTENT_TYPE, + tonic::metadata::GRPC_CONTENT_TYPE, + ); + Ok(response) }) } } } } - impl Clone for EndpointInfoServer { + impl Clone for EndpointInfoServer { fn clone(&self) -> Self { let inner = self.inner.clone(); Self { @@ -1075,17 +1085,9 @@ pub mod endpoint_info_server { } } } - impl Clone for _Inner { - fn clone(&self) -> Self { - Self(Arc::clone(&self.0)) - } - } - impl std::fmt::Debug for _Inner { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{:?}", self.0) - } - } - impl tonic::server::NamedService for EndpointInfoServer { - const NAME: &'static str = "sf.firehose.v2.EndpointInfo"; + /// Generated gRPC service name + pub const SERVICE_NAME: &str = "sf.firehose.v2.EndpointInfo"; + impl tonic::server::NamedService for EndpointInfoServer { + const NAME: &'static str = SERVICE_NAME; } } diff --git a/graph/src/firehose/sf.near.transform.v1.rs b/graph/src/firehose/sf.near.transform.v1.rs index f76839cbd4c..2ec950da40b 100644 --- a/graph/src/firehose/sf.near.transform.v1.rs +++ b/graph/src/firehose/sf.near.transform.v1.rs @@ -1,5 +1,4 @@ // This file is @generated by prost-build. -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct BasicReceiptFilter { #[prost(string, repeated, tag = "1")] @@ -14,7 +13,6 @@ pub struct BasicReceiptFilter { /// * {prefix="",suffix=""} is invalid /// /// Note that the suffix will usually have a TLD, ex: "mydomain.near" or "mydomain.testnet" -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct PrefixSuffixPair { #[prost(string, tag = "1")] diff --git a/graph/src/substreams/sf.substreams.v1.rs b/graph/src/substreams/sf.substreams.v1.rs index e27ed7b346d..dd6b8930293 100644 --- a/graph/src/substreams/sf.substreams.v1.rs +++ b/graph/src/substreams/sf.substreams.v1.rs @@ -1,5 +1,4 @@ // This file is @generated by prost-build. -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct Package { /// Needs to be one so this file can be used _directly_ as a @@ -22,7 +21,6 @@ pub struct Package { #[prost(string, tag = "11")] pub sink_module: ::prost::alloc::string::String, } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct PackageMetadata { #[prost(string, tag = "1")] @@ -34,7 +32,6 @@ pub struct PackageMetadata { #[prost(string, tag = "4")] pub doc: ::prost::alloc::string::String, } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct ModuleMetadata { /// Corresponds to the index in `Package.metadata.package_meta` @@ -43,7 +40,6 @@ pub struct ModuleMetadata { #[prost(string, tag = "2")] pub doc: ::prost::alloc::string::String, } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct Modules { #[prost(message, repeated, tag = "1")] @@ -52,7 +48,6 @@ pub struct Modules { pub binaries: ::prost::alloc::vec::Vec, } /// Binary represents some code compiled to its binary form. -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct Binary { #[prost(string, tag = "1")] @@ -60,7 +55,6 @@ pub struct Binary { #[prost(bytes = "vec", tag = "2")] pub content: ::prost::alloc::vec::Vec, } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct Module { #[prost(string, tag = "1")] @@ -82,7 +76,6 @@ pub struct Module { } /// Nested message and enum types in `Module`. pub mod module { - #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct BlockFilter { #[prost(string, tag = "1")] @@ -92,7 +85,6 @@ pub mod module { } /// Nested message and enum types in `BlockFilter`. pub mod block_filter { - #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Oneof)] pub enum Query { #[prost(string, tag = "2")] @@ -101,16 +93,13 @@ pub mod module { QueryFromParams(super::QueryFromParams), } } - #[allow(clippy::derive_partial_eq_without_eq)] - #[derive(Clone, PartialEq, ::prost::Message)] + #[derive(Clone, Copy, PartialEq, ::prost::Message)] pub struct QueryFromParams {} - #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct KindMap { #[prost(string, tag = "1")] pub output_type: ::prost::alloc::string::String, } - #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct KindStore { /// The `update_policy` determines the functions available to mutate the store @@ -164,14 +153,14 @@ pub mod module { /// (if the ProtoBuf definition does not change) and safe for programmatic use. pub fn as_str_name(&self) -> &'static str { match self { - UpdatePolicy::Unset => "UPDATE_POLICY_UNSET", - UpdatePolicy::Set => "UPDATE_POLICY_SET", - UpdatePolicy::SetIfNotExists => "UPDATE_POLICY_SET_IF_NOT_EXISTS", - UpdatePolicy::Add => "UPDATE_POLICY_ADD", - UpdatePolicy::Min => "UPDATE_POLICY_MIN", - UpdatePolicy::Max => "UPDATE_POLICY_MAX", - UpdatePolicy::Append => "UPDATE_POLICY_APPEND", - UpdatePolicy::SetSum => "UPDATE_POLICY_SET_SUM", + Self::Unset => "UPDATE_POLICY_UNSET", + Self::Set => "UPDATE_POLICY_SET", + Self::SetIfNotExists => "UPDATE_POLICY_SET_IF_NOT_EXISTS", + Self::Add => "UPDATE_POLICY_ADD", + Self::Min => "UPDATE_POLICY_MIN", + Self::Max => "UPDATE_POLICY_MAX", + Self::Append => "UPDATE_POLICY_APPEND", + Self::SetSum => "UPDATE_POLICY_SET_SUM", } } /// Creates an enum from field names used in the ProtoBuf definition. @@ -190,13 +179,11 @@ pub mod module { } } } - #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct KindBlockIndex { #[prost(string, tag = "1")] pub output_type: ::prost::alloc::string::String, } - #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct Input { #[prost(oneof = "input::Input", tags = "1, 2, 3, 4")] @@ -204,21 +191,18 @@ pub mod module { } /// Nested message and enum types in `Input`. pub mod input { - #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct Source { /// ex: "sf.ethereum.type.v1.Block" #[prost(string, tag = "1")] pub r#type: ::prost::alloc::string::String, } - #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct Map { /// ex: "block_to_pairs" #[prost(string, tag = "1")] pub module_name: ::prost::alloc::string::String, } - #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct Store { #[prost(string, tag = "1")] @@ -252,9 +236,9 @@ pub mod module { /// (if the ProtoBuf definition does not change) and safe for programmatic use. pub fn as_str_name(&self) -> &'static str { match self { - Mode::Unset => "UNSET", - Mode::Get => "GET", - Mode::Deltas => "DELTAS", + Self::Unset => "UNSET", + Self::Get => "GET", + Self::Deltas => "DELTAS", } } /// Creates an enum from field names used in the ProtoBuf definition. @@ -268,13 +252,11 @@ pub mod module { } } } - #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct Params { #[prost(string, tag = "1")] pub value: ::prost::alloc::string::String, } - #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Oneof)] pub enum Input { #[prost(message, tag = "1")] @@ -287,13 +269,11 @@ pub mod module { Params(Params), } } - #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct Output { #[prost(string, tag = "1")] pub r#type: ::prost::alloc::string::String, } - #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Oneof)] pub enum Kind { #[prost(message, tag = "2")] @@ -305,7 +285,6 @@ pub mod module { } } /// Clock is a pointer to a block with added timestamp -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct Clock { #[prost(string, tag = "1")] @@ -316,7 +295,6 @@ pub struct Clock { pub timestamp: ::core::option::Option<::prost_types::Timestamp>, } /// BlockRef is a pointer to a block to which we don't know the timestamp -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct BlockRef { #[prost(string, tag = "1")] diff --git a/graph/src/substreams_rpc/sf.firehose.v2.rs b/graph/src/substreams_rpc/sf.firehose.v2.rs index ac86a47e505..905a7038bf5 100644 --- a/graph/src/substreams_rpc/sf.firehose.v2.rs +++ b/graph/src/substreams_rpc/sf.firehose.v2.rs @@ -1,7 +1,13 @@ // This file is @generated by prost-build. /// Generated client implementations. pub mod stream_client { - #![allow(unused_variables, dead_code, missing_docs, clippy::let_unit_value)] + #![allow( + unused_variables, + dead_code, + missing_docs, + clippy::wildcard_imports, + clippy::let_unit_value, + )] use tonic::codegen::*; use tonic::codegen::http::Uri; #[derive(Debug, Clone)] @@ -23,8 +29,8 @@ pub mod stream_client { where T: tonic::client::GrpcService, T::Error: Into, - T::ResponseBody: Body + Send + 'static, - ::Error: Into + Send, + T::ResponseBody: Body + std::marker::Send + 'static, + ::Error: Into + std::marker::Send, { pub fn new(inner: T) -> Self { let inner = tonic::client::Grpc::new(inner); @@ -49,7 +55,7 @@ pub mod stream_client { >, , - >>::Error: Into + Send + Sync, + >>::Error: Into + std::marker::Send + std::marker::Sync, { StreamClient::new(InterceptedService::new(inner, interceptor)) } @@ -95,8 +101,7 @@ pub mod stream_client { .ready() .await .map_err(|e| { - tonic::Status::new( - tonic::Code::Unknown, + tonic::Status::unknown( format!("Service was not ready: {}", e.into()), ) })?; @@ -113,7 +118,13 @@ pub mod stream_client { } /// Generated client implementations. pub mod fetch_client { - #![allow(unused_variables, dead_code, missing_docs, clippy::let_unit_value)] + #![allow( + unused_variables, + dead_code, + missing_docs, + clippy::wildcard_imports, + clippy::let_unit_value, + )] use tonic::codegen::*; use tonic::codegen::http::Uri; #[derive(Debug, Clone)] @@ -135,8 +146,8 @@ pub mod fetch_client { where T: tonic::client::GrpcService, T::Error: Into, - T::ResponseBody: Body + Send + 'static, - ::Error: Into + Send, + T::ResponseBody: Body + std::marker::Send + 'static, + ::Error: Into + std::marker::Send, { pub fn new(inner: T) -> Self { let inner = tonic::client::Grpc::new(inner); @@ -161,7 +172,7 @@ pub mod fetch_client { >, , - >>::Error: Into + Send + Sync, + >>::Error: Into + std::marker::Send + std::marker::Sync, { FetchClient::new(InterceptedService::new(inner, interceptor)) } @@ -207,8 +218,7 @@ pub mod fetch_client { .ready() .await .map_err(|e| { - tonic::Status::new( - tonic::Code::Unknown, + tonic::Status::unknown( format!("Service was not ready: {}", e.into()), ) })?; @@ -225,7 +235,13 @@ pub mod fetch_client { } /// Generated client implementations. pub mod endpoint_info_client { - #![allow(unused_variables, dead_code, missing_docs, clippy::let_unit_value)] + #![allow( + unused_variables, + dead_code, + missing_docs, + clippy::wildcard_imports, + clippy::let_unit_value, + )] use tonic::codegen::*; use tonic::codegen::http::Uri; #[derive(Debug, Clone)] @@ -247,8 +263,8 @@ pub mod endpoint_info_client { where T: tonic::client::GrpcService, T::Error: Into, - T::ResponseBody: Body + Send + 'static, - ::Error: Into + Send, + T::ResponseBody: Body + std::marker::Send + 'static, + ::Error: Into + std::marker::Send, { pub fn new(inner: T) -> Self { let inner = tonic::client::Grpc::new(inner); @@ -273,7 +289,7 @@ pub mod endpoint_info_client { >, , - >>::Error: Into + Send + Sync, + >>::Error: Into + std::marker::Send + std::marker::Sync, { EndpointInfoClient::new(InterceptedService::new(inner, interceptor)) } @@ -319,8 +335,7 @@ pub mod endpoint_info_client { .ready() .await .map_err(|e| { - tonic::Status::new( - tonic::Code::Unknown, + tonic::Status::unknown( format!("Service was not ready: {}", e.into()), ) })?; @@ -337,16 +352,22 @@ pub mod endpoint_info_client { } /// Generated server implementations. pub mod stream_server { - #![allow(unused_variables, dead_code, missing_docs, clippy::let_unit_value)] + #![allow( + unused_variables, + dead_code, + missing_docs, + clippy::wildcard_imports, + clippy::let_unit_value, + )] use tonic::codegen::*; /// Generated trait containing gRPC methods that should be implemented for use with StreamServer. #[async_trait] - pub trait Stream: Send + Sync + 'static { + pub trait Stream: std::marker::Send + std::marker::Sync + 'static { /// Server streaming response type for the Blocks method. type BlocksStream: tonic::codegen::tokio_stream::Stream< Item = std::result::Result, > - + Send + + std::marker::Send + 'static; async fn blocks( &self, @@ -354,20 +375,18 @@ pub mod stream_server { ) -> std::result::Result, tonic::Status>; } #[derive(Debug)] - pub struct StreamServer { - inner: _Inner, + pub struct StreamServer { + inner: Arc, accept_compression_encodings: EnabledCompressionEncodings, send_compression_encodings: EnabledCompressionEncodings, max_decoding_message_size: Option, max_encoding_message_size: Option, } - struct _Inner(Arc); - impl StreamServer { + impl StreamServer { pub fn new(inner: T) -> Self { Self::from_arc(Arc::new(inner)) } pub fn from_arc(inner: Arc) -> Self { - let inner = _Inner(inner); Self { inner, accept_compression_encodings: Default::default(), @@ -417,8 +436,8 @@ pub mod stream_server { impl tonic::codegen::Service> for StreamServer where T: Stream, - B: Body + Send + 'static, - B::Error: Into + Send + 'static, + B: Body + std::marker::Send + 'static, + B::Error: Into + std::marker::Send + 'static, { type Response = http::Response; type Error = std::convert::Infallible; @@ -430,7 +449,6 @@ pub mod stream_server { Poll::Ready(Ok(())) } fn call(&mut self, req: http::Request) -> Self::Future { - let inner = self.inner.clone(); match req.uri().path() { "/sf.firehose.v2.Stream/Blocks" => { #[allow(non_camel_case_types)] @@ -462,7 +480,6 @@ pub mod stream_server { let max_encoding_message_size = self.max_encoding_message_size; let inner = self.inner.clone(); let fut = async move { - let inner = inner.0; let method = BlocksSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) @@ -481,20 +498,25 @@ pub mod stream_server { } _ => { Box::pin(async move { - Ok( - http::Response::builder() - .status(200) - .header("grpc-status", "12") - .header("content-type", "application/grpc") - .body(empty_body()) - .unwrap(), - ) + let mut response = http::Response::new(empty_body()); + let headers = response.headers_mut(); + headers + .insert( + tonic::Status::GRPC_STATUS, + (tonic::Code::Unimplemented as i32).into(), + ); + headers + .insert( + http::header::CONTENT_TYPE, + tonic::metadata::GRPC_CONTENT_TYPE, + ); + Ok(response) }) } } } } - impl Clone for StreamServer { + impl Clone for StreamServer { fn clone(&self) -> Self { let inner = self.inner.clone(); Self { @@ -506,27 +528,25 @@ pub mod stream_server { } } } - impl Clone for _Inner { - fn clone(&self) -> Self { - Self(Arc::clone(&self.0)) - } - } - impl std::fmt::Debug for _Inner { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{:?}", self.0) - } - } - impl tonic::server::NamedService for StreamServer { - const NAME: &'static str = "sf.firehose.v2.Stream"; + /// Generated gRPC service name + pub const SERVICE_NAME: &str = "sf.firehose.v2.Stream"; + impl tonic::server::NamedService for StreamServer { + const NAME: &'static str = SERVICE_NAME; } } /// Generated server implementations. pub mod fetch_server { - #![allow(unused_variables, dead_code, missing_docs, clippy::let_unit_value)] + #![allow( + unused_variables, + dead_code, + missing_docs, + clippy::wildcard_imports, + clippy::let_unit_value, + )] use tonic::codegen::*; /// Generated trait containing gRPC methods that should be implemented for use with FetchServer. #[async_trait] - pub trait Fetch: Send + Sync + 'static { + pub trait Fetch: std::marker::Send + std::marker::Sync + 'static { async fn block( &self, request: tonic::Request, @@ -536,20 +556,18 @@ pub mod fetch_server { >; } #[derive(Debug)] - pub struct FetchServer { - inner: _Inner, + pub struct FetchServer { + inner: Arc, accept_compression_encodings: EnabledCompressionEncodings, send_compression_encodings: EnabledCompressionEncodings, max_decoding_message_size: Option, max_encoding_message_size: Option, } - struct _Inner(Arc); - impl FetchServer { + impl FetchServer { pub fn new(inner: T) -> Self { Self::from_arc(Arc::new(inner)) } pub fn from_arc(inner: Arc) -> Self { - let inner = _Inner(inner); Self { inner, accept_compression_encodings: Default::default(), @@ -599,8 +617,8 @@ pub mod fetch_server { impl tonic::codegen::Service> for FetchServer where T: Fetch, - B: Body + Send + 'static, - B::Error: Into + Send + 'static, + B: Body + std::marker::Send + 'static, + B::Error: Into + std::marker::Send + 'static, { type Response = http::Response; type Error = std::convert::Infallible; @@ -612,7 +630,6 @@ pub mod fetch_server { Poll::Ready(Ok(())) } fn call(&mut self, req: http::Request) -> Self::Future { - let inner = self.inner.clone(); match req.uri().path() { "/sf.firehose.v2.Fetch/Block" => { #[allow(non_camel_case_types)] @@ -643,7 +660,6 @@ pub mod fetch_server { let max_encoding_message_size = self.max_encoding_message_size; let inner = self.inner.clone(); let fut = async move { - let inner = inner.0; let method = BlockSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) @@ -662,20 +678,25 @@ pub mod fetch_server { } _ => { Box::pin(async move { - Ok( - http::Response::builder() - .status(200) - .header("grpc-status", "12") - .header("content-type", "application/grpc") - .body(empty_body()) - .unwrap(), - ) + let mut response = http::Response::new(empty_body()); + let headers = response.headers_mut(); + headers + .insert( + tonic::Status::GRPC_STATUS, + (tonic::Code::Unimplemented as i32).into(), + ); + headers + .insert( + http::header::CONTENT_TYPE, + tonic::metadata::GRPC_CONTENT_TYPE, + ); + Ok(response) }) } } } } - impl Clone for FetchServer { + impl Clone for FetchServer { fn clone(&self) -> Self { let inner = self.inner.clone(); Self { @@ -687,27 +708,25 @@ pub mod fetch_server { } } } - impl Clone for _Inner { - fn clone(&self) -> Self { - Self(Arc::clone(&self.0)) - } - } - impl std::fmt::Debug for _Inner { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{:?}", self.0) - } - } - impl tonic::server::NamedService for FetchServer { - const NAME: &'static str = "sf.firehose.v2.Fetch"; + /// Generated gRPC service name + pub const SERVICE_NAME: &str = "sf.firehose.v2.Fetch"; + impl tonic::server::NamedService for FetchServer { + const NAME: &'static str = SERVICE_NAME; } } /// Generated server implementations. pub mod endpoint_info_server { - #![allow(unused_variables, dead_code, missing_docs, clippy::let_unit_value)] + #![allow( + unused_variables, + dead_code, + missing_docs, + clippy::wildcard_imports, + clippy::let_unit_value, + )] use tonic::codegen::*; /// Generated trait containing gRPC methods that should be implemented for use with EndpointInfoServer. #[async_trait] - pub trait EndpointInfo: Send + Sync + 'static { + pub trait EndpointInfo: std::marker::Send + std::marker::Sync + 'static { async fn info( &self, request: tonic::Request, @@ -717,20 +736,18 @@ pub mod endpoint_info_server { >; } #[derive(Debug)] - pub struct EndpointInfoServer { - inner: _Inner, + pub struct EndpointInfoServer { + inner: Arc, accept_compression_encodings: EnabledCompressionEncodings, send_compression_encodings: EnabledCompressionEncodings, max_decoding_message_size: Option, max_encoding_message_size: Option, } - struct _Inner(Arc); - impl EndpointInfoServer { + impl EndpointInfoServer { pub fn new(inner: T) -> Self { Self::from_arc(Arc::new(inner)) } pub fn from_arc(inner: Arc) -> Self { - let inner = _Inner(inner); Self { inner, accept_compression_encodings: Default::default(), @@ -780,8 +797,8 @@ pub mod endpoint_info_server { impl tonic::codegen::Service> for EndpointInfoServer where T: EndpointInfo, - B: Body + Send + 'static, - B::Error: Into + Send + 'static, + B: Body + std::marker::Send + 'static, + B::Error: Into + std::marker::Send + 'static, { type Response = http::Response; type Error = std::convert::Infallible; @@ -793,7 +810,6 @@ pub mod endpoint_info_server { Poll::Ready(Ok(())) } fn call(&mut self, req: http::Request) -> Self::Future { - let inner = self.inner.clone(); match req.uri().path() { "/sf.firehose.v2.EndpointInfo/Info" => { #[allow(non_camel_case_types)] @@ -824,7 +840,6 @@ pub mod endpoint_info_server { let max_encoding_message_size = self.max_encoding_message_size; let inner = self.inner.clone(); let fut = async move { - let inner = inner.0; let method = InfoSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) @@ -843,20 +858,25 @@ pub mod endpoint_info_server { } _ => { Box::pin(async move { - Ok( - http::Response::builder() - .status(200) - .header("grpc-status", "12") - .header("content-type", "application/grpc") - .body(empty_body()) - .unwrap(), - ) + let mut response = http::Response::new(empty_body()); + let headers = response.headers_mut(); + headers + .insert( + tonic::Status::GRPC_STATUS, + (tonic::Code::Unimplemented as i32).into(), + ); + headers + .insert( + http::header::CONTENT_TYPE, + tonic::metadata::GRPC_CONTENT_TYPE, + ); + Ok(response) }) } } } } - impl Clone for EndpointInfoServer { + impl Clone for EndpointInfoServer { fn clone(&self) -> Self { let inner = self.inner.clone(); Self { @@ -868,17 +888,9 @@ pub mod endpoint_info_server { } } } - impl Clone for _Inner { - fn clone(&self) -> Self { - Self(Arc::clone(&self.0)) - } - } - impl std::fmt::Debug for _Inner { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{:?}", self.0) - } - } - impl tonic::server::NamedService for EndpointInfoServer { - const NAME: &'static str = "sf.firehose.v2.EndpointInfo"; + /// Generated gRPC service name + pub const SERVICE_NAME: &str = "sf.firehose.v2.EndpointInfo"; + impl tonic::server::NamedService for EndpointInfoServer { + const NAME: &'static str = SERVICE_NAME; } } diff --git a/graph/src/substreams_rpc/sf.substreams.rpc.v2.rs b/graph/src/substreams_rpc/sf.substreams.rpc.v2.rs index 38a793e666f..ff69b343d29 100644 --- a/graph/src/substreams_rpc/sf.substreams.rpc.v2.rs +++ b/graph/src/substreams_rpc/sf.substreams.rpc.v2.rs @@ -1,5 +1,4 @@ // This file is @generated by prost-build. -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct Request { #[prost(int64, tag = "1")] @@ -47,7 +46,6 @@ pub struct Request { ::prost::alloc::string::String, >, } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct Response { #[prost(oneof = "response::Message", tags = "1, 2, 3, 4, 5, 10, 11")] @@ -55,7 +53,6 @@ pub struct Response { } /// Nested message and enum types in `Response`. pub mod response { - #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Oneof)] pub enum Message { /// Always sent first @@ -84,7 +81,6 @@ pub mod response { /// BlockUndoSignal informs you that every bit of data /// with a block number above 'last_valid_block' has been reverted /// on-chain. Delete that data and restart from 'last_valid_cursor' -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct BlockUndoSignal { #[prost(message, optional, tag = "1")] @@ -92,7 +88,6 @@ pub struct BlockUndoSignal { #[prost(string, tag = "2")] pub last_valid_cursor: ::prost::alloc::string::String, } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct BlockScopedData { #[prost(message, optional, tag = "1")] @@ -109,7 +104,6 @@ pub struct BlockScopedData { #[prost(message, repeated, tag = "11")] pub debug_store_outputs: ::prost::alloc::vec::Vec, } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct SessionInit { #[prost(string, tag = "1")] @@ -121,13 +115,11 @@ pub struct SessionInit { #[prost(uint64, tag = "4")] pub max_parallel_workers: u64, } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct InitialSnapshotComplete { #[prost(string, tag = "1")] pub cursor: ::prost::alloc::string::String, } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct InitialSnapshotData { #[prost(string, tag = "1")] @@ -139,7 +131,6 @@ pub struct InitialSnapshotData { #[prost(uint64, tag = "3")] pub total_keys: u64, } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct MapModuleOutput { #[prost(string, tag = "1")] @@ -154,7 +145,6 @@ pub struct MapModuleOutput { /// It is not possible to retrieve store models in production, with /// parallelization enabled. If you need the deltas directly, write a pass /// through mapper module that will get them down to you. -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct StoreModuleOutput { #[prost(string, tag = "1")] @@ -164,7 +154,6 @@ pub struct StoreModuleOutput { #[prost(message, optional, tag = "10")] pub debug_info: ::core::option::Option, } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct OutputDebugInfo { #[prost(string, repeated, tag = "1")] @@ -178,7 +167,6 @@ pub struct OutputDebugInfo { pub cached: bool, } /// ModulesProgress is a message that is sent every 500ms -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct ModulesProgress { /// List of jobs running on tier2 servers @@ -193,15 +181,13 @@ pub struct ModulesProgress { #[prost(message, optional, tag = "5")] pub processed_bytes: ::core::option::Option, } -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, ::prost::Message)] pub struct ProcessedBytes { #[prost(uint64, tag = "1")] pub total_bytes_read: u64, #[prost(uint64, tag = "2")] pub total_bytes_written: u64, } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct Error { #[prost(string, tag = "1")] @@ -216,8 +202,7 @@ pub struct Error { #[prost(bool, tag = "4")] pub logs_truncated: bool, } -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, ::prost::Message)] pub struct Job { #[prost(uint32, tag = "1")] pub stage: u32, @@ -230,7 +215,6 @@ pub struct Job { #[prost(uint64, tag = "5")] pub duration_ms: u64, } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct Stage { #[prost(string, repeated, tag = "1")] @@ -241,7 +225,6 @@ pub struct Stage { /// ModuleStats gathers metrics and statistics from each module, running on tier1 /// or tier2 All the 'count' and 'time_ms' values may include duplicate for each /// stage going over that module -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct ModuleStats { /// name of the module @@ -291,7 +274,6 @@ pub struct ModuleStats { #[prost(uint64, tag = "15")] pub highest_contiguous_block: u64, } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct ExternalCallMetric { #[prost(string, tag = "1")] @@ -301,7 +283,6 @@ pub struct ExternalCallMetric { #[prost(uint64, tag = "3")] pub time_ms: u64, } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct StoreDelta { #[prost(enumeration = "store_delta::Operation", tag = "1")] @@ -342,10 +323,10 @@ pub mod store_delta { /// (if the ProtoBuf definition does not change) and safe for programmatic use. pub fn as_str_name(&self) -> &'static str { match self { - Operation::Unset => "UNSET", - Operation::Create => "CREATE", - Operation::Update => "UPDATE", - Operation::Delete => "DELETE", + Self::Unset => "UNSET", + Self::Create => "CREATE", + Self::Update => "UPDATE", + Self::Delete => "DELETE", } } /// Creates an enum from field names used in the ProtoBuf definition. @@ -360,8 +341,7 @@ pub mod store_delta { } } } -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] +#[derive(Clone, Copy, PartialEq, ::prost::Message)] pub struct BlockRange { #[prost(uint64, tag = "2")] pub start_block: u64, @@ -370,7 +350,13 @@ pub struct BlockRange { } /// Generated client implementations. pub mod endpoint_info_client { - #![allow(unused_variables, dead_code, missing_docs, clippy::let_unit_value)] + #![allow( + unused_variables, + dead_code, + missing_docs, + clippy::wildcard_imports, + clippy::let_unit_value, + )] use tonic::codegen::*; use tonic::codegen::http::Uri; #[derive(Debug, Clone)] @@ -392,8 +378,8 @@ pub mod endpoint_info_client { where T: tonic::client::GrpcService, T::Error: Into, - T::ResponseBody: Body + Send + 'static, - ::Error: Into + Send, + T::ResponseBody: Body + std::marker::Send + 'static, + ::Error: Into + std::marker::Send, { pub fn new(inner: T) -> Self { let inner = tonic::client::Grpc::new(inner); @@ -418,7 +404,7 @@ pub mod endpoint_info_client { >, , - >>::Error: Into + Send + Sync, + >>::Error: Into + std::marker::Send + std::marker::Sync, { EndpointInfoClient::new(InterceptedService::new(inner, interceptor)) } @@ -464,8 +450,7 @@ pub mod endpoint_info_client { .ready() .await .map_err(|e| { - tonic::Status::new( - tonic::Code::Unknown, + tonic::Status::unknown( format!("Service was not ready: {}", e.into()), ) })?; @@ -482,7 +467,13 @@ pub mod endpoint_info_client { } /// Generated client implementations. pub mod stream_client { - #![allow(unused_variables, dead_code, missing_docs, clippy::let_unit_value)] + #![allow( + unused_variables, + dead_code, + missing_docs, + clippy::wildcard_imports, + clippy::let_unit_value, + )] use tonic::codegen::*; use tonic::codegen::http::Uri; #[derive(Debug, Clone)] @@ -504,8 +495,8 @@ pub mod stream_client { where T: tonic::client::GrpcService, T::Error: Into, - T::ResponseBody: Body + Send + 'static, - ::Error: Into + Send, + T::ResponseBody: Body + std::marker::Send + 'static, + ::Error: Into + std::marker::Send, { pub fn new(inner: T) -> Self { let inner = tonic::client::Grpc::new(inner); @@ -530,7 +521,7 @@ pub mod stream_client { >, , - >>::Error: Into + Send + Sync, + >>::Error: Into + std::marker::Send + std::marker::Sync, { StreamClient::new(InterceptedService::new(inner, interceptor)) } @@ -576,8 +567,7 @@ pub mod stream_client { .ready() .await .map_err(|e| { - tonic::Status::new( - tonic::Code::Unknown, + tonic::Status::unknown( format!("Service was not ready: {}", e.into()), ) })?; @@ -594,11 +584,17 @@ pub mod stream_client { } /// Generated server implementations. pub mod endpoint_info_server { - #![allow(unused_variables, dead_code, missing_docs, clippy::let_unit_value)] + #![allow( + unused_variables, + dead_code, + missing_docs, + clippy::wildcard_imports, + clippy::let_unit_value, + )] use tonic::codegen::*; /// Generated trait containing gRPC methods that should be implemented for use with EndpointInfoServer. #[async_trait] - pub trait EndpointInfo: Send + Sync + 'static { + pub trait EndpointInfo: std::marker::Send + std::marker::Sync + 'static { async fn info( &self, request: tonic::Request, @@ -608,20 +604,18 @@ pub mod endpoint_info_server { >; } #[derive(Debug)] - pub struct EndpointInfoServer { - inner: _Inner, + pub struct EndpointInfoServer { + inner: Arc, accept_compression_encodings: EnabledCompressionEncodings, send_compression_encodings: EnabledCompressionEncodings, max_decoding_message_size: Option, max_encoding_message_size: Option, } - struct _Inner(Arc); - impl EndpointInfoServer { + impl EndpointInfoServer { pub fn new(inner: T) -> Self { Self::from_arc(Arc::new(inner)) } pub fn from_arc(inner: Arc) -> Self { - let inner = _Inner(inner); Self { inner, accept_compression_encodings: Default::default(), @@ -671,8 +665,8 @@ pub mod endpoint_info_server { impl tonic::codegen::Service> for EndpointInfoServer where T: EndpointInfo, - B: Body + Send + 'static, - B::Error: Into + Send + 'static, + B: Body + std::marker::Send + 'static, + B::Error: Into + std::marker::Send + 'static, { type Response = http::Response; type Error = std::convert::Infallible; @@ -684,7 +678,6 @@ pub mod endpoint_info_server { Poll::Ready(Ok(())) } fn call(&mut self, req: http::Request) -> Self::Future { - let inner = self.inner.clone(); match req.uri().path() { "/sf.substreams.rpc.v2.EndpointInfo/Info" => { #[allow(non_camel_case_types)] @@ -715,7 +708,6 @@ pub mod endpoint_info_server { let max_encoding_message_size = self.max_encoding_message_size; let inner = self.inner.clone(); let fut = async move { - let inner = inner.0; let method = InfoSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) @@ -734,20 +726,25 @@ pub mod endpoint_info_server { } _ => { Box::pin(async move { - Ok( - http::Response::builder() - .status(200) - .header("grpc-status", "12") - .header("content-type", "application/grpc") - .body(empty_body()) - .unwrap(), - ) + let mut response = http::Response::new(empty_body()); + let headers = response.headers_mut(); + headers + .insert( + tonic::Status::GRPC_STATUS, + (tonic::Code::Unimplemented as i32).into(), + ); + headers + .insert( + http::header::CONTENT_TYPE, + tonic::metadata::GRPC_CONTENT_TYPE, + ); + Ok(response) }) } } } } - impl Clone for EndpointInfoServer { + impl Clone for EndpointInfoServer { fn clone(&self) -> Self { let inner = self.inner.clone(); Self { @@ -759,32 +756,30 @@ pub mod endpoint_info_server { } } } - impl Clone for _Inner { - fn clone(&self) -> Self { - Self(Arc::clone(&self.0)) - } - } - impl std::fmt::Debug for _Inner { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{:?}", self.0) - } - } - impl tonic::server::NamedService for EndpointInfoServer { - const NAME: &'static str = "sf.substreams.rpc.v2.EndpointInfo"; + /// Generated gRPC service name + pub const SERVICE_NAME: &str = "sf.substreams.rpc.v2.EndpointInfo"; + impl tonic::server::NamedService for EndpointInfoServer { + const NAME: &'static str = SERVICE_NAME; } } /// Generated server implementations. pub mod stream_server { - #![allow(unused_variables, dead_code, missing_docs, clippy::let_unit_value)] + #![allow( + unused_variables, + dead_code, + missing_docs, + clippy::wildcard_imports, + clippy::let_unit_value, + )] use tonic::codegen::*; /// Generated trait containing gRPC methods that should be implemented for use with StreamServer. #[async_trait] - pub trait Stream: Send + Sync + 'static { + pub trait Stream: std::marker::Send + std::marker::Sync + 'static { /// Server streaming response type for the Blocks method. type BlocksStream: tonic::codegen::tokio_stream::Stream< Item = std::result::Result, > - + Send + + std::marker::Send + 'static; async fn blocks( &self, @@ -792,20 +787,18 @@ pub mod stream_server { ) -> std::result::Result, tonic::Status>; } #[derive(Debug)] - pub struct StreamServer { - inner: _Inner, + pub struct StreamServer { + inner: Arc, accept_compression_encodings: EnabledCompressionEncodings, send_compression_encodings: EnabledCompressionEncodings, max_decoding_message_size: Option, max_encoding_message_size: Option, } - struct _Inner(Arc); - impl StreamServer { + impl StreamServer { pub fn new(inner: T) -> Self { Self::from_arc(Arc::new(inner)) } pub fn from_arc(inner: Arc) -> Self { - let inner = _Inner(inner); Self { inner, accept_compression_encodings: Default::default(), @@ -855,8 +848,8 @@ pub mod stream_server { impl tonic::codegen::Service> for StreamServer where T: Stream, - B: Body + Send + 'static, - B::Error: Into + Send + 'static, + B: Body + std::marker::Send + 'static, + B::Error: Into + std::marker::Send + 'static, { type Response = http::Response; type Error = std::convert::Infallible; @@ -868,7 +861,6 @@ pub mod stream_server { Poll::Ready(Ok(())) } fn call(&mut self, req: http::Request) -> Self::Future { - let inner = self.inner.clone(); match req.uri().path() { "/sf.substreams.rpc.v2.Stream/Blocks" => { #[allow(non_camel_case_types)] @@ -898,7 +890,6 @@ pub mod stream_server { let max_encoding_message_size = self.max_encoding_message_size; let inner = self.inner.clone(); let fut = async move { - let inner = inner.0; let method = BlocksSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) @@ -917,20 +908,25 @@ pub mod stream_server { } _ => { Box::pin(async move { - Ok( - http::Response::builder() - .status(200) - .header("grpc-status", "12") - .header("content-type", "application/grpc") - .body(empty_body()) - .unwrap(), - ) + let mut response = http::Response::new(empty_body()); + let headers = response.headers_mut(); + headers + .insert( + tonic::Status::GRPC_STATUS, + (tonic::Code::Unimplemented as i32).into(), + ); + headers + .insert( + http::header::CONTENT_TYPE, + tonic::metadata::GRPC_CONTENT_TYPE, + ); + Ok(response) }) } } } } - impl Clone for StreamServer { + impl Clone for StreamServer { fn clone(&self) -> Self { let inner = self.inner.clone(); Self { @@ -942,17 +938,9 @@ pub mod stream_server { } } } - impl Clone for _Inner { - fn clone(&self) -> Self { - Self(Arc::clone(&self.0)) - } - } - impl std::fmt::Debug for _Inner { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{:?}", self.0) - } - } - impl tonic::server::NamedService for StreamServer { - const NAME: &'static str = "sf.substreams.rpc.v2.Stream"; + /// Generated gRPC service name + pub const SERVICE_NAME: &str = "sf.substreams.rpc.v2.Stream"; + impl tonic::server::NamedService for StreamServer { + const NAME: &'static str = SERVICE_NAME; } } diff --git a/substreams/substreams-trigger-filter/Cargo.toml b/substreams/substreams-trigger-filter/Cargo.toml index a3c736f2c92..f1880c3412b 100644 --- a/substreams/substreams-trigger-filter/Cargo.toml +++ b/substreams/substreams-trigger-filter/Cargo.toml @@ -10,12 +10,12 @@ crate-type = ["cdylib"] [dependencies] hex = { version = "0.4", default-features = false } -prost = "0.11.9" -substreams = "0.5" -substreams-entity-change = "1.3" -substreams-near-core = "0.10.1" +prost.workspace = true +substreams.workspace = true +substreams-entity-change.workspace = true +substreams-near-core.workspace = true trigger-filters.path = "../trigger-filters" [build-dependencies] -tonic-build = { version = "0.11.0", features = ["prost"] } +tonic-build.workspace = true diff --git a/substreams/substreams-trigger-filter/build.rs b/substreams/substreams-trigger-filter/build.rs index 3bb9182d874..22b972babc5 100644 --- a/substreams/substreams-trigger-filter/build.rs +++ b/substreams/substreams-trigger-filter/build.rs @@ -6,12 +6,7 @@ fn main() { ".sf.near.codec.v1", "::substreams_near_core::pb::sf::near::type::v1", ) - // .extern_path( - // ".sf.ethereum.type.v2", - // "graph_chain_ethereum::codec::pbcodec", - // ) - // .extern_path(".sf.arweave.type.v1", "graph_chain_arweave::codec::pbcodec") .out_dir("src/pb") - .compile(&["proto/receipts.proto"], &["proto"]) + .compile_protos(&["proto/receipts.proto"], &["proto"]) .expect("Failed to compile Substreams entity proto(s)"); } diff --git a/substreams/substreams-trigger-filter/src/pb/receipts.v1.rs b/substreams/substreams-trigger-filter/src/pb/receipts.v1.rs index dc5b47203ef..76b6d1fe456 100644 --- a/substreams/substreams-trigger-filter/src/pb/receipts.v1.rs +++ b/substreams/substreams-trigger-filter/src/pb/receipts.v1.rs @@ -1,5 +1,4 @@ // This file is @generated by prost-build. -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct BlockAndReceipts { #[prost(message, optional, tag = "1")] diff --git a/substreams/substreams-trigger-filter/src/pb/sf.near.type.v1.rs b/substreams/substreams-trigger-filter/src/pb/sf.near.type.v1.rs deleted file mode 100644 index ed60d39b47e..00000000000 --- a/substreams/substreams-trigger-filter/src/pb/sf.near.type.v1.rs +++ /dev/null @@ -1,1181 +0,0 @@ -// @generated -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Block { - #[prost(string, tag="1")] - pub author: ::prost::alloc::string::String, - #[prost(message, optional, tag="2")] - pub header: ::core::option::Option, - #[prost(message, repeated, tag="3")] - pub chunk_headers: ::prost::alloc::vec::Vec, - #[prost(message, repeated, tag="4")] - pub shards: ::prost::alloc::vec::Vec, - #[prost(message, repeated, tag="5")] - pub state_changes: ::prost::alloc::vec::Vec, -} -/// HeaderOnlyBlock is a standard \[Block\] structure where all other fields are -/// removed so that hydrating that object from a \[Block\] bytes payload will -/// drastically reduced allocated memory required to hold the full block. -/// -/// This can be used to unpack a \[Block\] when only the \[BlockHeader\] information -/// is required and greatly reduced required memory. -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct HeaderOnlyBlock { - #[prost(message, optional, tag="2")] - pub header: ::core::option::Option, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct StateChangeWithCause { - #[prost(message, optional, tag="1")] - pub value: ::core::option::Option, - #[prost(message, optional, tag="2")] - pub cause: ::core::option::Option, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct StateChangeCause { - #[prost(oneof="state_change_cause::Cause", tags="1, 2, 3, 4, 5, 6, 7, 8, 9, 10")] - pub cause: ::core::option::Option, -} -/// Nested message and enum types in `StateChangeCause`. -pub mod state_change_cause { - #[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] - pub struct NotWritableToDisk { - } - #[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] - pub struct InitialState { - } - #[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] - pub struct TransactionProcessing { - #[prost(message, optional, tag="1")] - pub tx_hash: ::core::option::Option, - } - #[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] - pub struct ActionReceiptProcessingStarted { - #[prost(message, optional, tag="1")] - pub receipt_hash: ::core::option::Option, - } - #[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] - pub struct ActionReceiptGasReward { - #[prost(message, optional, tag="1")] - pub tx_hash: ::core::option::Option, - } - #[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] - pub struct ReceiptProcessing { - #[prost(message, optional, tag="1")] - pub tx_hash: ::core::option::Option, - } - #[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] - pub struct PostponedReceipt { - #[prost(message, optional, tag="1")] - pub tx_hash: ::core::option::Option, - } - #[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] - pub struct UpdatedDelayedReceipts { - } - #[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] - pub struct ValidatorAccountsUpdate { - } - #[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] - pub struct Migration { - } - #[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Oneof)] - pub enum Cause { - #[prost(message, tag="1")] - NotWritableToDisk(NotWritableToDisk), - #[prost(message, tag="2")] - InitialState(InitialState), - #[prost(message, tag="3")] - TransactionProcessing(TransactionProcessing), - #[prost(message, tag="4")] - ActionReceiptProcessingStarted(ActionReceiptProcessingStarted), - #[prost(message, tag="5")] - ActionReceiptGasReward(ActionReceiptGasReward), - #[prost(message, tag="6")] - ReceiptProcessing(ReceiptProcessing), - #[prost(message, tag="7")] - PostponedReceipt(PostponedReceipt), - #[prost(message, tag="8")] - UpdatedDelayedReceipts(UpdatedDelayedReceipts), - #[prost(message, tag="9")] - ValidatorAccountsUpdate(ValidatorAccountsUpdate), - #[prost(message, tag="10")] - Migration(Migration), - } -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct StateChangeValue { - #[prost(oneof="state_change_value::Value", tags="1, 2, 3, 4, 5, 6, 7, 8")] - pub value: ::core::option::Option, -} -/// Nested message and enum types in `StateChangeValue`. -pub mod state_change_value { - #[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] - pub struct AccountUpdate { - #[prost(string, tag="1")] - pub account_id: ::prost::alloc::string::String, - #[prost(message, optional, tag="2")] - pub account: ::core::option::Option, - } - #[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] - pub struct AccountDeletion { - #[prost(string, tag="1")] - pub account_id: ::prost::alloc::string::String, - } - #[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] - pub struct AccessKeyUpdate { - #[prost(string, tag="1")] - pub account_id: ::prost::alloc::string::String, - #[prost(message, optional, tag="2")] - pub public_key: ::core::option::Option, - #[prost(message, optional, tag="3")] - pub access_key: ::core::option::Option, - } - #[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] - pub struct AccessKeyDeletion { - #[prost(string, tag="1")] - pub account_id: ::prost::alloc::string::String, - #[prost(message, optional, tag="2")] - pub public_key: ::core::option::Option, - } - #[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] - pub struct DataUpdate { - #[prost(string, tag="1")] - pub account_id: ::prost::alloc::string::String, - #[prost(bytes="vec", tag="2")] - pub key: ::prost::alloc::vec::Vec, - #[prost(bytes="vec", tag="3")] - pub value: ::prost::alloc::vec::Vec, - } - #[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] - pub struct DataDeletion { - #[prost(string, tag="1")] - pub account_id: ::prost::alloc::string::String, - #[prost(bytes="vec", tag="2")] - pub key: ::prost::alloc::vec::Vec, - } - #[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] - pub struct ContractCodeUpdate { - #[prost(string, tag="1")] - pub account_id: ::prost::alloc::string::String, - #[prost(bytes="vec", tag="2")] - pub code: ::prost::alloc::vec::Vec, - } - #[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] - pub struct ContractCodeDeletion { - #[prost(string, tag="1")] - pub account_id: ::prost::alloc::string::String, - } - #[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Oneof)] - pub enum Value { - #[prost(message, tag="1")] - AccountUpdate(AccountUpdate), - #[prost(message, tag="2")] - AccountDeletion(AccountDeletion), - #[prost(message, tag="3")] - AccessKeyUpdate(AccessKeyUpdate), - #[prost(message, tag="4")] - AccessKeyDeletion(AccessKeyDeletion), - #[prost(message, tag="5")] - DataUpdate(DataUpdate), - #[prost(message, tag="6")] - DataDeletion(DataDeletion), - #[prost(message, tag="7")] - ContractCodeUpdate(ContractCodeUpdate), - #[prost(message, tag="8")] - ContractDeletion(ContractCodeDeletion), - } -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Account { - #[prost(message, optional, tag="1")] - pub amount: ::core::option::Option, - #[prost(message, optional, tag="2")] - pub locked: ::core::option::Option, - #[prost(message, optional, tag="3")] - pub code_hash: ::core::option::Option, - #[prost(uint64, tag="4")] - pub storage_usage: u64, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct BlockHeader { - #[prost(uint64, tag="1")] - pub height: u64, - #[prost(uint64, tag="2")] - pub prev_height: u64, - #[prost(message, optional, tag="3")] - pub epoch_id: ::core::option::Option, - #[prost(message, optional, tag="4")] - pub next_epoch_id: ::core::option::Option, - #[prost(message, optional, tag="5")] - pub hash: ::core::option::Option, - #[prost(message, optional, tag="6")] - pub prev_hash: ::core::option::Option, - #[prost(message, optional, tag="7")] - pub prev_state_root: ::core::option::Option, - #[prost(message, optional, tag="8")] - pub chunk_receipts_root: ::core::option::Option, - #[prost(message, optional, tag="9")] - pub chunk_headers_root: ::core::option::Option, - #[prost(message, optional, tag="10")] - pub chunk_tx_root: ::core::option::Option, - #[prost(message, optional, tag="11")] - pub outcome_root: ::core::option::Option, - #[prost(uint64, tag="12")] - pub chunks_included: u64, - #[prost(message, optional, tag="13")] - pub challenges_root: ::core::option::Option, - #[prost(uint64, tag="14")] - pub timestamp: u64, - #[prost(uint64, tag="15")] - pub timestamp_nanosec: u64, - #[prost(message, optional, tag="16")] - pub random_value: ::core::option::Option, - #[prost(message, repeated, tag="17")] - pub validator_proposals: ::prost::alloc::vec::Vec, - #[prost(bool, repeated, tag="18")] - pub chunk_mask: ::prost::alloc::vec::Vec, - #[prost(message, optional, tag="19")] - pub gas_price: ::core::option::Option, - #[prost(uint64, tag="20")] - pub block_ordinal: u64, - #[prost(message, optional, tag="21")] - pub total_supply: ::core::option::Option, - #[prost(message, repeated, tag="22")] - pub challenges_result: ::prost::alloc::vec::Vec, - #[prost(uint64, tag="23")] - pub last_final_block_height: u64, - #[prost(message, optional, tag="24")] - pub last_final_block: ::core::option::Option, - #[prost(uint64, tag="25")] - pub last_ds_final_block_height: u64, - #[prost(message, optional, tag="26")] - pub last_ds_final_block: ::core::option::Option, - #[prost(message, optional, tag="27")] - pub next_bp_hash: ::core::option::Option, - #[prost(message, optional, tag="28")] - pub block_merkle_root: ::core::option::Option, - #[prost(bytes="vec", tag="29")] - pub epoch_sync_data_hash: ::prost::alloc::vec::Vec, - #[prost(message, repeated, tag="30")] - pub approvals: ::prost::alloc::vec::Vec, - #[prost(message, optional, tag="31")] - pub signature: ::core::option::Option, - #[prost(uint32, tag="32")] - pub latest_protocol_version: u32, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct BigInt { - #[prost(bytes="vec", tag="1")] - pub bytes: ::prost::alloc::vec::Vec, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct CryptoHash { - #[prost(bytes="vec", tag="1")] - pub bytes: ::prost::alloc::vec::Vec, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Signature { - #[prost(enumeration="CurveKind", tag="1")] - pub r#type: i32, - #[prost(bytes="vec", tag="2")] - pub bytes: ::prost::alloc::vec::Vec, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct PublicKey { - #[prost(enumeration="CurveKind", tag="1")] - pub r#type: i32, - #[prost(bytes="vec", tag="2")] - pub bytes: ::prost::alloc::vec::Vec, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ValidatorStake { - #[prost(string, tag="1")] - pub account_id: ::prost::alloc::string::String, - #[prost(message, optional, tag="2")] - pub public_key: ::core::option::Option, - #[prost(message, optional, tag="3")] - pub stake: ::core::option::Option, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct SlashedValidator { - #[prost(string, tag="1")] - pub account_id: ::prost::alloc::string::String, - #[prost(bool, tag="2")] - pub is_double_sign: bool, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ChunkHeader { - #[prost(bytes="vec", tag="1")] - pub chunk_hash: ::prost::alloc::vec::Vec, - #[prost(bytes="vec", tag="2")] - pub prev_block_hash: ::prost::alloc::vec::Vec, - #[prost(bytes="vec", tag="3")] - pub outcome_root: ::prost::alloc::vec::Vec, - #[prost(bytes="vec", tag="4")] - pub prev_state_root: ::prost::alloc::vec::Vec, - #[prost(bytes="vec", tag="5")] - pub encoded_merkle_root: ::prost::alloc::vec::Vec, - #[prost(uint64, tag="6")] - pub encoded_length: u64, - #[prost(uint64, tag="7")] - pub height_created: u64, - #[prost(uint64, tag="8")] - pub height_included: u64, - #[prost(uint64, tag="9")] - pub shard_id: u64, - #[prost(uint64, tag="10")] - pub gas_used: u64, - #[prost(uint64, tag="11")] - pub gas_limit: u64, - #[prost(message, optional, tag="12")] - pub validator_reward: ::core::option::Option, - #[prost(message, optional, tag="13")] - pub balance_burnt: ::core::option::Option, - #[prost(bytes="vec", tag="14")] - pub outgoing_receipts_root: ::prost::alloc::vec::Vec, - #[prost(bytes="vec", tag="15")] - pub tx_root: ::prost::alloc::vec::Vec, - #[prost(message, repeated, tag="16")] - pub validator_proposals: ::prost::alloc::vec::Vec, - #[prost(message, optional, tag="17")] - pub signature: ::core::option::Option, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct IndexerShard { - #[prost(uint64, tag="1")] - pub shard_id: u64, - #[prost(message, optional, tag="2")] - pub chunk: ::core::option::Option, - #[prost(message, repeated, tag="3")] - pub receipt_execution_outcomes: ::prost::alloc::vec::Vec, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct IndexerExecutionOutcomeWithReceipt { - #[prost(message, optional, tag="1")] - pub execution_outcome: ::core::option::Option, - #[prost(message, optional, tag="2")] - pub receipt: ::core::option::Option, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct IndexerChunk { - #[prost(string, tag="1")] - pub author: ::prost::alloc::string::String, - #[prost(message, optional, tag="2")] - pub header: ::core::option::Option, - #[prost(message, repeated, tag="3")] - pub transactions: ::prost::alloc::vec::Vec, - #[prost(message, repeated, tag="4")] - pub receipts: ::prost::alloc::vec::Vec, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct IndexerTransactionWithOutcome { - #[prost(message, optional, tag="1")] - pub transaction: ::core::option::Option, - #[prost(message, optional, tag="2")] - pub outcome: ::core::option::Option, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct SignedTransaction { - #[prost(string, tag="1")] - pub signer_id: ::prost::alloc::string::String, - #[prost(message, optional, tag="2")] - pub public_key: ::core::option::Option, - #[prost(uint64, tag="3")] - pub nonce: u64, - #[prost(string, tag="4")] - pub receiver_id: ::prost::alloc::string::String, - #[prost(message, repeated, tag="5")] - pub actions: ::prost::alloc::vec::Vec, - #[prost(message, optional, tag="6")] - pub signature: ::core::option::Option, - #[prost(message, optional, tag="7")] - pub hash: ::core::option::Option, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct IndexerExecutionOutcomeWithOptionalReceipt { - #[prost(message, optional, tag="1")] - pub execution_outcome: ::core::option::Option, - #[prost(message, optional, tag="2")] - pub receipt: ::core::option::Option, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Receipt { - #[prost(string, tag="1")] - pub predecessor_id: ::prost::alloc::string::String, - #[prost(string, tag="2")] - pub receiver_id: ::prost::alloc::string::String, - #[prost(message, optional, tag="3")] - pub receipt_id: ::core::option::Option, - #[prost(oneof="receipt::Receipt", tags="10, 11")] - pub receipt: ::core::option::Option, -} -/// Nested message and enum types in `Receipt`. -pub mod receipt { - #[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Oneof)] - pub enum Receipt { - #[prost(message, tag="10")] - Action(super::ReceiptAction), - #[prost(message, tag="11")] - Data(super::ReceiptData), - } -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ReceiptData { - #[prost(message, optional, tag="1")] - pub data_id: ::core::option::Option, - #[prost(bytes="vec", tag="2")] - pub data: ::prost::alloc::vec::Vec, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ReceiptAction { - #[prost(string, tag="1")] - pub signer_id: ::prost::alloc::string::String, - #[prost(message, optional, tag="2")] - pub signer_public_key: ::core::option::Option, - #[prost(message, optional, tag="3")] - pub gas_price: ::core::option::Option, - #[prost(message, repeated, tag="4")] - pub output_data_receivers: ::prost::alloc::vec::Vec, - #[prost(message, repeated, tag="5")] - pub input_data_ids: ::prost::alloc::vec::Vec, - #[prost(message, repeated, tag="6")] - pub actions: ::prost::alloc::vec::Vec, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct DataReceiver { - #[prost(message, optional, tag="1")] - pub data_id: ::core::option::Option, - #[prost(string, tag="2")] - pub receiver_id: ::prost::alloc::string::String, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ExecutionOutcomeWithId { - #[prost(message, optional, tag="1")] - pub proof: ::core::option::Option, - #[prost(message, optional, tag="2")] - pub block_hash: ::core::option::Option, - #[prost(message, optional, tag="3")] - pub id: ::core::option::Option, - #[prost(message, optional, tag="4")] - pub outcome: ::core::option::Option, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ExecutionOutcome { - #[prost(string, repeated, tag="1")] - pub logs: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, - #[prost(message, repeated, tag="2")] - pub receipt_ids: ::prost::alloc::vec::Vec, - #[prost(uint64, tag="3")] - pub gas_burnt: u64, - #[prost(message, optional, tag="4")] - pub tokens_burnt: ::core::option::Option, - #[prost(string, tag="5")] - pub executor_id: ::prost::alloc::string::String, - #[prost(enumeration="ExecutionMetadata", tag="6")] - pub metadata: i32, - #[prost(oneof="execution_outcome::Status", tags="20, 21, 22, 23")] - pub status: ::core::option::Option, -} -/// Nested message and enum types in `ExecutionOutcome`. -pub mod execution_outcome { - #[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Oneof)] - pub enum Status { - #[prost(message, tag="20")] - Unknown(super::UnknownExecutionStatus), - #[prost(message, tag="21")] - Failure(super::FailureExecutionStatus), - #[prost(message, tag="22")] - SuccessValue(super::SuccessValueExecutionStatus), - #[prost(message, tag="23")] - SuccessReceiptId(super::SuccessReceiptIdExecutionStatus), - } -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct SuccessValueExecutionStatus { - #[prost(bytes="vec", tag="1")] - pub value: ::prost::alloc::vec::Vec, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct SuccessReceiptIdExecutionStatus { - #[prost(message, optional, tag="1")] - pub id: ::core::option::Option, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct UnknownExecutionStatus { -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct FailureExecutionStatus { - #[prost(oneof="failure_execution_status::Failure", tags="1, 2")] - pub failure: ::core::option::Option, -} -/// Nested message and enum types in `FailureExecutionStatus`. -pub mod failure_execution_status { - #[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Oneof)] - pub enum Failure { - #[prost(message, tag="1")] - ActionError(super::ActionError), - #[prost(enumeration="super::InvalidTxError", tag="2")] - InvalidTxError(i32), - } -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ActionError { - #[prost(uint64, tag="1")] - pub index: u64, - #[prost(oneof="action_error::Kind", tags="21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42")] - pub kind: ::core::option::Option, -} -/// Nested message and enum types in `ActionError`. -pub mod action_error { - #[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Oneof)] - pub enum Kind { - #[prost(message, tag="21")] - AccountAlreadyExist(super::AccountAlreadyExistsErrorKind), - #[prost(message, tag="22")] - AccountDoesNotExist(super::AccountDoesNotExistErrorKind), - #[prost(message, tag="23")] - CreateAccountOnlyByRegistrar(super::CreateAccountOnlyByRegistrarErrorKind), - #[prost(message, tag="24")] - CreateAccountNotAllowed(super::CreateAccountNotAllowedErrorKind), - #[prost(message, tag="25")] - ActorNoPermission(super::ActorNoPermissionErrorKind), - #[prost(message, tag="26")] - DeleteKeyDoesNotExist(super::DeleteKeyDoesNotExistErrorKind), - #[prost(message, tag="27")] - AddKeyAlreadyExists(super::AddKeyAlreadyExistsErrorKind), - #[prost(message, tag="28")] - DeleteAccountStaking(super::DeleteAccountStakingErrorKind), - #[prost(message, tag="29")] - LackBalanceForState(super::LackBalanceForStateErrorKind), - #[prost(message, tag="30")] - TriesToUnstake(super::TriesToUnstakeErrorKind), - #[prost(message, tag="31")] - TriesToStake(super::TriesToStakeErrorKind), - #[prost(message, tag="32")] - InsufficientStake(super::InsufficientStakeErrorKind), - #[prost(message, tag="33")] - FunctionCall(super::FunctionCallErrorKind), - #[prost(message, tag="34")] - NewReceiptValidation(super::NewReceiptValidationErrorKind), - #[prost(message, tag="35")] - OnlyImplicitAccountCreationAllowed(super::OnlyImplicitAccountCreationAllowedErrorKind), - #[prost(message, tag="36")] - DeleteAccountWithLargeState(super::DeleteAccountWithLargeStateErrorKind), - #[prost(message, tag="37")] - DelegateActionInvalidSignature(super::DelegateActionInvalidSignatureKind), - #[prost(message, tag="38")] - DelegateActionSenderDoesNotMatchTxReceiver(super::DelegateActionSenderDoesNotMatchTxReceiverKind), - #[prost(message, tag="39")] - DelegateActionExpired(super::DelegateActionExpiredKind), - #[prost(message, tag="40")] - DelegateActionAccessKeyError(super::DelegateActionAccessKeyErrorKind), - #[prost(message, tag="41")] - DelegateActionInvalidNonce(super::DelegateActionInvalidNonceKind), - #[prost(message, tag="42")] - DelegateActionNonceTooLarge(super::DelegateActionNonceTooLargeKind), - } -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct AccountAlreadyExistsErrorKind { - #[prost(string, tag="1")] - pub account_id: ::prost::alloc::string::String, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct AccountDoesNotExistErrorKind { - #[prost(string, tag="1")] - pub account_id: ::prost::alloc::string::String, -} -/// / A top-level account ID can only be created by registrar. -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct CreateAccountOnlyByRegistrarErrorKind { - #[prost(string, tag="1")] - pub account_id: ::prost::alloc::string::String, - #[prost(string, tag="2")] - pub registrar_account_id: ::prost::alloc::string::String, - #[prost(string, tag="3")] - pub predecessor_id: ::prost::alloc::string::String, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct CreateAccountNotAllowedErrorKind { - #[prost(string, tag="1")] - pub account_id: ::prost::alloc::string::String, - #[prost(string, tag="2")] - pub predecessor_id: ::prost::alloc::string::String, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct ActorNoPermissionErrorKind { - #[prost(string, tag="1")] - pub account_id: ::prost::alloc::string::String, - #[prost(string, tag="2")] - pub actor_id: ::prost::alloc::string::String, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct DeleteKeyDoesNotExistErrorKind { - #[prost(string, tag="1")] - pub account_id: ::prost::alloc::string::String, - #[prost(message, optional, tag="2")] - pub public_key: ::core::option::Option, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct AddKeyAlreadyExistsErrorKind { - #[prost(string, tag="1")] - pub account_id: ::prost::alloc::string::String, - #[prost(message, optional, tag="2")] - pub public_key: ::core::option::Option, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct DeleteAccountStakingErrorKind { - #[prost(string, tag="1")] - pub account_id: ::prost::alloc::string::String, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct LackBalanceForStateErrorKind { - #[prost(string, tag="1")] - pub account_id: ::prost::alloc::string::String, - #[prost(message, optional, tag="2")] - pub balance: ::core::option::Option, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct TriesToUnstakeErrorKind { - #[prost(string, tag="1")] - pub account_id: ::prost::alloc::string::String, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct TriesToStakeErrorKind { - #[prost(string, tag="1")] - pub account_id: ::prost::alloc::string::String, - #[prost(message, optional, tag="2")] - pub stake: ::core::option::Option, - #[prost(message, optional, tag="3")] - pub locked: ::core::option::Option, - #[prost(message, optional, tag="4")] - pub balance: ::core::option::Option, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct InsufficientStakeErrorKind { - #[prost(string, tag="1")] - pub account_id: ::prost::alloc::string::String, - #[prost(message, optional, tag="2")] - pub stake: ::core::option::Option, - #[prost(message, optional, tag="3")] - pub minimum_stake: ::core::option::Option, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct FunctionCallErrorKind { - #[prost(enumeration="FunctionCallErrorSer", tag="1")] - pub error: i32, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct NewReceiptValidationErrorKind { - #[prost(enumeration="ReceiptValidationError", tag="1")] - pub error: i32, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct OnlyImplicitAccountCreationAllowedErrorKind { - #[prost(string, tag="1")] - pub account_id: ::prost::alloc::string::String, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct DeleteAccountWithLargeStateErrorKind { - #[prost(string, tag="1")] - pub account_id: ::prost::alloc::string::String, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct DelegateActionInvalidSignatureKind { -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct DelegateActionSenderDoesNotMatchTxReceiverKind { - #[prost(string, tag="1")] - pub sender_id: ::prost::alloc::string::String, - #[prost(string, tag="2")] - pub receiver_id: ::prost::alloc::string::String, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct DelegateActionExpiredKind { -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct DelegateActionAccessKeyErrorKind { - /// InvalidAccessKeyError - #[prost(enumeration="InvalidTxError", tag="1")] - pub error: i32, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct DelegateActionInvalidNonceKind { - #[prost(uint64, tag="1")] - pub delegate_nonce: u64, - #[prost(uint64, tag="2")] - pub ak_nonce: u64, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct DelegateActionNonceTooLargeKind { - #[prost(uint64, tag="1")] - pub delegate_nonce: u64, - #[prost(uint64, tag="2")] - pub upper_bound: u64, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct MerklePath { - #[prost(message, repeated, tag="1")] - pub path: ::prost::alloc::vec::Vec, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct MerklePathItem { - #[prost(message, optional, tag="1")] - pub hash: ::core::option::Option, - #[prost(enumeration="Direction", tag="2")] - pub direction: i32, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Action { - #[prost(oneof="action::Action", tags="1, 2, 3, 4, 5, 6, 7, 8, 9")] - pub action: ::core::option::Option, -} -/// Nested message and enum types in `Action`. -pub mod action { - #[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Oneof)] - pub enum Action { - #[prost(message, tag="1")] - CreateAccount(super::CreateAccountAction), - #[prost(message, tag="2")] - DeployContract(super::DeployContractAction), - #[prost(message, tag="3")] - FunctionCall(super::FunctionCallAction), - #[prost(message, tag="4")] - Transfer(super::TransferAction), - #[prost(message, tag="5")] - Stake(super::StakeAction), - #[prost(message, tag="6")] - AddKey(super::AddKeyAction), - #[prost(message, tag="7")] - DeleteKey(super::DeleteKeyAction), - #[prost(message, tag="8")] - DeleteAccount(super::DeleteAccountAction), - #[prost(message, tag="9")] - Delegate(super::SignedDelegateAction), - } -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct CreateAccountAction { -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct DeployContractAction { - #[prost(bytes="vec", tag="1")] - pub code: ::prost::alloc::vec::Vec, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct FunctionCallAction { - #[prost(string, tag="1")] - pub method_name: ::prost::alloc::string::String, - #[prost(bytes="vec", tag="2")] - pub args: ::prost::alloc::vec::Vec, - #[prost(uint64, tag="3")] - pub gas: u64, - #[prost(message, optional, tag="4")] - pub deposit: ::core::option::Option, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct TransferAction { - #[prost(message, optional, tag="1")] - pub deposit: ::core::option::Option, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct StakeAction { - #[prost(message, optional, tag="1")] - pub stake: ::core::option::Option, - #[prost(message, optional, tag="2")] - pub public_key: ::core::option::Option, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct AddKeyAction { - #[prost(message, optional, tag="1")] - pub public_key: ::core::option::Option, - #[prost(message, optional, tag="2")] - pub access_key: ::core::option::Option, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct DeleteKeyAction { - #[prost(message, optional, tag="1")] - pub public_key: ::core::option::Option, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct DeleteAccountAction { - #[prost(string, tag="1")] - pub beneficiary_id: ::prost::alloc::string::String, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct SignedDelegateAction { - #[prost(message, optional, tag="1")] - pub signature: ::core::option::Option, - #[prost(message, optional, tag="2")] - pub delegate_action: ::core::option::Option, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct DelegateAction { - #[prost(string, tag="1")] - pub sender_id: ::prost::alloc::string::String, - #[prost(string, tag="2")] - pub receiver_id: ::prost::alloc::string::String, - #[prost(message, repeated, tag="3")] - pub actions: ::prost::alloc::vec::Vec, - #[prost(uint64, tag="4")] - pub nonce: u64, - #[prost(uint64, tag="5")] - pub max_block_height: u64, - #[prost(message, optional, tag="6")] - pub public_key: ::core::option::Option, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct AccessKey { - #[prost(uint64, tag="1")] - pub nonce: u64, - #[prost(message, optional, tag="2")] - pub permission: ::core::option::Option, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct AccessKeyPermission { - #[prost(oneof="access_key_permission::Permission", tags="1, 2")] - pub permission: ::core::option::Option, -} -/// Nested message and enum types in `AccessKeyPermission`. -pub mod access_key_permission { - #[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Oneof)] - pub enum Permission { - #[prost(message, tag="1")] - FunctionCall(super::FunctionCallPermission), - #[prost(message, tag="2")] - FullAccess(super::FullAccessPermission), - } -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct FunctionCallPermission { - #[prost(message, optional, tag="1")] - pub allowance: ::core::option::Option, - #[prost(string, tag="2")] - pub receiver_id: ::prost::alloc::string::String, - #[prost(string, repeated, tag="3")] - pub method_names: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, -} -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct FullAccessPermission { -} -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] -#[repr(i32)] -pub enum CurveKind { - Ed25519 = 0, - Secp256k1 = 1, -} -impl CurveKind { - /// String value of the enum field names used in the ProtoBuf definition. - /// - /// The values are not transformed in any way and thus are considered stable - /// (if the ProtoBuf definition does not change) and safe for programmatic use. - pub fn as_str_name(&self) -> &'static str { - match self { - CurveKind::Ed25519 => "ED25519", - CurveKind::Secp256k1 => "SECP256K1", - } - } - /// Creates an enum from field names used in the ProtoBuf definition. - pub fn from_str_name(value: &str) -> ::core::option::Option { - match value { - "ED25519" => Some(Self::Ed25519), - "SECP256K1" => Some(Self::Secp256k1), - _ => None, - } - } -} -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] -#[repr(i32)] -pub enum ExecutionMetadata { - V1 = 0, -} -impl ExecutionMetadata { - /// String value of the enum field names used in the ProtoBuf definition. - /// - /// The values are not transformed in any way and thus are considered stable - /// (if the ProtoBuf definition does not change) and safe for programmatic use. - pub fn as_str_name(&self) -> &'static str { - match self { - ExecutionMetadata::V1 => "ExecutionMetadataV1", - } - } - /// Creates an enum from field names used in the ProtoBuf definition. - pub fn from_str_name(value: &str) -> ::core::option::Option { - match value { - "ExecutionMetadataV1" => Some(Self::V1), - _ => None, - } - } -} -/// todo: add more detail? -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] -#[repr(i32)] -pub enum FunctionCallErrorSer { - CompilationError = 0, - LinkError = 1, - MethodResolveError = 2, - WasmTrap = 3, - WasmUnknownError = 4, - HostError = 5, - EvmError = 6, - ExecutionError = 7, -} -impl FunctionCallErrorSer { - /// String value of the enum field names used in the ProtoBuf definition. - /// - /// The values are not transformed in any way and thus are considered stable - /// (if the ProtoBuf definition does not change) and safe for programmatic use. - pub fn as_str_name(&self) -> &'static str { - match self { - FunctionCallErrorSer::CompilationError => "CompilationError", - FunctionCallErrorSer::LinkError => "LinkError", - FunctionCallErrorSer::MethodResolveError => "MethodResolveError", - FunctionCallErrorSer::WasmTrap => "WasmTrap", - FunctionCallErrorSer::WasmUnknownError => "WasmUnknownError", - FunctionCallErrorSer::HostError => "HostError", - FunctionCallErrorSer::EvmError => "_EVMError", - FunctionCallErrorSer::ExecutionError => "ExecutionError", - } - } - /// Creates an enum from field names used in the ProtoBuf definition. - pub fn from_str_name(value: &str) -> ::core::option::Option { - match value { - "CompilationError" => Some(Self::CompilationError), - "LinkError" => Some(Self::LinkError), - "MethodResolveError" => Some(Self::MethodResolveError), - "WasmTrap" => Some(Self::WasmTrap), - "WasmUnknownError" => Some(Self::WasmUnknownError), - "HostError" => Some(Self::HostError), - "_EVMError" => Some(Self::EvmError), - "ExecutionError" => Some(Self::ExecutionError), - _ => None, - } - } -} -/// todo: add more detail? -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] -#[repr(i32)] -pub enum ReceiptValidationError { - InvalidPredecessorId = 0, - InvalidReceiverAccountId = 1, - InvalidSignerAccountId = 2, - InvalidDataReceiverId = 3, - ReturnedValueLengthExceeded = 4, - NumberInputDataDependenciesExceeded = 5, - ActionsValidationError = 6, -} -impl ReceiptValidationError { - /// String value of the enum field names used in the ProtoBuf definition. - /// - /// The values are not transformed in any way and thus are considered stable - /// (if the ProtoBuf definition does not change) and safe for programmatic use. - pub fn as_str_name(&self) -> &'static str { - match self { - ReceiptValidationError::InvalidPredecessorId => "InvalidPredecessorId", - ReceiptValidationError::InvalidReceiverAccountId => "InvalidReceiverAccountId", - ReceiptValidationError::InvalidSignerAccountId => "InvalidSignerAccountId", - ReceiptValidationError::InvalidDataReceiverId => "InvalidDataReceiverId", - ReceiptValidationError::ReturnedValueLengthExceeded => "ReturnedValueLengthExceeded", - ReceiptValidationError::NumberInputDataDependenciesExceeded => "NumberInputDataDependenciesExceeded", - ReceiptValidationError::ActionsValidationError => "ActionsValidationError", - } - } - /// Creates an enum from field names used in the ProtoBuf definition. - pub fn from_str_name(value: &str) -> ::core::option::Option { - match value { - "InvalidPredecessorId" => Some(Self::InvalidPredecessorId), - "InvalidReceiverAccountId" => Some(Self::InvalidReceiverAccountId), - "InvalidSignerAccountId" => Some(Self::InvalidSignerAccountId), - "InvalidDataReceiverId" => Some(Self::InvalidDataReceiverId), - "ReturnedValueLengthExceeded" => Some(Self::ReturnedValueLengthExceeded), - "NumberInputDataDependenciesExceeded" => Some(Self::NumberInputDataDependenciesExceeded), - "ActionsValidationError" => Some(Self::ActionsValidationError), - _ => None, - } - } -} -/// todo: add more detail? -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] -#[repr(i32)] -pub enum InvalidTxError { - InvalidAccessKeyError = 0, - InvalidSignerId = 1, - SignerDoesNotExist = 2, - InvalidNonce = 3, - NonceTooLarge = 4, - InvalidReceiverId = 5, - InvalidSignature = 6, - NotEnoughBalance = 7, - LackBalanceForState = 8, - CostOverflow = 9, - InvalidChain = 10, - Expired = 11, - ActionsValidation = 12, - TransactionSizeExceeded = 13, -} -impl InvalidTxError { - /// String value of the enum field names used in the ProtoBuf definition. - /// - /// The values are not transformed in any way and thus are considered stable - /// (if the ProtoBuf definition does not change) and safe for programmatic use. - pub fn as_str_name(&self) -> &'static str { - match self { - InvalidTxError::InvalidAccessKeyError => "InvalidAccessKeyError", - InvalidTxError::InvalidSignerId => "InvalidSignerId", - InvalidTxError::SignerDoesNotExist => "SignerDoesNotExist", - InvalidTxError::InvalidNonce => "InvalidNonce", - InvalidTxError::NonceTooLarge => "NonceTooLarge", - InvalidTxError::InvalidReceiverId => "InvalidReceiverId", - InvalidTxError::InvalidSignature => "InvalidSignature", - InvalidTxError::NotEnoughBalance => "NotEnoughBalance", - InvalidTxError::LackBalanceForState => "LackBalanceForState", - InvalidTxError::CostOverflow => "CostOverflow", - InvalidTxError::InvalidChain => "InvalidChain", - InvalidTxError::Expired => "Expired", - InvalidTxError::ActionsValidation => "ActionsValidation", - InvalidTxError::TransactionSizeExceeded => "TransactionSizeExceeded", - } - } - /// Creates an enum from field names used in the ProtoBuf definition. - pub fn from_str_name(value: &str) -> ::core::option::Option { - match value { - "InvalidAccessKeyError" => Some(Self::InvalidAccessKeyError), - "InvalidSignerId" => Some(Self::InvalidSignerId), - "SignerDoesNotExist" => Some(Self::SignerDoesNotExist), - "InvalidNonce" => Some(Self::InvalidNonce), - "NonceTooLarge" => Some(Self::NonceTooLarge), - "InvalidReceiverId" => Some(Self::InvalidReceiverId), - "InvalidSignature" => Some(Self::InvalidSignature), - "NotEnoughBalance" => Some(Self::NotEnoughBalance), - "LackBalanceForState" => Some(Self::LackBalanceForState), - "CostOverflow" => Some(Self::CostOverflow), - "InvalidChain" => Some(Self::InvalidChain), - "Expired" => Some(Self::Expired), - "ActionsValidation" => Some(Self::ActionsValidation), - "TransactionSizeExceeded" => Some(Self::TransactionSizeExceeded), - _ => None, - } - } -} -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] -#[repr(i32)] -pub enum Direction { - Left = 0, - Right = 1, -} -impl Direction { - /// String value of the enum field names used in the ProtoBuf definition. - /// - /// The values are not transformed in any way and thus are considered stable - /// (if the ProtoBuf definition does not change) and safe for programmatic use. - pub fn as_str_name(&self) -> &'static str { - match self { - Direction::Left => "left", - Direction::Right => "right", - } - } - /// Creates an enum from field names used in the ProtoBuf definition. - pub fn from_str_name(value: &str) -> ::core::option::Option { - match value { - "left" => Some(Self::Left), - "right" => Some(Self::Right), - _ => None, - } - } -} -// @@protoc_insertion_point(module) From cc1f398962f8fc501024a4a0d3b6ce575cb93fb8 Mon Sep 17 00:00:00 2001 From: shiyasmohd Date: Wed, 5 Mar 2025 11:35:52 +0530 Subject: [PATCH 047/150] graph: improve error message for block not found in Ethereum network --- graph/src/data/subgraph/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/graph/src/data/subgraph/mod.rs b/graph/src/data/subgraph/mod.rs index d14b2c89b29..269038d08d0 100644 --- a/graph/src/data/subgraph/mod.rs +++ b/graph/src/data/subgraph/mod.rs @@ -348,7 +348,7 @@ pub enum SubgraphManifestValidationError { MultipleEthereumNetworks, #[error("subgraph must have at least one Ethereum network data source")] EthereumNetworkRequired, - #[error("the specified block must exist on the Ethereum network")] + #[error("the specified block {0} must exist on the Ethereum network")] BlockNotFound(String), #[error("schema validation failed: {0:?}")] SchemaValidationError(Vec), From 4191eb7877cbe6352203f6d9d676959c101e1977 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 6 Mar 2025 09:48:18 +0000 Subject: [PATCH 048/150] build(deps): bump uuid from 1.9.1 to 1.15.1 (#5864) Bumps [uuid](https://github.com/uuid-rs/uuid) from 1.9.1 to 1.15.1. - [Release notes](https://github.com/uuid-rs/uuid/releases) - [Commits](https://github.com/uuid-rs/uuid/compare/1.9.1...v1.15.1) --- updated-dependencies: - dependency-name: uuid dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 6 +++--- chain/ethereum/Cargo.toml | 2 +- core/Cargo.toml | 2 +- runtime/wasm/Cargo.toml | 2 +- store/postgres/Cargo.toml | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c49377cce75..c3b80fe7443 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5719,11 +5719,11 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.9.1" +version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5de17fd2f7da591098415cff336e12965a28061ddace43b59cb3c430179c9439" +checksum = "e0f540e3240398cce6128b64ba83fdbdd86129c16a3aa1a3a252efd66eb3d587" dependencies = [ - "getrandom 0.2.15", + "getrandom 0.3.1", "serde", ] diff --git a/chain/ethereum/Cargo.toml b/chain/ethereum/Cargo.toml index 61ae59ab4af..30378ec5586 100644 --- a/chain/ethereum/Cargo.toml +++ b/chain/ethereum/Cargo.toml @@ -22,7 +22,7 @@ graph-runtime-derive = { path = "../../runtime/derive" } [dev-dependencies] base64 = "0" -uuid = { version = "1.9.1", features = ["v4"] } +uuid = { version = "1.15.1", features = ["v4"] } [build-dependencies] tonic-build = { workspace = true } diff --git a/core/Cargo.toml b/core/Cargo.toml index 06c3d7d9862..1043fc307c2 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -23,5 +23,5 @@ anyhow = "1.0" [dev-dependencies] tower-test = { git = "https://github.com/tower-rs/tower.git" } -uuid = { version = "1.9.1", features = ["v4"] } +uuid = { version = "1.15.1", features = ["v4"] } wiremock = "0.6.1" diff --git a/runtime/wasm/Cargo.toml b/runtime/wasm/Cargo.toml index 3e74e9f985e..c728f0bfb65 100644 --- a/runtime/wasm/Cargo.toml +++ b/runtime/wasm/Cargo.toml @@ -11,7 +11,7 @@ graph = { path = "../../graph" } bs58 = "0.4.0" graph-runtime-derive = { path = "../derive" } semver = "1.0.23" -uuid = { version = "1.9.1", features = ["v4"] } +uuid = { version = "1.15.1", features = ["v4"] } anyhow = "1.0" never = "0.1" diff --git a/store/postgres/Cargo.toml b/store/postgres/Cargo.toml index a70c2b8b614..bbf08701633 100644 --- a/store/postgres/Cargo.toml +++ b/store/postgres/Cargo.toml @@ -26,7 +26,7 @@ postgres-openssl = "0.5.0" rand = "0.8.4" serde = { workspace = true } serde_json = { workspace = true } -uuid = { version = "1.9.1", features = ["v4"] } +uuid = { version = "1.15.1", features = ["v4"] } stable-hash_legacy = { git = "https://github.com/graphprotocol/stable-hash", branch = "old", package = "stable-hash" } anyhow = "1.0.86" git-testament = "0.2.5" From 1d550cb0e4a854db3d4cc5d429272a3e7e28488e Mon Sep 17 00:00:00 2001 From: Yaro Shkvorets Date: Thu, 6 Mar 2025 04:52:32 -0500 Subject: [PATCH 049/150] make manifest resolve error more informational (#5852) --- graph/src/data/subgraph/mod.rs | 2 +- graph/src/data_source/common.rs | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/graph/src/data/subgraph/mod.rs b/graph/src/data/subgraph/mod.rs index 269038d08d0..3e7bc7061ab 100644 --- a/graph/src/data/subgraph/mod.rs +++ b/graph/src/data/subgraph/mod.rs @@ -370,7 +370,7 @@ pub enum SubgraphManifestResolveError { NonUtf8, #[error("subgraph is not valid YAML")] InvalidFormat, - #[error("resolve error: {0}")] + #[error("resolve error: {0:#}")] ResolveError(#[from] anyhow::Error), } diff --git a/graph/src/data_source/common.rs b/graph/src/data_source/common.rs index a70f0ab8e17..57781815f5f 100644 --- a/graph/src/data_source/common.rs +++ b/graph/src/data_source/common.rs @@ -81,7 +81,8 @@ impl UnresolvedMappingABI { self.name, self.file.link ) })?; - let contract = Contract::load(&*contract_bytes)?; + let contract = Contract::load(&*contract_bytes) + .with_context(|| format!("failed to load ABI {}", self.name))?; Ok(MappingABI { name: self.name, contract, From 2374ddf1469247c651964de936491549c27784c3 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Mon, 3 Mar 2025 18:14:50 +0100 Subject: [PATCH 050/150] core, graph, store: Remove WritableStore.supports_proof_of_indexing Subgraphs have in fact supported proof of indexing for almost 5 years since commit 40fd330c and we can therefore stop pretending that that method would ever return `false` --- core/src/subgraph/runner.rs | 24 +++++++------------- graph/src/components/store/traits.rs | 2 -- store/postgres/src/deployment_store.rs | 18 --------------- store/postgres/src/relational.rs | 4 ---- store/postgres/src/writable.rs | 13 ----------- store/test-store/tests/graph/entity_cache.rs | 4 ---- 6 files changed, 8 insertions(+), 57 deletions(-) diff --git a/core/src/subgraph/runner.rs b/core/src/subgraph/runner.rs index 1e0fa2d4c8e..380cf70f88d 100644 --- a/core/src/subgraph/runner.rs +++ b/core/src/subgraph/runner.rs @@ -367,14 +367,10 @@ where debug!(logger, "Start processing block"; "triggers" => triggers.len()); - let proof_of_indexing = if self.inputs.store.supports_proof_of_indexing().await? { - Some(Arc::new(AtomicRefCell::new(ProofOfIndexing::new( - block_ptr.number, - self.inputs.poi_version, - )))) - } else { - None - }; + let proof_of_indexing = Some(Arc::new(AtomicRefCell::new(ProofOfIndexing::new( + block_ptr.number, + self.inputs.poi_version, + )))); // Causality region for onchain triggers. let causality_region = PoICausalityRegion::from_network(&self.inputs.network); @@ -1318,14 +1314,10 @@ where .deployment_head .set(block_ptr.number as f64); - let proof_of_indexing = if self.inputs.store.supports_proof_of_indexing().await? { - Some(Arc::new(AtomicRefCell::new(ProofOfIndexing::new( - block_ptr.number, - self.inputs.poi_version, - )))) - } else { - None - }; + let proof_of_indexing = Some(Arc::new(AtomicRefCell::new(ProofOfIndexing::new( + block_ptr.number, + self.inputs.poi_version, + )))); // Causality region for onchain triggers. let causality_region = PoICausalityRegion::from_network(&self.inputs.network); diff --git a/graph/src/components/store/traits.rs b/graph/src/components/store/traits.rs index e2cd0640ee0..27cb3768e2c 100644 --- a/graph/src/components/store/traits.rs +++ b/graph/src/components/store/traits.rs @@ -365,8 +365,6 @@ pub trait WritableStore: ReadStore + DeploymentCursorTracker { /// Set subgraph status to failed with the given error as the cause. async fn fail_subgraph(&self, error: SubgraphError) -> Result<(), StoreError>; - async fn supports_proof_of_indexing(&self) -> Result; - /// Transact the entity changes from a single block atomically into the store, and update the /// subgraph block pointer to `block_ptr_to`, and update the firehose cursor to `firehose_cursor` /// diff --git a/store/postgres/src/deployment_store.rs b/store/postgres/src/deployment_store.rs index 05f2b44a33e..b148129d924 100644 --- a/store/postgres/src/deployment_store.rs +++ b/store/postgres/src/deployment_store.rs @@ -916,20 +916,6 @@ impl DeploymentStore { } } - pub(crate) async fn supports_proof_of_indexing<'a>( - &self, - site: Arc, - ) -> Result { - let store = self.clone(); - self.with_conn(move |conn, cancel| { - cancel.check_cancel()?; - let layout = store.layout(conn, site)?; - Ok(layout.supports_proof_of_indexing()) - }) - .await - .map_err(Into::into) - } - pub(crate) async fn get_proof_of_indexing( &self, site: Arc, @@ -950,10 +936,6 @@ impl DeploymentStore { let layout = store.layout(conn, site.cheap_clone())?; - if !layout.supports_proof_of_indexing() { - return Ok(None); - } - conn.transaction::<_, CancelableError, _>(move |conn| { let mut block_ptr = block.cheap_clone(); let latest_block_ptr = diff --git a/store/postgres/src/relational.rs b/store/postgres/src/relational.rs index 8fa896c22d3..d148060efc2 100644 --- a/store/postgres/src/relational.rs +++ b/store/postgres/src/relational.rs @@ -375,10 +375,6 @@ impl Layout { } } - pub fn supports_proof_of_indexing(&self) -> bool { - self.tables.contains_key(&self.input_schema.poi_type()) - } - pub fn create_relational_schema( conn: &mut PgConnection, site: Arc, diff --git a/store/postgres/src/writable.rs b/store/postgres/src/writable.rs index 33b5b40e3b7..07d116790c0 100644 --- a/store/postgres/src/writable.rs +++ b/store/postgres/src/writable.rs @@ -279,15 +279,6 @@ impl SyncStore { .await } - async fn supports_proof_of_indexing(&self) -> Result { - retry::forever_async(&self.logger, "supports_proof_of_indexing", || async { - self.writable - .supports_proof_of_indexing(self.site.clone()) - .await - }) - .await - } - fn get(&self, key: &EntityKey, block: BlockNumber) -> Result, StoreError> { retry::forever(&self.logger, "get", || { self.writable.get(self.site.cheap_clone(), key, block) @@ -1667,10 +1658,6 @@ impl WritableStoreTrait for WritableStore { self.store.fail_subgraph(error).await } - async fn supports_proof_of_indexing(&self) -> Result { - self.store.supports_proof_of_indexing().await - } - async fn transact_block_operations( &self, block_ptr_to: BlockPtr, diff --git a/store/test-store/tests/graph/entity_cache.rs b/store/test-store/tests/graph/entity_cache.rs index d54a88751b8..c4353ffb63b 100644 --- a/store/test-store/tests/graph/entity_cache.rs +++ b/store/test-store/tests/graph/entity_cache.rs @@ -121,10 +121,6 @@ impl WritableStore for MockStore { unimplemented!() } - async fn supports_proof_of_indexing(&self) -> Result { - unimplemented!() - } - async fn transact_block_operations( &self, _: BlockPtr, From 8687adad9458ff4b92bf2c059d6fc7b66f328bf5 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Mon, 3 Mar 2025 19:50:29 +0100 Subject: [PATCH 051/150] all: Encapsulate manipulation of SharedProofOfIndexing This brings a small amount of sanity to the code using that. --- chain/substreams/src/trigger.rs | 19 +----- core/src/subgraph/context/mod.rs | 20 ++---- core/src/subgraph/runner.rs | 21 ++---- core/src/subgraph/trigger_processor.rs | 20 ++---- .../subgraph/proof_of_indexing/mod.rs | 66 ++++++++++++++++--- runtime/test/src/common.rs | 3 +- runtime/wasm/src/host_exports.rs | 19 +----- 7 files changed, 81 insertions(+), 87 deletions(-) diff --git a/chain/substreams/src/trigger.rs b/chain/substreams/src/trigger.rs index 13a5bed1a1d..405b6f8a116 100644 --- a/chain/substreams/src/trigger.rs +++ b/chain/substreams/src/trigger.rs @@ -15,7 +15,6 @@ use graph::{ substreams::Modules, }; use graph_runtime_wasm::module::ToAscPtr; -use lazy_static::__Deref; use std::{collections::BTreeSet, sync::Arc}; use crate::{Block, Chain, NoopDataSourceTemplate, ParsedChanges}; @@ -179,18 +178,6 @@ impl blockchain::TriggersAdapter for TriggersAdapter { } } -fn write_poi_event( - proof_of_indexing: &SharedProofOfIndexing, - poi_event: &ProofOfIndexingEvent, - causality_region: &str, - logger: &Logger, -) { - if let Some(proof_of_indexing) = proof_of_indexing { - let mut proof_of_indexing = proof_of_indexing.deref().borrow_mut(); - proof_of_indexing.write(logger, causality_region, poi_event); - } -} - pub struct TriggerProcessor { pub locator: DeploymentLocator, } @@ -226,8 +213,7 @@ where return Err(MappingError::Unknown(anyhow!("Detected UNSET entity operation, either a server error or there's a new type of operation and we're running an outdated protobuf"))); } ParsedChanges::Upsert { key, entity } => { - write_poi_event( - proof_of_indexing, + proof_of_indexing.write_event( &ProofOfIndexingEvent::SetEntity { entity_type: key.entity_type.typename(), id: &key.entity_id.to_string(), @@ -249,8 +235,7 @@ where let id = entity_key.entity_id.clone(); state.entity_cache.remove(entity_key); - write_poi_event( - proof_of_indexing, + proof_of_indexing.write_event( &ProofOfIndexingEvent::RemoveEntity { entity_type: entity_type.typename(), id: &id.to_string(), diff --git a/core/src/subgraph/context/mod.rs b/core/src/subgraph/context/mod.rs index ef265978ede..3f35d570a7d 100644 --- a/core/src/subgraph/context/mod.rs +++ b/core/src/subgraph/context/mod.rs @@ -126,11 +126,7 @@ impl> IndexingContext { ) -> Result { let error_count = state.deterministic_errors.len(); - if let Some(proof_of_indexing) = proof_of_indexing { - proof_of_indexing - .borrow_mut() - .start_handler(causality_region); - } + proof_of_indexing.start_handler(causality_region); let start = Instant::now(); @@ -156,16 +152,12 @@ impl> IndexingContext { let elapsed = start.elapsed().as_secs_f64(); subgraph_metrics.observe_trigger_processing_duration(elapsed); - if let Some(proof_of_indexing) = proof_of_indexing { - if state.deterministic_errors.len() != error_count { - assert!(state.deterministic_errors.len() == error_count + 1); + if state.deterministic_errors.len() != error_count { + assert!(state.deterministic_errors.len() == error_count + 1); - // If a deterministic error has happened, write a new - // ProofOfIndexingEvent::DeterministicError to the SharedProofOfIndexing. - proof_of_indexing - .borrow_mut() - .write_deterministic_error(logger, causality_region); - } + // If a deterministic error has happened, write a new + // ProofOfIndexingEvent::DeterministicError to the SharedProofOfIndexing. + proof_of_indexing.write_deterministic_error(logger, causality_region); } Ok(state) diff --git a/core/src/subgraph/runner.rs b/core/src/subgraph/runner.rs index 380cf70f88d..922c7a4003c 100644 --- a/core/src/subgraph/runner.rs +++ b/core/src/subgraph/runner.rs @@ -3,7 +3,6 @@ use crate::subgraph::error::BlockProcessingError; use crate::subgraph::inputs::IndexingInputs; use crate::subgraph::state::IndexingState; use crate::subgraph::stream::new_block_stream; -use atomic_refcell::AtomicRefCell; use graph::blockchain::block_stream::{ BlockStreamError, BlockStreamEvent, BlockWithTriggers, FirehoseCursor, }; @@ -367,10 +366,8 @@ where debug!(logger, "Start processing block"; "triggers" => triggers.len()); - let proof_of_indexing = Some(Arc::new(AtomicRefCell::new(ProofOfIndexing::new( - block_ptr.number, - self.inputs.poi_version, - )))); + let proof_of_indexing = + SharedProofOfIndexing::new(block_ptr.number, self.inputs.poi_version); // Causality region for onchain triggers. let causality_region = PoICausalityRegion::from_network(&self.inputs.network); @@ -629,8 +626,7 @@ where return Err(BlockProcessingError::Canceled); } - if let Some(proof_of_indexing) = proof_of_indexing { - let proof_of_indexing = Arc::try_unwrap(proof_of_indexing).unwrap().into_inner(); + if let Some(proof_of_indexing) = proof_of_indexing.into_inner() { update_proof_of_indexing( proof_of_indexing, block.timestamp(), @@ -1156,7 +1152,7 @@ where // PoI ignores offchain events. // See also: poi-ignores-offchain - let proof_of_indexing = None; + let proof_of_indexing = SharedProofOfIndexing::ignored(); let causality_region = ""; let trigger = TriggerData::Offchain(trigger); @@ -1314,10 +1310,8 @@ where .deployment_head .set(block_ptr.number as f64); - let proof_of_indexing = Some(Arc::new(AtomicRefCell::new(ProofOfIndexing::new( - block_ptr.number, - self.inputs.poi_version, - )))); + let proof_of_indexing = + SharedProofOfIndexing::new(block_ptr.number, self.inputs.poi_version); // Causality region for onchain triggers. let causality_region = PoICausalityRegion::from_network(&self.inputs.network); @@ -1372,8 +1366,7 @@ where return Err(BlockProcessingError::Canceled.into()); } - if let Some(proof_of_indexing) = proof_of_indexing { - let proof_of_indexing = Arc::try_unwrap(proof_of_indexing).unwrap().into_inner(); + if let Some(proof_of_indexing) = proof_of_indexing.into_inner() { update_proof_of_indexing( proof_of_indexing, block_time, diff --git a/core/src/subgraph/trigger_processor.rs b/core/src/subgraph/trigger_processor.rs index 0057e9e1354..c3123e87268 100644 --- a/core/src/subgraph/trigger_processor.rs +++ b/core/src/subgraph/trigger_processor.rs @@ -39,11 +39,7 @@ where return Ok(state); } - if let Some(proof_of_indexing) = proof_of_indexing { - proof_of_indexing - .borrow_mut() - .start_handler(causality_region); - } + proof_of_indexing.start_handler(causality_region); for HostedTrigger { host, @@ -73,16 +69,12 @@ where } } - if let Some(proof_of_indexing) = proof_of_indexing { - if state.deterministic_errors.len() != error_count { - assert!(state.deterministic_errors.len() == error_count + 1); + if state.deterministic_errors.len() != error_count { + assert!(state.deterministic_errors.len() == error_count + 1); - // If a deterministic error has happened, write a new - // ProofOfIndexingEvent::DeterministicError to the SharedProofOfIndexing. - proof_of_indexing - .borrow_mut() - .write_deterministic_error(logger, causality_region); - } + // If a deterministic error has happened, write a new + // ProofOfIndexingEvent::DeterministicError to the SharedProofOfIndexing. + proof_of_indexing.write_deterministic_error(logger, causality_region); } Ok(state) diff --git a/graph/src/components/subgraph/proof_of_indexing/mod.rs b/graph/src/components/subgraph/proof_of_indexing/mod.rs index c8dd8c13314..36eeabc22cd 100644 --- a/graph/src/components/subgraph/proof_of_indexing/mod.rs +++ b/graph/src/components/subgraph/proof_of_indexing/mod.rs @@ -3,11 +3,15 @@ mod online; mod reference; pub use event::ProofOfIndexingEvent; +use graph_derive::CheapClone; pub use online::{ProofOfIndexing, ProofOfIndexingFinisher}; pub use reference::PoICausalityRegion; use atomic_refcell::AtomicRefCell; -use std::sync::Arc; +use slog::Logger; +use std::{ops::Deref, sync::Arc}; + +use crate::prelude::BlockNumber; #[derive(Copy, Clone, Debug)] pub enum ProofOfIndexingVersion { @@ -22,15 +26,57 @@ pub enum ProofOfIndexingVersion { /// intentionally disallowed - PoI requires sequential access to the hash /// function within a given causality region even if ownership is shared across /// multiple mapping contexts. -/// -/// The Option<_> is because not all subgraphs support PoI until re-deployed. -/// Eventually this can be removed. -/// -/// This is not a great place to define this type, since the ProofOfIndexing -/// shouldn't "know" these details about wasmtime and subgraph re-deployments, -/// but the APIs that would make use of this are in graph/components so this -/// lives here for lack of a better choice. -pub type SharedProofOfIndexing = Option>>; +#[derive(Clone, CheapClone)] +pub struct SharedProofOfIndexing { + poi: Option>>, +} + +impl SharedProofOfIndexing { + pub fn new(block: BlockNumber, version: ProofOfIndexingVersion) -> Self { + SharedProofOfIndexing { + poi: Some(Arc::new(AtomicRefCell::new(ProofOfIndexing::new( + block, version, + )))), + } + } + + pub fn ignored() -> Self { + SharedProofOfIndexing { poi: None } + } + + pub fn write_event( + &self, + poi_event: &ProofOfIndexingEvent, + causality_region: &str, + logger: &Logger, + ) { + if let Some(poi) = &self.poi { + let mut poi = poi.deref().borrow_mut(); + poi.write(logger, causality_region, poi_event); + } + } + + pub fn start_handler(&self, causality_region: &str) { + if let Some(poi) = &self.poi { + let mut poi = poi.deref().borrow_mut(); + poi.start_handler(causality_region); + } + } + + pub fn write_deterministic_error(&self, logger: &Logger, causality_region: &str) { + if let Some(proof_of_indexing) = &self.poi { + proof_of_indexing + .deref() + .borrow_mut() + .write_deterministic_error(logger, causality_region); + } + } + + pub fn into_inner(self) -> Option { + self.poi + .map(|poi| Arc::try_unwrap(poi).unwrap().into_inner()) + } +} #[cfg(test)] mod tests { diff --git a/runtime/test/src/common.rs b/runtime/test/src/common.rs index 7641dd06d8b..461d4a08256 100644 --- a/runtime/test/src/common.rs +++ b/runtime/test/src/common.rs @@ -1,6 +1,7 @@ use ethabi::Contract; use graph::blockchain::BlockTime; use graph::components::store::DeploymentLocator; +use graph::components::subgraph::SharedProofOfIndexing; use graph::data::subgraph::*; use graph::data_source; use graph::data_source::common::MappingABI; @@ -127,7 +128,7 @@ pub fn mock_context( .unwrap(), Default::default(), ), - proof_of_indexing: None, + proof_of_indexing: SharedProofOfIndexing::ignored(), host_fns: Arc::new(Vec::new()), debug_fork: None, mapping_logger: Logger::root(slog::Discard, o!()), diff --git a/runtime/wasm/src/host_exports.rs b/runtime/wasm/src/host_exports.rs index bd1c8706c4a..12099c55b7e 100644 --- a/runtime/wasm/src/host_exports.rs +++ b/runtime/wasm/src/host_exports.rs @@ -1,5 +1,4 @@ use std::collections::HashMap; -use std::ops::Deref; use std::str::FromStr; use std::time::{Duration, Instant}; @@ -33,18 +32,6 @@ use crate::{error::DeterminismLevel, module::IntoTrap}; use super::module::WasmInstanceData; -fn write_poi_event( - proof_of_indexing: &SharedProofOfIndexing, - poi_event: &ProofOfIndexingEvent, - causality_region: &str, - logger: &Logger, -) { - if let Some(proof_of_indexing) = proof_of_indexing { - let mut proof_of_indexing = proof_of_indexing.deref().borrow_mut(); - proof_of_indexing.write(logger, causality_region, poi_event); - } -} - impl IntoTrap for HostExportError { fn determinism_level(&self) -> DeterminismLevel { match self { @@ -336,8 +323,7 @@ impl HostExports { .map_err(|e| HostExportError::Deterministic(anyhow!(e)))?; let poi_section = stopwatch.start_section("host_export_store_set__proof_of_indexing"); - write_poi_event( - proof_of_indexing, + proof_of_indexing.write_event( &ProofOfIndexingEvent::SetEntity { entity_type: &key.entity_type.typename(), id: &key.entity_id.to_string(), @@ -369,8 +355,7 @@ impl HostExports { entity_id: String, gas: &GasCounter, ) -> Result<(), HostExportError> { - write_poi_event( - proof_of_indexing, + proof_of_indexing.write_event( &ProofOfIndexingEvent::RemoveEntity { entity_type: &entity_type, id: &entity_id, From 0a85e660041022bcb16d4cc7d300e980a50ff8d9 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Tue, 4 Mar 2025 12:42:59 +0100 Subject: [PATCH 052/150] chain/ethereum: Suppress warnings about API version specific types --- chain/ethereum/src/runtime/abi.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/chain/ethereum/src/runtime/abi.rs b/chain/ethereum/src/runtime/abi.rs index d88bf2b22d7..ebda24044a5 100644 --- a/chain/ethereum/src/runtime/abi.rs +++ b/chain/ethereum/src/runtime/abi.rs @@ -121,6 +121,7 @@ impl AscIndexId for AscLogArray { #[repr(C)] #[derive(AscType)] +#[allow(non_camel_case_types)] pub struct AscUnresolvedContractCall_0_0_4 { pub contract_name: AscPtr, pub contract_address: AscPtr, @@ -201,6 +202,7 @@ impl AscIndexId for AscEthereumBlock { #[repr(C)] #[derive(AscType)] +#[allow(non_camel_case_types)] pub(crate) struct AscEthereumBlock_0_0_6 { pub hash: AscPtr, pub parent_hash: AscPtr, @@ -225,6 +227,7 @@ impl AscIndexId for AscEthereumBlock_0_0_6 { #[repr(C)] #[derive(AscType)] +#[allow(non_camel_case_types)] pub(crate) struct AscEthereumTransaction_0_0_1 { pub hash: AscPtr, pub index: AscPtr, @@ -241,6 +244,7 @@ impl AscIndexId for AscEthereumTransaction_0_0_1 { #[repr(C)] #[derive(AscType)] +#[allow(non_camel_case_types)] pub(crate) struct AscEthereumTransaction_0_0_2 { pub hash: AscPtr, pub index: AscPtr, @@ -258,6 +262,7 @@ impl AscIndexId for AscEthereumTransaction_0_0_2 { #[repr(C)] #[derive(AscType)] +#[allow(non_camel_case_types)] pub(crate) struct AscEthereumTransaction_0_0_6 { pub hash: AscPtr, pub index: AscPtr, @@ -346,6 +351,7 @@ impl AscIndexId for AscEthereumTransactionReceipt { /// `receipt` field. #[repr(C)] #[derive(AscType)] +#[allow(non_camel_case_types)] pub(crate) struct AscEthereumEvent_0_0_7 where T: AscType, @@ -392,6 +398,7 @@ impl AscIndexId for AscEthereumCall { #[repr(C)] #[derive(AscType)] +#[allow(non_camel_case_types)] pub(crate) struct AscEthereumCall_0_0_3 where T: AscType, From 7e96a20d8e32cda2f6f23aae23a71c2e3da3c7a2 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Tue, 4 Mar 2025 12:30:35 +0100 Subject: [PATCH 053/150] chain/ethereum: Eliminate one clone of the headers in ABI conversion --- chain/ethereum/src/runtime/abi.rs | 82 +++++++++++----------- chain/ethereum/src/trigger.rs | 113 +++++++++++++++++++----------- 2 files changed, 114 insertions(+), 81 deletions(-) diff --git a/chain/ethereum/src/runtime/abi.rs b/chain/ethereum/src/runtime/abi.rs index ebda24044a5..e749ffcc7c9 100644 --- a/chain/ethereum/src/runtime/abi.rs +++ b/chain/ethereum/src/runtime/abi.rs @@ -420,68 +420,68 @@ where const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::EthereumCall; } -impl ToAscObj for EthereumBlockData { +impl<'a> ToAscObj for EthereumBlockData<'a> { fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, ) -> Result { Ok(AscEthereumBlock { - hash: asc_new(heap, &self.hash, gas)?, - parent_hash: asc_new(heap, &self.parent_hash, gas)?, - uncles_hash: asc_new(heap, &self.uncles_hash, gas)?, - author: asc_new(heap, &self.author, gas)?, - state_root: asc_new(heap, &self.state_root, gas)?, - transactions_root: asc_new(heap, &self.transactions_root, gas)?, - receipts_root: asc_new(heap, &self.receipts_root, gas)?, - number: asc_new(heap, &BigInt::from(self.number), gas)?, - gas_used: asc_new(heap, &BigInt::from_unsigned_u256(&self.gas_used), gas)?, - gas_limit: asc_new(heap, &BigInt::from_unsigned_u256(&self.gas_limit), gas)?, - timestamp: asc_new(heap, &BigInt::from_unsigned_u256(&self.timestamp), gas)?, - difficulty: asc_new(heap, &BigInt::from_unsigned_u256(&self.difficulty), gas)?, + hash: asc_new(heap, self.hash(), gas)?, + parent_hash: asc_new(heap, self.parent_hash(), gas)?, + uncles_hash: asc_new(heap, self.uncles_hash(), gas)?, + author: asc_new(heap, self.author(), gas)?, + state_root: asc_new(heap, self.state_root(), gas)?, + transactions_root: asc_new(heap, self.transactions_root(), gas)?, + receipts_root: asc_new(heap, self.receipts_root(), gas)?, + number: asc_new(heap, &BigInt::from(self.number()), gas)?, + gas_used: asc_new(heap, &BigInt::from_unsigned_u256(self.gas_used()), gas)?, + gas_limit: asc_new(heap, &BigInt::from_unsigned_u256(self.gas_limit()), gas)?, + timestamp: asc_new(heap, &BigInt::from_unsigned_u256(self.timestamp()), gas)?, + difficulty: asc_new(heap, &BigInt::from_unsigned_u256(self.difficulty()), gas)?, total_difficulty: asc_new( heap, - &BigInt::from_unsigned_u256(&self.total_difficulty), + &BigInt::from_unsigned_u256(self.total_difficulty()), gas, )?, size: self - .size + .size() .map(|size| asc_new(heap, &BigInt::from_unsigned_u256(&size), gas)) .unwrap_or(Ok(AscPtr::null()))?, }) } } -impl ToAscObj for EthereumBlockData { +impl<'a> ToAscObj for EthereumBlockData<'a> { fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, ) -> Result { Ok(AscEthereumBlock_0_0_6 { - hash: asc_new(heap, &self.hash, gas)?, - parent_hash: asc_new(heap, &self.parent_hash, gas)?, - uncles_hash: asc_new(heap, &self.uncles_hash, gas)?, - author: asc_new(heap, &self.author, gas)?, - state_root: asc_new(heap, &self.state_root, gas)?, - transactions_root: asc_new(heap, &self.transactions_root, gas)?, - receipts_root: asc_new(heap, &self.receipts_root, gas)?, - number: asc_new(heap, &BigInt::from(self.number), gas)?, - gas_used: asc_new(heap, &BigInt::from_unsigned_u256(&self.gas_used), gas)?, - gas_limit: asc_new(heap, &BigInt::from_unsigned_u256(&self.gas_limit), gas)?, - timestamp: asc_new(heap, &BigInt::from_unsigned_u256(&self.timestamp), gas)?, - difficulty: asc_new(heap, &BigInt::from_unsigned_u256(&self.difficulty), gas)?, + hash: asc_new(heap, self.hash(), gas)?, + parent_hash: asc_new(heap, self.parent_hash(), gas)?, + uncles_hash: asc_new(heap, self.uncles_hash(), gas)?, + author: asc_new(heap, self.author(), gas)?, + state_root: asc_new(heap, self.state_root(), gas)?, + transactions_root: asc_new(heap, self.transactions_root(), gas)?, + receipts_root: asc_new(heap, self.receipts_root(), gas)?, + number: asc_new(heap, &BigInt::from(self.number()), gas)?, + gas_used: asc_new(heap, &BigInt::from_unsigned_u256(self.gas_used()), gas)?, + gas_limit: asc_new(heap, &BigInt::from_unsigned_u256(self.gas_limit()), gas)?, + timestamp: asc_new(heap, &BigInt::from_unsigned_u256(self.timestamp()), gas)?, + difficulty: asc_new(heap, &BigInt::from_unsigned_u256(self.difficulty()), gas)?, total_difficulty: asc_new( heap, - &BigInt::from_unsigned_u256(&self.total_difficulty), + &BigInt::from_unsigned_u256(self.total_difficulty()), gas, )?, size: self - .size + .size() .map(|size| asc_new(heap, &BigInt::from_unsigned_u256(&size), gas)) .unwrap_or(Ok(AscPtr::null()))?, base_fee_per_block: self - .base_fee_per_gas + .base_fee_per_gas() .map(|base_fee| asc_new(heap, &BigInt::from_unsigned_u256(&base_fee), gas)) .unwrap_or(Ok(AscPtr::null()))?, }) @@ -554,12 +554,12 @@ impl ToAscObj for EthereumTransactionData { } } -impl ToAscObj> for EthereumEventData +impl<'a, T, B> ToAscObj> for EthereumEventData<'a> where T: AscType + AscIndexId, B: AscType + AscIndexId, EthereumTransactionData: ToAscObj, - EthereumBlockData: ToAscObj, + EthereumBlockData<'a>: ToAscObj, { fn to_asc_obj( &self, @@ -586,13 +586,13 @@ where } } -impl ToAscObj> - for (EthereumEventData, Option<&TransactionReceipt>) +impl<'a, T, B> ToAscObj> + for (EthereumEventData<'a>, Option<&TransactionReceipt>) where T: AscType + AscIndexId, B: AscType + AscIndexId, EthereumTransactionData: ToAscObj, - EthereumBlockData: ToAscObj, + EthereumBlockData<'a>: ToAscObj, { fn to_asc_obj( &self, @@ -718,7 +718,7 @@ impl ToAscObj for &TransactionReceipt { } } -impl ToAscObj for EthereumCallData { +impl<'a> ToAscObj for EthereumCallData<'a> { fn to_asc_obj( &self, heap: &mut H, @@ -734,8 +734,8 @@ impl ToAscObj for EthereumCallData { } } -impl ToAscObj> - for EthereumCallData +impl<'a> ToAscObj> + for EthereumCallData<'a> { fn to_asc_obj( &self, @@ -756,8 +756,8 @@ impl ToAscObj> - for EthereumCallData +impl<'a> ToAscObj> + for EthereumCallData<'a> { fn to_asc_obj( &self, diff --git a/chain/ethereum/src/trigger.rs b/chain/ethereum/src/trigger.rs index a5d83690b4b..e7c53038cd4 100644 --- a/chain/ethereum/src/trigger.rs +++ b/chain/ethereum/src/trigger.rs @@ -420,44 +420,77 @@ impl TriggerData for EthereumTrigger { } /// Ethereum block data. -#[derive(Clone, Debug, Default)] -pub struct EthereumBlockData { - pub hash: H256, - pub parent_hash: H256, - pub uncles_hash: H256, - pub author: H160, - pub state_root: H256, - pub transactions_root: H256, - pub receipts_root: H256, - pub number: U64, - pub gas_used: U256, - pub gas_limit: U256, - pub timestamp: U256, - pub difficulty: U256, - pub total_difficulty: U256, - pub size: Option, - pub base_fee_per_gas: Option, +#[derive(Clone, Debug)] +pub struct EthereumBlockData<'a> { + block: &'a Block, } -impl<'a, T> From<&'a Block> for EthereumBlockData { - fn from(block: &'a Block) -> EthereumBlockData { - EthereumBlockData { - hash: block.hash.unwrap(), - parent_hash: block.parent_hash, - uncles_hash: block.uncles_hash, - author: block.author, - state_root: block.state_root, - transactions_root: block.transactions_root, - receipts_root: block.receipts_root, - number: block.number.unwrap(), - gas_used: block.gas_used, - gas_limit: block.gas_limit, - timestamp: block.timestamp, - difficulty: block.difficulty, - total_difficulty: block.total_difficulty.unwrap_or_default(), - size: block.size, - base_fee_per_gas: block.base_fee_per_gas, - } +impl<'a> From<&'a Block> for EthereumBlockData<'a> { + fn from(block: &'a Block) -> EthereumBlockData<'a> { + EthereumBlockData { block } + } +} + +impl<'a> EthereumBlockData<'a> { + pub fn hash(&self) -> &H256 { + self.block.hash.as_ref().unwrap() + } + + pub fn parent_hash(&self) -> &H256 { + &self.block.parent_hash + } + + pub fn uncles_hash(&self) -> &H256 { + &self.block.uncles_hash + } + + pub fn author(&self) -> &H160 { + &self.block.author + } + + pub fn state_root(&self) -> &H256 { + &self.block.state_root + } + + pub fn transactions_root(&self) -> &H256 { + &self.block.transactions_root + } + + pub fn receipts_root(&self) -> &H256 { + &self.block.receipts_root + } + + pub fn number(&self) -> U64 { + self.block.number.unwrap() + } + + pub fn gas_used(&self) -> &U256 { + &self.block.gas_used + } + + pub fn gas_limit(&self) -> &U256 { + &self.block.gas_limit + } + + pub fn timestamp(&self) -> &U256 { + &self.block.timestamp + } + + pub fn difficulty(&self) -> &U256 { + &self.block.difficulty + } + + pub fn total_difficulty(&self) -> &U256 { + static DEFAULT: U256 = U256::zero(); + self.block.total_difficulty.as_ref().unwrap_or(&DEFAULT) + } + + pub fn size(&self) -> &Option { + &self.block.size + } + + pub fn base_fee_per_gas(&self) -> &Option { + &self.block.base_fee_per_gas } } @@ -496,22 +529,22 @@ impl From<&'_ Transaction> for EthereumTransactionData { /// An Ethereum event logged from a specific contract address and block. #[derive(Debug, Clone)] -pub struct EthereumEventData { +pub struct EthereumEventData<'a> { pub address: Address, pub log_index: U256, pub transaction_log_index: U256, pub log_type: Option, - pub block: EthereumBlockData, + pub block: EthereumBlockData<'a>, pub transaction: EthereumTransactionData, pub params: Vec, } /// An Ethereum call executed within a transaction within a block to a contract address. #[derive(Debug, Clone)] -pub struct EthereumCallData { +pub struct EthereumCallData<'a> { pub from: Address, pub to: Address, - pub block: EthereumBlockData, + pub block: EthereumBlockData<'a>, pub transaction: EthereumTransactionData, pub inputs: Vec, pub outputs: Vec, From c771226a9a17b98c3da5abbcbf791c62b48e1dc1 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Tue, 4 Mar 2025 13:03:14 +0100 Subject: [PATCH 054/150] chain/ethereum: Avoid one clone of txn data in ABI conversion --- chain/ethereum/src/runtime/abi.rs | 58 ++++++++++----------- chain/ethereum/src/trigger.rs | 85 ++++++++++++++++++++----------- 2 files changed, 83 insertions(+), 60 deletions(-) diff --git a/chain/ethereum/src/runtime/abi.rs b/chain/ethereum/src/runtime/abi.rs index e749ffcc7c9..856171b22cc 100644 --- a/chain/ethereum/src/runtime/abi.rs +++ b/chain/ethereum/src/runtime/abi.rs @@ -488,68 +488,68 @@ impl<'a> ToAscObj for EthereumBlockData<'a> { } } -impl ToAscObj for EthereumTransactionData { +impl<'a> ToAscObj for EthereumTransactionData<'a> { fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, ) -> Result { Ok(AscEthereumTransaction_0_0_1 { - hash: asc_new(heap, &self.hash, gas)?, - index: asc_new(heap, &BigInt::from_unsigned_u128(self.index), gas)?, - from: asc_new(heap, &self.from, gas)?, + hash: asc_new(heap, self.hash(), gas)?, + index: asc_new(heap, &BigInt::from_unsigned_u128(self.index()), gas)?, + from: asc_new(heap, self.from(), gas)?, to: self - .to + .to() .map(|to| asc_new(heap, &to, gas)) .unwrap_or(Ok(AscPtr::null()))?, - value: asc_new(heap, &BigInt::from_unsigned_u256(&self.value), gas)?, - gas_limit: asc_new(heap, &BigInt::from_unsigned_u256(&self.gas_limit), gas)?, - gas_price: asc_new(heap, &BigInt::from_unsigned_u256(&self.gas_price), gas)?, + value: asc_new(heap, &BigInt::from_unsigned_u256(self.value()), gas)?, + gas_limit: asc_new(heap, &BigInt::from_unsigned_u256(self.gas_limit()), gas)?, + gas_price: asc_new(heap, &BigInt::from_unsigned_u256(self.gas_price()), gas)?, }) } } -impl ToAscObj for EthereumTransactionData { +impl<'a> ToAscObj for EthereumTransactionData<'a> { fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, ) -> Result { Ok(AscEthereumTransaction_0_0_2 { - hash: asc_new(heap, &self.hash, gas)?, - index: asc_new(heap, &BigInt::from_unsigned_u128(self.index), gas)?, - from: asc_new(heap, &self.from, gas)?, + hash: asc_new(heap, self.hash(), gas)?, + index: asc_new(heap, &BigInt::from_unsigned_u128(self.index()), gas)?, + from: asc_new(heap, self.from(), gas)?, to: self - .to + .to() .map(|to| asc_new(heap, &to, gas)) .unwrap_or(Ok(AscPtr::null()))?, - value: asc_new(heap, &BigInt::from_unsigned_u256(&self.value), gas)?, - gas_limit: asc_new(heap, &BigInt::from_unsigned_u256(&self.gas_limit), gas)?, - gas_price: asc_new(heap, &BigInt::from_unsigned_u256(&self.gas_price), gas)?, - input: asc_new(heap, &*self.input, gas)?, + value: asc_new(heap, &BigInt::from_unsigned_u256(self.value()), gas)?, + gas_limit: asc_new(heap, &BigInt::from_unsigned_u256(self.gas_limit()), gas)?, + gas_price: asc_new(heap, &BigInt::from_unsigned_u256(self.gas_price()), gas)?, + input: asc_new(heap, self.input(), gas)?, }) } } -impl ToAscObj for EthereumTransactionData { +impl<'a> ToAscObj for EthereumTransactionData<'a> { fn to_asc_obj( &self, heap: &mut H, gas: &GasCounter, ) -> Result { Ok(AscEthereumTransaction_0_0_6 { - hash: asc_new(heap, &self.hash, gas)?, - index: asc_new(heap, &BigInt::from_unsigned_u128(self.index), gas)?, - from: asc_new(heap, &self.from, gas)?, + hash: asc_new(heap, self.hash(), gas)?, + index: asc_new(heap, &BigInt::from_unsigned_u128(self.index()), gas)?, + from: asc_new(heap, self.from(), gas)?, to: self - .to + .to() .map(|to| asc_new(heap, &to, gas)) .unwrap_or(Ok(AscPtr::null()))?, - value: asc_new(heap, &BigInt::from_unsigned_u256(&self.value), gas)?, - gas_limit: asc_new(heap, &BigInt::from_unsigned_u256(&self.gas_limit), gas)?, - gas_price: asc_new(heap, &BigInt::from_unsigned_u256(&self.gas_price), gas)?, - input: asc_new(heap, &*self.input, gas)?, - nonce: asc_new(heap, &BigInt::from_unsigned_u256(&self.nonce), gas)?, + value: asc_new(heap, &BigInt::from_unsigned_u256(self.value()), gas)?, + gas_limit: asc_new(heap, &BigInt::from_unsigned_u256(self.gas_limit()), gas)?, + gas_price: asc_new(heap, &BigInt::from_unsigned_u256(self.gas_price()), gas)?, + input: asc_new(heap, self.input(), gas)?, + nonce: asc_new(heap, &BigInt::from_unsigned_u256(self.nonce()), gas)?, }) } } @@ -558,7 +558,7 @@ impl<'a, T, B> ToAscObj> for EthereumEventData<'a> where T: AscType + AscIndexId, B: AscType + AscIndexId, - EthereumTransactionData: ToAscObj, + EthereumTransactionData<'a>: ToAscObj, EthereumBlockData<'a>: ToAscObj, { fn to_asc_obj( @@ -591,7 +591,7 @@ impl<'a, T, B> ToAscObj> where T: AscType + AscIndexId, B: AscType + AscIndexId, - EthereumTransactionData: ToAscObj, + EthereumTransactionData<'a>: ToAscObj, EthereumBlockData<'a>: ToAscObj, { fn to_asc_obj( diff --git a/chain/ethereum/src/trigger.rs b/chain/ethereum/src/trigger.rs index e7c53038cd4..125cd65f1de 100644 --- a/chain/ethereum/src/trigger.rs +++ b/chain/ethereum/src/trigger.rs @@ -10,7 +10,6 @@ use graph::prelude::ethabi::ethereum_types::U128; use graph::prelude::ethabi::ethereum_types::U256; use graph::prelude::ethabi::ethereum_types::U64; use graph::prelude::ethabi::Address; -use graph::prelude::ethabi::Bytes; use graph::prelude::ethabi::LogParam; use graph::prelude::web3::types::Block; use graph::prelude::web3::types::Log; @@ -42,6 +41,8 @@ use crate::runtime::abi::AscEthereumTransaction_0_0_6; // ETHDEP: This should be defined in only one place. type LightEthereumBlock = Block; +static U256_DEFAULT: U256 = U256::zero(); + pub enum MappingTrigger { Log { block: Arc, @@ -147,7 +148,7 @@ impl ToAscPtr for MappingTrigger { let api_version = heap.api_version(); let ethereum_event_data = EthereumEventData { block: EthereumBlockData::from(block.as_ref()), - transaction: EthereumTransactionData::from(transaction.deref()), + transaction: EthereumTransactionData::new(transaction.deref()), address: log.address, log_index: log.log_index.unwrap_or(U256::zero()), transaction_log_index: log.log_index.unwrap_or(U256::zero()), @@ -198,7 +199,7 @@ impl ToAscPtr for MappingTrigger { to: call.to, from: call.from, block: EthereumBlockData::from(block.as_ref()), - transaction: EthereumTransactionData::from(transaction.deref()), + transaction: EthereumTransactionData::new(transaction.deref()), inputs, outputs, }; @@ -481,8 +482,10 @@ impl<'a> EthereumBlockData<'a> { } pub fn total_difficulty(&self) -> &U256 { - static DEFAULT: U256 = U256::zero(); - self.block.total_difficulty.as_ref().unwrap_or(&DEFAULT) + self.block + .total_difficulty + .as_ref() + .unwrap_or(&U256_DEFAULT) } pub fn size(&self) -> &Option { @@ -496,34 +499,54 @@ impl<'a> EthereumBlockData<'a> { /// Ethereum transaction data. #[derive(Clone, Debug)] -pub struct EthereumTransactionData { - pub hash: H256, - pub index: U128, - pub from: H160, - pub to: Option, - pub value: U256, - pub gas_limit: U256, - pub gas_price: U256, - pub input: Bytes, - pub nonce: U256, +pub struct EthereumTransactionData<'a> { + tx: &'a Transaction, } -impl From<&'_ Transaction> for EthereumTransactionData { - fn from(tx: &Transaction) -> EthereumTransactionData { +impl<'a> EthereumTransactionData<'a> { + // We don't implement `From` because it causes confusion with the `from` + // accessor method + fn new(tx: &'a Transaction) -> EthereumTransactionData<'a> { + EthereumTransactionData { tx } + } + + pub fn hash(&self) -> &H256 { + &self.tx.hash + } + + pub fn index(&self) -> U128 { + self.tx.transaction_index.unwrap().as_u64().into() + } + + pub fn from(&self) -> &H160 { // unwrap: this is always `Some` for txns that have been mined // (see https://github.com/tomusdrw/rust-web3/pull/407) - let from = tx.from.unwrap(); - EthereumTransactionData { - hash: tx.hash, - index: tx.transaction_index.unwrap().as_u64().into(), - from, - to: tx.to, - value: tx.value, - gas_limit: tx.gas, - gas_price: tx.gas_price.unwrap_or(U256::zero()), // EIP-1559 made this optional. - input: tx.input.0.clone(), - nonce: tx.nonce, - } + self.tx.from.as_ref().unwrap() + } + + pub fn to(&self) -> &Option { + &self.tx.to + } + + pub fn value(&self) -> &U256 { + &self.tx.value + } + + pub fn gas_limit(&self) -> &U256 { + &self.tx.gas + } + + pub fn gas_price(&self) -> &U256 { + // EIP-1559 made this optional. + self.tx.gas_price.as_ref().unwrap_or(&U256_DEFAULT) + } + + pub fn input(&self) -> &[u8] { + &self.tx.input.0 + } + + pub fn nonce(&self) -> &U256 { + &self.tx.nonce } } @@ -535,7 +558,7 @@ pub struct EthereumEventData<'a> { pub transaction_log_index: U256, pub log_type: Option, pub block: EthereumBlockData<'a>, - pub transaction: EthereumTransactionData, + pub transaction: EthereumTransactionData<'a>, pub params: Vec, } @@ -545,7 +568,7 @@ pub struct EthereumCallData<'a> { pub from: Address, pub to: Address, pub block: EthereumBlockData<'a>, - pub transaction: EthereumTransactionData, + pub transaction: EthereumTransactionData<'a>, pub inputs: Vec, pub outputs: Vec, } From 6bf741d242dc5454decf00404b92ca601d41b3b5 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Tue, 4 Mar 2025 13:25:57 +0100 Subject: [PATCH 055/150] chain/ethereum: Avoid a clone for EthereumEventData in ABI conversion --- chain/ethereum/src/runtime/abi.rs | 24 +++++++++---- chain/ethereum/src/trigger.rs | 57 +++++++++++++++++++++++-------- 2 files changed, 61 insertions(+), 20 deletions(-) diff --git a/chain/ethereum/src/runtime/abi.rs b/chain/ethereum/src/runtime/abi.rs index 856171b22cc..ca75dee4242 100644 --- a/chain/ethereum/src/runtime/abi.rs +++ b/chain/ethereum/src/runtime/abi.rs @@ -49,6 +49,18 @@ impl ToAscObj for Vec { } } +impl ToAscObj for &[ethabi::LogParam] { + fn to_asc_obj( + &self, + heap: &mut H, + gas: &GasCounter, + ) -> Result { + let content: Result, _> = self.iter().map(|x| asc_new(heap, x, gas)).collect(); + let content = content?; + Ok(AscLogParamArray(Array::new(&content, heap, gas)?)) + } +} + impl AscIndexId for AscLogParamArray { const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::ArrayEventParam; } @@ -567,17 +579,17 @@ where gas: &GasCounter, ) -> Result, HostExportError> { Ok(AscEthereumEvent { - address: asc_new(heap, &self.address, gas)?, - log_index: asc_new(heap, &BigInt::from_unsigned_u256(&self.log_index), gas)?, + address: asc_new(heap, self.address(), gas)?, + log_index: asc_new(heap, &BigInt::from_unsigned_u256(self.log_index()), gas)?, transaction_log_index: asc_new( heap, - &BigInt::from_unsigned_u256(&self.transaction_log_index), + &BigInt::from_unsigned_u256(self.transaction_log_index()), gas, )?, log_type: self - .log_type - .clone() - .map(|log_type| asc_new(heap, &log_type, gas)) + .log_type() + .as_ref() + .map(|log_type| asc_new(heap, log_type, gas)) .unwrap_or(Ok(AscPtr::null()))?, block: asc_new::(heap, &self.block, gas)?, transaction: asc_new::(heap, &self.transaction, gas)?, diff --git a/chain/ethereum/src/trigger.rs b/chain/ethereum/src/trigger.rs index 125cd65f1de..063ad490e6d 100644 --- a/chain/ethereum/src/trigger.rs +++ b/chain/ethereum/src/trigger.rs @@ -146,15 +146,12 @@ impl ToAscPtr for MappingTrigger { calls: _, } => { let api_version = heap.api_version(); - let ethereum_event_data = EthereumEventData { - block: EthereumBlockData::from(block.as_ref()), - transaction: EthereumTransactionData::new(transaction.deref()), - address: log.address, - log_index: log.log_index.unwrap_or(U256::zero()), - transaction_log_index: log.log_index.unwrap_or(U256::zero()), - log_type: log.log_type.clone(), - params, - }; + let ethereum_event_data = EthereumEventData::new( + block.as_ref(), + transaction.as_ref(), + log.as_ref(), + ¶ms, + ); if api_version >= API_VERSION_0_0_7 { asc_new::< AscEthereumEvent_0_0_7< @@ -553,13 +550,45 @@ impl<'a> EthereumTransactionData<'a> { /// An Ethereum event logged from a specific contract address and block. #[derive(Debug, Clone)] pub struct EthereumEventData<'a> { - pub address: Address, - pub log_index: U256, - pub transaction_log_index: U256, - pub log_type: Option, pub block: EthereumBlockData<'a>, pub transaction: EthereumTransactionData<'a>, - pub params: Vec, + pub params: &'a [LogParam], + log: &'a Log, +} + +impl<'a> EthereumEventData<'a> { + pub fn new( + block: &'a Block, + tx: &'a Transaction, + log: &'a Log, + params: &'a [LogParam], + ) -> Self { + EthereumEventData { + block: EthereumBlockData::from(block), + transaction: EthereumTransactionData::new(tx), + log, + params, + } + } + + pub fn address(&self) -> &Address { + &self.log.address + } + + pub fn log_index(&self) -> &U256 { + self.log.log_index.as_ref().unwrap_or(&U256_DEFAULT) + } + + pub fn transaction_log_index(&self) -> &U256 { + self.log + .transaction_log_index + .as_ref() + .unwrap_or(&U256_DEFAULT) + } + + pub fn log_type(&self) -> &Option { + &self.log.log_type + } } /// An Ethereum call executed within a transaction within a block to a contract address. From 737014af272e91e79ecd8b38d3c62af6929d6a56 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Tue, 4 Mar 2025 15:26:12 +0100 Subject: [PATCH 056/150] chain/ethereum: Avoid a clone for EthereumCallData in ABI conversion --- chain/ethereum/src/runtime/abi.rs | 22 ++++------------ chain/ethereum/src/trigger.rs | 43 +++++++++++++++++++++---------- 2 files changed, 35 insertions(+), 30 deletions(-) diff --git a/chain/ethereum/src/runtime/abi.rs b/chain/ethereum/src/runtime/abi.rs index ca75dee4242..1a4af0663cb 100644 --- a/chain/ethereum/src/runtime/abi.rs +++ b/chain/ethereum/src/runtime/abi.rs @@ -37,18 +37,6 @@ impl AscType for AscLogParamArray { } } -impl ToAscObj for Vec { - fn to_asc_obj( - &self, - heap: &mut H, - gas: &GasCounter, - ) -> Result { - let content: Result, _> = self.iter().map(|x| asc_new(heap, x, gas)).collect(); - let content = content?; - Ok(AscLogParamArray(Array::new(&content, heap, gas)?)) - } -} - impl ToAscObj for &[ethabi::LogParam] { fn to_asc_obj( &self, @@ -737,7 +725,7 @@ impl<'a> ToAscObj for EthereumCallData<'a> { gas: &GasCounter, ) -> Result { Ok(AscEthereumCall { - address: asc_new(heap, &self.to, gas)?, + address: asc_new(heap, self.to(), gas)?, block: asc_new(heap, &self.block, gas)?, transaction: asc_new(heap, &self.transaction, gas)?, inputs: asc_new(heap, &self.inputs, gas)?, @@ -758,8 +746,8 @@ impl<'a> ToAscObj { Ok(AscEthereumCall_0_0_3 { - to: asc_new(heap, &self.to, gas)?, - from: asc_new(heap, &self.from, gas)?, + to: asc_new(heap, self.to(), gas)?, + from: asc_new(heap, self.from(), gas)?, block: asc_new(heap, &self.block, gas)?, transaction: asc_new(heap, &self.transaction, gas)?, inputs: asc_new(heap, &self.inputs, gas)?, @@ -780,8 +768,8 @@ impl<'a> ToAscObj { Ok(AscEthereumCall_0_0_3 { - to: asc_new(heap, &self.to, gas)?, - from: asc_new(heap, &self.from, gas)?, + to: asc_new(heap, self.to(), gas)?, + from: asc_new(heap, self.from(), gas)?, block: asc_new(heap, &self.block, gas)?, transaction: asc_new(heap, &self.transaction, gas)?, inputs: asc_new(heap, &self.inputs, gas)?, diff --git a/chain/ethereum/src/trigger.rs b/chain/ethereum/src/trigger.rs index 063ad490e6d..f9f4ced2978 100644 --- a/chain/ethereum/src/trigger.rs +++ b/chain/ethereum/src/trigger.rs @@ -25,7 +25,6 @@ use graph::runtime::AscPtr; use graph::runtime::HostExportError; use graph::semver::Version; use graph_runtime_wasm::module::ToAscPtr; -use std::ops::Deref; use std::{cmp::Ordering, sync::Arc}; use crate::runtime::abi::AscEthereumBlock; @@ -192,14 +191,7 @@ impl ToAscPtr for MappingTrigger { inputs, outputs, } => { - let call = EthereumCallData { - to: call.to, - from: call.from, - block: EthereumBlockData::from(block.as_ref()), - transaction: EthereumTransactionData::new(transaction.deref()), - inputs, - outputs, - }; + let call = EthereumCallData::new(&block, &transaction, &call, &inputs, &outputs); if heap.api_version() >= Version::new(0, 0, 6) { asc_new::< AscEthereumCall_0_0_3, @@ -594,10 +586,35 @@ impl<'a> EthereumEventData<'a> { /// An Ethereum call executed within a transaction within a block to a contract address. #[derive(Debug, Clone)] pub struct EthereumCallData<'a> { - pub from: Address, - pub to: Address, pub block: EthereumBlockData<'a>, pub transaction: EthereumTransactionData<'a>, - pub inputs: Vec, - pub outputs: Vec, + pub inputs: &'a [LogParam], + pub outputs: &'a [LogParam], + call: &'a EthereumCall, +} + +impl<'a> EthereumCallData<'a> { + fn new( + block: &'a Block, + transaction: &'a Transaction, + call: &'a EthereumCall, + inputs: &'a [LogParam], + outputs: &'a [LogParam], + ) -> EthereumCallData<'a> { + EthereumCallData { + block: EthereumBlockData::from(block), + transaction: EthereumTransactionData::new(transaction), + inputs, + outputs, + call, + } + } + + pub fn from(&self) -> &Address { + &self.call.from + } + + pub fn to(&self) -> &Address { + &self.call.to + } } From 1342f346b76f847edac360f1bfd519962cd0b8de Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Mon, 17 Feb 2025 15:52:26 -0800 Subject: [PATCH 057/150] all: Update diesel to latest version --- Cargo.lock | 13 ++++++------- Cargo.toml | 14 +++----------- 2 files changed, 9 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c3b80fe7443..3226e8a27c6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1196,9 +1196,9 @@ dependencies = [ [[package]] name = "diesel" -version = "2.2.4" +version = "2.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "158fe8e2e68695bd615d7e4f3227c0727b151330d3e253b525086c348d055d5e" +checksum = "04001f23ba8843dc315804fa324000376084dfb1c30794ff68dd279e6e5696d5" dependencies = [ "bigdecimal 0.3.1", "bitflags 2.6.0", @@ -1212,7 +1212,6 @@ dependencies = [ "pq-sys", "r2d2", "serde_json", - "uuid", ] [[package]] @@ -1229,18 +1228,18 @@ dependencies = [ [[package]] name = "diesel-dynamic-schema" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71eda9b13a55533594231b0763c36bc21058ccb82ed17eaeb41b3cbb897c1bb1" +checksum = "061bbe2d02508364c50153226524b7fc224f56031a5e927b0bc5f1f2b48de6a6" dependencies = [ "diesel", ] [[package]] name = "diesel_derives" -version = "2.2.1" +version = "2.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59de76a222c2b8059f789cbe07afbfd8deb8c31dd0bc2a21f85e256c1def8259" +checksum = "e7f2c3de51e2ba6bf2a648285696137aaf0f5f487bcbea93972fe8a364e131a4" dependencies = [ "diesel_table_macro_syntax", "dsl_auto_type", diff --git a/Cargo.toml b/Cargo.toml index 0cf70f16736..b01bc4d4fd1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -47,18 +47,10 @@ chrono = "0.4.38" bs58 = "0.5.1" clap = { version = "4.5.4", features = ["derive", "env"] } derivative = "2.2.0" -diesel = { version = "2.2.4", features = [ - "postgres", - "serde_json", - "numeric", - "r2d2", - "chrono", - "uuid", - "i-implement-a-third-party-backend-and-opt-into-breaking-changes", -] } +diesel = { version = "2.2.7", features = ["postgres", "serde_json", "numeric", "r2d2", "chrono", "i-implement-a-third-party-backend-and-opt-into-breaking-changes"] } diesel-derive-enum = { version = "2.1.0", features = ["postgres"] } -diesel-dynamic-schema = { version = "0.2.1", features = ["postgres"] } -diesel_derives = "2.1.4" +diesel-dynamic-schema = { version = "0.2.3", features = ["postgres"] } +diesel_derives = "2.2.3" diesel_migrations = "2.1.0" graph = { path = "./graph" } graph-core = { path = "./core" } From 4a2f4c4fc438dabb10a36004cdb374b6e7bba4cd Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Mon, 17 Feb 2025 16:15:07 -0800 Subject: [PATCH 058/150] runtime/wasm: Remove use of uuid --- Cargo.lock | 1 - runtime/wasm/Cargo.toml | 1 - runtime/wasm/src/mapping.rs | 7 +++++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3226e8a27c6..50214ad644f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2075,7 +2075,6 @@ dependencies = [ "parity-wasm", "semver", "serde_yaml", - "uuid", "wasm-instrument", "wasmtime", ] diff --git a/runtime/wasm/Cargo.toml b/runtime/wasm/Cargo.toml index c728f0bfb65..dfbd7983b1c 100644 --- a/runtime/wasm/Cargo.toml +++ b/runtime/wasm/Cargo.toml @@ -11,7 +11,6 @@ graph = { path = "../../graph" } bs58 = "0.4.0" graph-runtime-derive = { path = "../derive" } semver = "1.0.23" -uuid = { version = "1.15.1", features = ["v4"] } anyhow = "1.0" never = "0.1" diff --git a/runtime/wasm/src/mapping.rs b/runtime/wasm/src/mapping.rs index 8086051961a..c5678bf2aa2 100644 --- a/runtime/wasm/src/mapping.rs +++ b/runtime/wasm/src/mapping.rs @@ -12,6 +12,7 @@ use graph::runtime::gas::Gas; use parity_wasm::elements::ExportEntry; use std::collections::BTreeMap; use std::panic::AssertUnwindSafe; +use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; use std::{panic, thread}; @@ -28,6 +29,8 @@ pub fn spawn_module( where ::MappingTrigger: ToAscPtr, { + static THREAD_COUNT: AtomicUsize = AtomicUsize::new(0); + let valid_module = Arc::new(ValidModule::new(&logger, raw_module, timeout)?); // Create channel for event handling requests @@ -39,8 +42,8 @@ where // In case of failure, this thread may panic or simply terminate, // dropping the `mapping_request_receiver` which ultimately causes the // subgraph to fail the next time it tries to handle an event. - let conf = - thread::Builder::new().name(format!("mapping-{}-{}", &subgraph_id, uuid::Uuid::new_v4())); + let next_id = THREAD_COUNT.fetch_add(1, Ordering::SeqCst); + let conf = thread::Builder::new().name(format!("mapping-{}-{:0>4}", &subgraph_id, next_id)); conf.spawn(move || { let _runtime_guard = runtime.enter(); From 397a3f419603f14d961c1b9b058e6e5412351db0 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Mon, 17 Feb 2025 16:18:51 -0800 Subject: [PATCH 059/150] chain/ethereum: Remove use of uuid --- Cargo.lock | 1 - chain/ethereum/Cargo.toml | 1 - chain/ethereum/src/network.rs | 29 ++++++++++------------------- 3 files changed, 10 insertions(+), 21 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 50214ad644f..a648e996f8e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1929,7 +1929,6 @@ dependencies = [ "serde", "tiny-keccak 1.5.0", "tonic-build", - "uuid", ] [[package]] diff --git a/chain/ethereum/Cargo.toml b/chain/ethereum/Cargo.toml index 30378ec5586..43d1afb9bd3 100644 --- a/chain/ethereum/Cargo.toml +++ b/chain/ethereum/Cargo.toml @@ -22,7 +22,6 @@ graph-runtime-derive = { path = "../../runtime/derive" } [dev-dependencies] base64 = "0" -uuid = { version = "1.15.1", features = ["v4"] } [build-dependencies] tonic-build = { workspace = true } diff --git a/chain/ethereum/src/network.rs b/chain/ethereum/src/network.rs index 43b5a04b63a..d654db71276 100644 --- a/chain/ethereum/src/network.rs +++ b/chain/ethereum/src/network.rs @@ -325,7 +325,6 @@ mod tests { url::Url, }; use std::sync::Arc; - use uuid::Uuid; use crate::{EthereumAdapter, EthereumAdapterTrait, ProviderEthRpcMetrics, Transport}; @@ -679,18 +678,14 @@ mod tests { #[tokio::test] async fn eth_adapter_selection_multiple_adapters() { let logger = Logger::root(Discard, o!()); - let unavailable_provider = Uuid::new_v4().to_string(); - let error_provider = Uuid::new_v4().to_string(); - let no_error_provider = Uuid::new_v4().to_string(); + let unavailable_provider = "unavailable-provider"; + let error_provider = "error-provider"; + let no_error_provider = "no-error-provider"; let mock_registry = Arc::new(MetricsRegistry::mock()); let metrics = Arc::new(EndpointMetrics::new( logger, - &[ - unavailable_provider.clone(), - error_provider.clone(), - no_error_provider.clone(), - ], + &[unavailable_provider, error_provider, no_error_provider], mock_registry.clone(), )); let logger = graph::log::logger(true); @@ -718,7 +713,7 @@ mod tests { ]; // Set errors - metrics.report_for_test(&ProviderName::from(error_provider.clone()), false); + metrics.report_for_test(&ProviderName::from(error_provider), false); let mut no_retest_adapters = vec![]; let mut always_retest_adapters = vec![]; @@ -796,18 +791,14 @@ mod tests { #[tokio::test] async fn eth_adapter_selection_single_adapter() { let logger = Logger::root(Discard, o!()); - let unavailable_provider = Uuid::new_v4().to_string(); - let error_provider = Uuid::new_v4().to_string(); - let no_error_provider = Uuid::new_v4().to_string(); + let unavailable_provider = "unavailable-provider"; + let error_provider = "error-provider"; + let no_error_provider = "no-error-provider"; let mock_registry = Arc::new(MetricsRegistry::mock()); let metrics = Arc::new(EndpointMetrics::new( logger, - &[ - unavailable_provider, - error_provider.clone(), - no_error_provider.clone(), - ], + &[unavailable_provider, error_provider, no_error_provider], mock_registry.clone(), )); let chain_id: Word = "chain_id".into(); @@ -815,7 +806,7 @@ mod tests { let provider_metrics = Arc::new(ProviderEthRpcMetrics::new(mock_registry.clone())); // Set errors - metrics.report_for_test(&ProviderName::from(error_provider.clone()), false); + metrics.report_for_test(&ProviderName::from(error_provider), false); let mut no_retest_adapters = vec![]; no_retest_adapters.push(EthereumNetworkAdapter { From 0e420a7540c3a612e7f0203be08b9cd2da85aeb2 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Mon, 17 Feb 2025 16:21:33 -0800 Subject: [PATCH 060/150] core: Remove use of uuid --- Cargo.lock | 1 - core/Cargo.toml | 1 - core/src/polling_monitor/ipfs_service.rs | 7 +++++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a648e996f8e..80178776216 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1983,7 +1983,6 @@ dependencies = [ "serde_yaml", "tower 0.4.13 (git+https://github.com/tower-rs/tower.git)", "tower-test", - "uuid", "wiremock", ] diff --git a/core/Cargo.toml b/core/Cargo.toml index 1043fc307c2..4140b37ffbe 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -23,5 +23,4 @@ anyhow = "1.0" [dev-dependencies] tower-test = { git = "https://github.com/tower-rs/tower.git" } -uuid = { version = "1.15.1", features = ["v4"] } wiremock = "0.6.1" diff --git a/core/src/polling_monitor/ipfs_service.rs b/core/src/polling_monitor/ipfs_service.rs index f8c68976216..86a5feef0ab 100644 --- a/core/src/polling_monitor/ipfs_service.rs +++ b/core/src/polling_monitor/ipfs_service.rs @@ -104,7 +104,6 @@ mod test { use graph::log::discard; use graph::tokio; use tower::ServiceExt; - use uuid::Uuid; use wiremock::matchers as m; use wiremock::Mock; use wiremock::MockServer; @@ -114,7 +113,11 @@ mod test { #[tokio::test] async fn cat_file_in_folder() { - let random_bytes = Uuid::new_v4().as_bytes().to_vec(); + let random_bytes = "One morning, when Gregor Samsa woke \ + from troubled dreams, he found himself transformed in his bed \ + into a horrible vermin" + .as_bytes() + .to_vec(); let ipfs_file = ("dir/file.txt", random_bytes.clone()); let add_resp = add_files_to_local_ipfs_node_for_testing([ipfs_file]) From 8f9afd41490aa5d143a76673ac5a5daf45d6ef1b Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Mon, 17 Feb 2025 16:34:03 -0800 Subject: [PATCH 061/150] store: Remove use of uuid --- Cargo.lock | 1 - store/postgres/Cargo.toml | 1 - store/postgres/src/store_events.rs | 6 +++--- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 80178776216..6c1e8bb0018 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2151,7 +2151,6 @@ dependencies = [ "serde", "serde_json", "stable-hash 0.3.4", - "uuid", ] [[package]] diff --git a/store/postgres/Cargo.toml b/store/postgres/Cargo.toml index bbf08701633..9a746646807 100644 --- a/store/postgres/Cargo.toml +++ b/store/postgres/Cargo.toml @@ -26,7 +26,6 @@ postgres-openssl = "0.5.0" rand = "0.8.4" serde = { workspace = true } serde_json = { workspace = true } -uuid = { version = "1.15.1", features = ["v4"] } stable-hash_legacy = { git = "https://github.com/graphprotocol/stable-hash", branch = "old", package = "stable-hash" } anyhow = "1.0.86" git-testament = "0.2.5" diff --git a/store/postgres/src/store_events.rs b/store/postgres/src/store_events.rs index 503b5cae08a..83b8e3b069b 100644 --- a/store/postgres/src/store_events.rs +++ b/store/postgres/src/store_events.rs @@ -6,7 +6,6 @@ use graph::tokio_stream::wrappers::ReceiverStream; use std::sync::{atomic::Ordering, Arc, RwLock}; use std::{collections::HashMap, sync::atomic::AtomicUsize}; use tokio::sync::mpsc::{channel, Sender}; -use uuid::Uuid; use crate::notification_listener::{NotificationListener, SafeChannelName}; use graph::components::store::SubscriptionManager as SubscriptionManagerTrait; @@ -89,7 +88,7 @@ impl StoreEventListener { /// Manage subscriptions to the `StoreEvent` stream. Keep a list of /// currently active subscribers and forward new events to each of them pub struct SubscriptionManager { - subscriptions: Arc>>>>, + subscriptions: Arc>>>>, /// Keep the notification listener alive listener: StoreEventListener, @@ -180,7 +179,8 @@ impl SubscriptionManager { impl SubscriptionManagerTrait for SubscriptionManager { fn subscribe(&self) -> StoreEventStreamBox { - let id = Uuid::new_v4().to_string(); + static SUBSCRIPTION_COUNTER: AtomicUsize = AtomicUsize::new(0); + let id = SUBSCRIPTION_COUNTER.fetch_add(1, Ordering::SeqCst); // Prepare the new subscription by creating a channel and a subscription object let (sender, receiver) = channel(100); From 4b7498ba73d7ca4db41dd3be9ddeab948f787ef8 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Mon, 17 Feb 2025 16:38:45 -0800 Subject: [PATCH 062/150] all: Don't enable uuid feature in async-graphql --- Cargo.lock | 5 ----- Cargo.toml | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6c1e8bb0018..c511d00601f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -211,7 +211,6 @@ dependencies = [ "static_assertions_next", "tempfile", "thiserror 1.0.61", - "uuid", ] [[package]] @@ -5717,10 +5716,6 @@ name = "uuid" version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e0f540e3240398cce6128b64ba83fdbdd86129c16a3aa1a3a252efd66eb3d587" -dependencies = [ - "getrandom 0.3.1", - "serde", -] [[package]] name = "vcpkg" diff --git a/Cargo.toml b/Cargo.toml index b01bc4d4fd1..a8193c3f0ed 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -40,7 +40,7 @@ license = "MIT OR Apache-2.0" [workspace.dependencies] anyhow = "1.0" -async-graphql = { version = "7.0.15", features = ["chrono", "uuid"] } +async-graphql = { version = "7.0.15", features = ["chrono"] } async-graphql-axum = "7.0.15" axum = "0.8.1" chrono = "0.4.38" From 541d169873b4cce9fd63169cbf6670f740228f35 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Mon, 17 Mar 2025 12:26:48 +0100 Subject: [PATCH 063/150] store: Allow copying of failed deployments Generally, it makes no sense to copy failed deployments, but when deployments need to be moved to different shards, it does. --- store/postgres/src/subgraph_store.rs | 6 ------ 1 file changed, 6 deletions(-) diff --git a/store/postgres/src/subgraph_store.rs b/store/postgres/src/subgraph_store.rs index e9f5f2cce34..0beeadf345d 100644 --- a/store/postgres/src/subgraph_store.rs +++ b/store/postgres/src/subgraph_store.rs @@ -699,12 +699,6 @@ impl SubgraphStoreInner { ))); } let deployment = src_store.load_deployment(src.clone())?; - if deployment.failed { - return Err(StoreError::Unknown(anyhow!( - "can not copy deployment {} because it has failed", - src_loc - ))); - } let index_def = src_store.load_indexes(src.clone())?; // Transmogrify the deployment into a new one From 8a8b428650d514377c551fb8ad224dd9b5804342 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Mon, 17 Mar 2025 13:49:09 +0100 Subject: [PATCH 064/150] store: Do not prune deployments that are being copied Pruning during copying can cause the copy to fail because data it needs disappears during copying. --- store/postgres/src/copy.rs | 14 ++++++++++++++ store/postgres/src/deployment_store.rs | 10 +++++++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/store/postgres/src/copy.rs b/store/postgres/src/copy.rs index f2f7e9f1d66..5a31acfb959 100644 --- a/store/postgres/src/copy.rs +++ b/store/postgres/src/copy.rs @@ -94,6 +94,20 @@ table! { } } +/// Return `true` if the site is the source of a copy operation. The copy +/// operation might be just queued or in progress already +pub fn is_source(conn: &mut PgConnection, site: &Site) -> Result { + use active_copies as ac; + + select(diesel::dsl::exists( + ac::table + .filter(ac::src.eq(site.id)) + .filter(ac::cancelled_at.is_null()), + )) + .get_result::(conn) + .map_err(StoreError::from) +} + #[derive(Copy, Clone, PartialEq, Eq)] pub enum Status { Finished, diff --git a/store/postgres/src/deployment_store.rs b/store/postgres/src/deployment_store.rs index b148129d924..df2295c6d54 100644 --- a/store/postgres/src/deployment_store.rs +++ b/store/postgres/src/deployment_store.rs @@ -55,7 +55,7 @@ use crate::primary::DeploymentId; use crate::relational::index::{CreateIndex, IndexList, Method}; use crate::relational::{Layout, LayoutCache, SqlName, Table}; use crate::relational_queries::FromEntityData; -use crate::{advisory_lock, catalog, retry}; +use crate::{advisory_lock, catalog, copy, retry}; use crate::{connection_pool::ConnectionPool, detail}; use crate::{dynds, primary::Site}; @@ -1234,6 +1234,14 @@ impl DeploymentStore { site: Arc, req: PruneRequest, ) -> Result<(), StoreError> { + let mut conn = store.get_conn()?; + if copy::is_source(&mut conn, &site)? { + debug!( + logger, + "Skipping pruning since this deployment is being copied" + ); + return Ok(()); + } let logger2 = logger.cheap_clone(); retry::forever_async(&logger2, "prune", move || { let store = store.cheap_clone(); From 0cc798630024e1550159562f3d0eabd58c6601cd Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Mon, 17 Mar 2025 18:50:35 +0100 Subject: [PATCH 065/150] store: Do not hold a connection unnecessarily --- store/postgres/src/deployment_store.rs | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/store/postgres/src/deployment_store.rs b/store/postgres/src/deployment_store.rs index df2295c6d54..b196cd3c539 100644 --- a/store/postgres/src/deployment_store.rs +++ b/store/postgres/src/deployment_store.rs @@ -1234,13 +1234,15 @@ impl DeploymentStore { site: Arc, req: PruneRequest, ) -> Result<(), StoreError> { - let mut conn = store.get_conn()?; - if copy::is_source(&mut conn, &site)? { - debug!( - logger, - "Skipping pruning since this deployment is being copied" - ); - return Ok(()); + { + let mut conn = store.get_conn()?; + if copy::is_source(&mut conn, &site)? { + debug!( + logger, + "Skipping pruning since this deployment is being copied" + ); + return Ok(()); + } } let logger2 = logger.cheap_clone(); retry::forever_async(&logger2, "prune", move || { From 900f10a8483ecb6a0efb480ebc95c843973b4d79 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Wed, 19 Mar 2025 12:20:31 +0100 Subject: [PATCH 066/150] store: Move filtering of histogram_bounds into VidBatcher::new --- store/postgres/src/vid_batcher.rs | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/store/postgres/src/vid_batcher.rs b/store/postgres/src/vid_batcher.rs index 81da5382e3d..5e640ca586f 100644 --- a/store/postgres/src/vid_batcher.rs +++ b/store/postgres/src/vid_batcher.rs @@ -112,20 +112,6 @@ pub(crate) struct VidBatcher { } impl VidBatcher { - fn histogram_bounds( - conn: &mut PgConnection, - nsp: &Namespace, - table: &Table, - range: VidRange, - ) -> Result, StoreError> { - let bounds = catalog::histogram_bounds(conn, nsp, &table.name, VID_COLUMN)? - .into_iter() - .filter(|bound| range.min < *bound && range.max > *bound) - .chain(vec![range.min, range.max].into_iter()) - .collect::>(); - Ok(bounds) - } - /// Initialize a batcher for batching through entries in `table` with /// `vid` in the given `vid_range` /// @@ -138,7 +124,7 @@ impl VidBatcher { table: &Table, vid_range: VidRange, ) -> Result { - let bounds = Self::histogram_bounds(conn, nsp, table, vid_range)?; + let bounds = catalog::histogram_bounds(conn, nsp, &table.name, VID_COLUMN)?; let batch_size = AdaptiveBatchSize::new(table); Self::new(bounds, vid_range, batch_size) } @@ -150,6 +136,12 @@ impl VidBatcher { ) -> Result { let start = range.min; + let bounds = bounds + .into_iter() + .filter(|bound| range.min < *bound && range.max > *bound) + .chain(vec![range.min, range.max].into_iter()) + .collect::>(); + let mut ogive = if range.is_empty() { None } else { From 2038f1c2ee830813106e2259d8090477222f2adf Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Wed, 19 Mar 2025 12:28:44 +0100 Subject: [PATCH 067/150] store: Fix mistake in handling of histogram bounds in VidBatcher When setting upa VidBatcher we have both accurate values for the range of vids as well as Postgres' estimate of bounds for a histogram with roughly the same number of entries in each bucket. As an example, say we have min and max of 1 and 100, and histogram bounds [5, 50, 96]. We used to then add min and max to these bounds resulting in an ogive over [1, 5, 50, 96, 100]. With that, it seems that there is a bucket [1, 5] with just as many entries as the bucket [5, 50], which is not what the Posgres staistics indicate. Using this ogive will cause e.g. pruning to increase batch size quickly as it tries to get out of the [1, 5] bucket resulting in a batch size that is way too big for the next bucket and a batch that can take a very long time. The first and last entry of the bounds are Postgres' estimate of the min and max. We now simply replace the first and last bound with our known min and max, resulting in an ogive over [1, 50, 100], which reflects the statistics much more accurately and avoids impossibly short buckets. --- store/postgres/src/vid_batcher.rs | 56 +++++++++++++++++++++++++++---- 1 file changed, 50 insertions(+), 6 deletions(-) diff --git a/store/postgres/src/vid_batcher.rs b/store/postgres/src/vid_batcher.rs index 5e640ca586f..2a1c30e7889 100644 --- a/store/postgres/src/vid_batcher.rs +++ b/store/postgres/src/vid_batcher.rs @@ -136,12 +136,26 @@ impl VidBatcher { ) -> Result { let start = range.min; - let bounds = bounds - .into_iter() - .filter(|bound| range.min < *bound && range.max > *bound) - .chain(vec![range.min, range.max].into_iter()) - .collect::>(); - + let bounds = { + // Keep only histogram bounds that are relevent for the range + let mut bounds = bounds + .into_iter() + .filter(|bound| range.min <= *bound && range.max >= *bound) + .collect::>(); + // The first and last entry in `bounds` are Postgres' estimates + // of the min and max `vid` values in the table. We use the + // actual min and max `vid` values from the `vid_range` instead + let len = bounds.len(); + if len > 1 { + bounds[0] = range.min; + bounds[len - 1] = range.max; + } else { + // If Postgres doesn't have a histogram, just use one bucket + // from min to max + bounds = vec![range.min, range.max]; + } + bounds + }; let mut ogive = if range.is_empty() { None } else { @@ -363,6 +377,17 @@ mod tests { } } + impl std::fmt::Debug for Batcher { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Batcher") + .field("start", &self.vid.start) + .field("end", &self.vid.end) + .field("size", &self.vid.batch_size.size) + .field("duration", &self.vid.batch_size.target.as_secs()) + .finish() + } + } + #[test] fn simple() { let bounds = vec![10, 20, 30, 40, 49]; @@ -414,4 +439,23 @@ mod tests { batcher.at(360, 359, 80); batcher.step(360, 359, S010); } + + #[test] + fn vid_batcher_adjusts_bounds() { + // The first and last entry in `bounds` are estimats of the min and + // max that are slightly off compared to the actual min and max we + // put in `vid_range`. Check that `VidBatcher` uses the actual min + // and max from `vid_range`. + let bounds = vec![639, 20_000, 40_000, 60_000, 80_000, 90_000]; + let vid_range = VidRange::new(1, 100_000); + let batch_size = AdaptiveBatchSize { + size: 1000, + target: S100, + }; + + let vid_batcher = VidBatcher::new(bounds, vid_range, batch_size).unwrap(); + let ogive = vid_batcher.ogive.as_ref().unwrap(); + assert_eq!(1, ogive.start()); + assert_eq!(100_000, ogive.end()); + } } From ee52ac7104a1972862c654026fd2b2f35e23c972 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 7 Mar 2025 17:07:19 +0000 Subject: [PATCH 068/150] build(deps): bump ring from 0.17.8 to 0.17.13 Bumps [ring](https://github.com/briansmith/ring) from 0.17.8 to 0.17.13. - [Changelog](https://github.com/briansmith/ring/blob/main/RELEASES.md) - [Commits](https://github.com/briansmith/ring/commits) --- updated-dependencies: - dependency-name: ring dependency-type: indirect ... Signed-off-by: dependabot[bot] --- Cargo.lock | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c511d00601f..0ce5eeef843 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4113,15 +4113,14 @@ dependencies = [ [[package]] name = "ring" -version = "0.17.8" +version = "0.17.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" +checksum = "70ac5d832aa16abd7d1def883a8545280c20a60f523a370aa3a9617c2b8550ee" dependencies = [ "cc", "cfg-if 1.0.0", "getrandom 0.2.15", "libc", - "spin", "untrusted", "windows-sys 0.52.0", ] From 61500fbc64e6377838b6c51f049c0927d7bb8e1e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Mar 2025 21:08:39 +0000 Subject: [PATCH 069/150] build(deps): bump url from 2.5.2 to 2.5.4 Bumps [url](https://github.com/servo/rust-url) from 2.5.2 to 2.5.4. - [Release notes](https://github.com/servo/rust-url/releases) - [Commits](https://github.com/servo/rust-url/compare/v2.5.2...v2.5.4) --- updated-dependencies: - dependency-name: url dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Cargo.lock | 272 +++++++++++++++++++++++++++++++++++++++++++++-- graph/Cargo.toml | 2 +- node/Cargo.toml | 2 +- 3 files changed, 267 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0ce5eeef843..a7ea378f9e3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1351,6 +1351,17 @@ dependencies = [ "winapi", ] +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + [[package]] name = "dsl_auto_type" version = "0.1.1" @@ -2643,6 +2654,124 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "icu_collections" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locid" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_locid_transform" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_locid_transform_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_locid_transform_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e" + +[[package]] +name = "icu_normalizer" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "utf16_iter", + "utf8_iter", + "write16", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516" + +[[package]] +name = "icu_properties" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locid_transform", + "icu_properties_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569" + +[[package]] +name = "icu_provider" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_provider_macros", + "stable_deref_trait", + "tinystr", + "writeable", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_provider_macros" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + [[package]] name = "id-arena" version = "2.2.1" @@ -2668,12 +2797,23 @@ dependencies = [ [[package]] name = "idna" -version = "0.5.0" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" +checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e" dependencies = [ - "unicode-bidi", - "unicode-normalization", + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daca1df1c957320b2cf139ac61e7bd64fed304c5040df000a745aa1de3b4ef71" +dependencies = [ + "icu_normalizer", + "icu_properties", ] [[package]] @@ -2974,6 +3114,12 @@ version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" +[[package]] +name = "litemap" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23fb14cb19457329c82206317a5663005a4d404783dc74f4252769b0d5f42856" + [[package]] name = "lock_api" version = "0.4.12" @@ -4912,6 +5058,17 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" +[[package]] +name = "synstructure" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + [[package]] name = "system-configuration" version = "0.5.1" @@ -5109,6 +5266,16 @@ dependencies = [ "crunchy", ] +[[package]] +name = "tinystr" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" +dependencies = [ + "displaydoc", + "zerovec", +] + [[package]] name = "tinyvec" version = "1.7.0" @@ -5689,12 +5856,12 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "url" -version = "2.5.2" +version = "2.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c" +checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60" dependencies = [ "form_urlencoded", - "idna 0.5.0", + "idna 1.0.3", "percent-encoding", ] @@ -5704,6 +5871,18 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" +[[package]] +name = "utf16_iter" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + [[package]] name = "utf8parse" version = "0.2.2" @@ -6543,6 +6722,18 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "write16" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" + +[[package]] +name = "writeable" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" + [[package]] name = "wyz" version = "0.5.1" @@ -6564,6 +6755,30 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec" +[[package]] +name = "yoke" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "120e6aef9aa629e3d4f52dc8cc43a015c7724194c97dfaf45180d2daf2b77f40" +dependencies = [ + "serde", + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", + "synstructure", +] + [[package]] name = "zerocopy" version = "0.7.35" @@ -6604,12 +6819,55 @@ dependencies = [ "syn 2.0.87", ] +[[package]] +name = "zerofrom" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", + "synstructure", +] + [[package]] name = "zeroize" version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" +[[package]] +name = "zerovec" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + [[package]] name = "zstd" version = "0.11.2+zstd.1.5.2" diff --git a/graph/Cargo.toml b/graph/Cargo.toml index 3ea0c0bf349..163838f5d00 100644 --- a/graph/Cargo.toml +++ b/graph/Cargo.toml @@ -75,7 +75,7 @@ tokio = { version = "1.38.0", features = [ tokio-stream = { version = "0.1.15", features = ["sync"] } tokio-retry = "0.3.0" toml = "0.8.8" -url = "2.5.2" +url = "2.5.4" prometheus = "0.13.4" priority-queue = "2.0.3" tonic = { workspace = true } diff --git a/node/Cargo.toml b/node/Cargo.toml index ee6411fc87c..444b18784fc 100644 --- a/node/Cargo.toml +++ b/node/Cargo.toml @@ -19,7 +19,7 @@ clap.workspace = true git-testament = "0.2" itertools = { workspace = true } lazy_static = "1.5.0" -url = "2.5.2" +url = "2.5.4" graph = { path = "../graph" } graph-core = { path = "../core" } graph-chain-arweave = { path = "../chain/arweave" } From c1011eeaaadb17e64687c69a759c6b0f4ee1a407 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Fri, 21 Mar 2025 08:30:14 -0700 Subject: [PATCH 070/150] store: Fix syntax error in 'alter server' --- store/postgres/src/connection_pool.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/store/postgres/src/connection_pool.rs b/store/postgres/src/connection_pool.rs index 374a1adc5ab..5ad9a60c5e1 100644 --- a/store/postgres/src/connection_pool.rs +++ b/store/postgres/src/connection_pool.rs @@ -185,7 +185,7 @@ impl ForeignServer { alter server \"{name}\" options (set host '{remote_host}', \ {set_port} port '{remote_port}', \ - set dbname '{remote_db}, \ + set dbname '{remote_db}', \ {set_fetch_size} fetch_size '{fetch_size}'); alter user mapping for current_user server \"{name}\" From 3bc203201f7aee2f48a05aed44623dd7fc884b87 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Sat, 22 Mar 2025 09:48:53 -0700 Subject: [PATCH 071/150] store: Analyze tables earlier during copying Analyzing earlier makes it so that Postgres has statistics when rewinding the subgraph --- store/postgres/src/deployment_store.rs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/store/postgres/src/deployment_store.rs b/store/postgres/src/deployment_store.rs index b196cd3c539..01f705158d3 100644 --- a/store/postgres/src/deployment_store.rs +++ b/store/postgres/src/deployment_store.rs @@ -1553,6 +1553,12 @@ impl DeploymentStore { catalog::copy_account_like(conn, &src.site, &dst.site)?; + // Analyze all tables for this deployment + info!(logger, "Analyzing all {} tables", dst.tables.len()); + for entity_name in dst.tables.keys() { + self.analyze_with_conn(site.cheap_clone(), entity_name.as_str(), conn)?; + } + // Rewind the subgraph so that entity versions that are // clamped in the future (beyond `block`) become valid for // all blocks after `block`. `revert_block` gets rid of @@ -1563,6 +1569,7 @@ impl DeploymentStore { .number .checked_add(1) .expect("block numbers fit into an i32"); + info!(logger, "Rewinding to block {}", block.number); let count = dst.revert_block(conn, block_to_revert)?; deployment::update_entity_count(conn, &dst.site, count)?; @@ -1575,11 +1582,6 @@ impl DeploymentStore { src_deployment.manifest.history_blocks, )?; - // Analyze all tables for this deployment - for entity_name in dst.tables.keys() { - self.analyze_with_conn(site.cheap_clone(), entity_name.as_str(), conn)?; - } - // The `earliest_block` for `src` might have changed while // we did the copy if `src` was pruned while we copied; // adjusting it very late in the copy process ensures that From c6f02fc7ef2d022746c70bc75493593b1e58c9c0 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Fri, 21 Mar 2025 09:22:50 -0700 Subject: [PATCH 072/150] store: Make sure we always map the right set of foreign tables --- store/postgres/src/catalog.rs | 10 ++++ store/postgres/src/connection_pool.rs | 80 +++++++++++++++++++++------ 2 files changed, 72 insertions(+), 18 deletions(-) diff --git a/store/postgres/src/catalog.rs b/store/postgres/src/catalog.rs index 1524a768acc..ba532dd53ff 100644 --- a/store/postgres/src/catalog.rs +++ b/store/postgres/src/catalog.rs @@ -398,6 +398,16 @@ pub fn drop_foreign_schema(conn: &mut PgConnection, src: &Site) -> Result<(), St Ok(()) } +pub fn foreign_tables(conn: &mut PgConnection, nsp: &str) -> Result, StoreError> { + use foreign_tables as ft; + + ft::table + .filter(ft::foreign_table_schema.eq(nsp)) + .select(ft::foreign_table_name) + .get_results::(conn) + .map_err(StoreError::from) +} + /// Drop the schema `nsp` and all its contents if it exists, and create it /// again so that `nsp` is an empty schema pub fn recreate_schema(conn: &mut PgConnection, nsp: &str) -> Result<(), StoreError> { diff --git a/store/postgres/src/connection_pool.rs b/store/postgres/src/connection_pool.rs index 5ad9a60c5e1..c4ba365cfd7 100644 --- a/store/postgres/src/connection_pool.rs +++ b/store/postgres/src/connection_pool.rs @@ -37,6 +37,11 @@ use crate::primary::{self, NAMESPACE_PUBLIC}; use crate::{advisory_lock, catalog}; use crate::{Shard, PRIMARY_SHARD}; +/// Tables that we map from the primary into `primary_public` in each shard +const PRIMARY_TABLES: [&str; 3] = ["deployment_schemas", "chains", "active_copies"]; + +/// Tables that we map from each shard into each other shard into the +/// `shard__subgraphs` namespace const SHARDED_TABLES: [(&str, &[&str]); 2] = [ ("public", &["ethereum_networks"]), ( @@ -209,7 +214,7 @@ impl ForeignServer { catalog::recreate_schema(conn, Self::PRIMARY_PUBLIC)?; let mut query = String::new(); - for table_name in ["deployment_schemas", "chains", "active_copies"] { + for table_name in PRIMARY_TABLES { let create_stmt = if shard == &*PRIMARY_SHARD { format!( "create view {nsp}.{table_name} as select * from public.{table_name};", @@ -246,6 +251,33 @@ impl ForeignServer { } Ok(conn.batch_execute(&query)?) } + + fn needs_remap(&self, conn: &mut PgConnection) -> Result { + fn different(mut existing: Vec, mut needed: Vec) -> bool { + existing.sort(); + needed.sort(); + existing != needed + } + + if &self.shard == &*PRIMARY_SHARD { + let existing = catalog::foreign_tables(conn, Self::PRIMARY_PUBLIC)?; + let needed = PRIMARY_TABLES + .into_iter() + .map(String::from) + .collect::>(); + if different(existing, needed) { + return Ok(true); + } + } + + let existing = catalog::foreign_tables(conn, &Self::metadata_schema(&self.shard))?; + let needed = SHARDED_TABLES + .iter() + .flat_map(|(_, tables)| *tables) + .map(|table| table.to_string()) + .collect::>(); + Ok(different(existing, needed)) + } } /// How long to keep connections in the `fdw_pool` around before closing @@ -1037,16 +1069,14 @@ impl PoolInner { let result = pool .configure_fdw(coord.servers.as_ref()) .and_then(|()| pool.drop_cross_shard_views()) - .and_then(|()| migrate_schema(&pool.logger, &mut conn)) - .and_then(|count| { - pool.create_cross_shard_views(coord.servers.as_ref()) - .map(|()| count) - }); + .and_then(|()| migrate_schema(&pool.logger, &mut conn)); debug!(&pool.logger, "Release migration lock"); advisory_lock::unlock_migration(&mut conn).unwrap_or_else(|err| { die(&pool.logger, "failed to release migration lock", &err); }); - let result = result.and_then(|count| coord.propagate(&pool, count)); + let result = result + .and_then(|count| coord.propagate(&pool, count)) + .and_then(|()| pool.create_cross_shard_views(coord.servers.as_ref())); result.unwrap_or_else(|err| die(&pool.logger, "migrations failed", &err)); // Locale check @@ -1178,9 +1208,9 @@ impl PoolInner { .await } - // The foreign server `server` had schema changes, and we therefore need - // to remap anything that we are importing via fdw to make sure we are - // using this updated schema + /// The foreign server `server` had schema changes, and we therefore + /// need to remap anything that we are importing via fdw to make sure we + /// are using this updated schema pub fn remap(&self, server: &ForeignServer) -> Result<(), StoreError> { if &server.shard == &*PRIMARY_SHARD { info!(&self.logger, "Mapping primary"); @@ -1198,6 +1228,15 @@ impl PoolInner { } Ok(()) } + + pub fn needs_remap(&self, server: &ForeignServer) -> Result { + if &server.shard == &self.shard { + return Ok(false); + } + + let mut conn = self.get()?; + server.needs_remap(&mut conn) + } } pub const MIGRATIONS: EmbeddedMigrations = embed_migrations!("./migrations"); @@ -1211,10 +1250,6 @@ impl MigrationCount { fn had_migrations(&self) -> bool { self.old != self.new } - - fn is_new(&self) -> bool { - self.old == 0 - } } /// Run all schema migrations. @@ -1334,13 +1369,22 @@ impl PoolCoordinator { /// code that does _not_ hold the migration lock as it will otherwise /// deadlock fn propagate(&self, pool: &PoolInner, count: MigrationCount) -> Result<(), StoreError> { - // pool is a new shard, map all other shards into it - if count.is_new() { - for server in self.servers.iter() { + // We need to remap all these servers into `pool` if the list of + // tables that are mapped have changed from the code of the previous + // version. Since dropping and recreating the foreign table + // definitions can slow the startup of other nodes down because of + // locking, we try to only do this when it is actually needed + for server in self.servers.iter() { + if pool.needs_remap(server)? { pool.remap(server)?; } } - // pool had schema changes, refresh the import from pool into all other shards + + // pool had schema changes, refresh the import from pool into all + // other shards. This makes sure that schema changes to + // already-mapped tables are propagated to all other shards. Since + // we run `propagate` after migrations have been applied to `pool`, + // we can be sure that these mappings use the correct schema if count.had_migrations() { let server = self.server(&pool.shard)?; for pool in self.pools.lock().unwrap().values() { From 5f2ecb72352038ddf60ebe63c8536e7199fad5ad Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Sat, 22 Mar 2025 10:44:25 -0700 Subject: [PATCH 073/150] store: Do not map the subgraph_features table It's only maintained in the primary, and there's no point in mapping it across shards --- store/postgres/src/connection_pool.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/store/postgres/src/connection_pool.rs b/store/postgres/src/connection_pool.rs index c4ba365cfd7..6267a41628a 100644 --- a/store/postgres/src/connection_pool.rs +++ b/store/postgres/src/connection_pool.rs @@ -52,7 +52,6 @@ const SHARDED_TABLES: [(&str, &[&str]); 2] = [ "dynamic_ethereum_contract_data_source", "subgraph_deployment", "subgraph_error", - "subgraph_features", "subgraph_manifest", "table_stats", ], From 701f77d2d39decfef2ec2d91aa5df0cf5abb7c69 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Sun, 23 Mar 2025 10:46:42 -0700 Subject: [PATCH 074/150] graphman: Annotate failures in 'copy create' with source When creating many copies with a shell script, it is useful to have the deployment we are trying to copy in the error message --- node/src/manager/commands/copy.rs | 34 ++++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/node/src/manager/commands/copy.rs b/node/src/manager/commands/copy.rs index ab007ea319d..d3280823e76 100644 --- a/node/src/manager/commands/copy.rs +++ b/node/src/manager/commands/copy.rs @@ -2,7 +2,7 @@ use diesel::{ExpressionMethods, JoinOnDsl, OptionalExtension, QueryDsl, RunQuery use std::{collections::HashMap, sync::Arc, time::SystemTime}; use graph::{ - components::store::{BlockStore as _, DeploymentId}, + components::store::{BlockStore as _, DeploymentId, DeploymentLocator}, data::query::QueryTarget, prelude::{ anyhow::{anyhow, bail, Error}, @@ -84,10 +84,9 @@ impl CopyState { } } -pub async fn create( +async fn create_inner( store: Arc, - primary: ConnectionPool, - src: DeploymentSearch, + src: &DeploymentLocator, shard: String, shards: Vec, node: String, @@ -104,7 +103,6 @@ pub async fn create( }; let subgraph_store = store.subgraph_store(); - let src = src.locate_unique(&primary)?; let query_store = store .query_store(QueryTarget::Deployment( src.hash.clone(), @@ -154,6 +152,32 @@ pub async fn create( Ok(()) } +pub async fn create( + store: Arc, + primary: ConnectionPool, + src: DeploymentSearch, + shard: String, + shards: Vec, + node: String, + block_offset: u32, + activate: bool, + replace: bool, +) -> Result<(), Error> { + let src = src.locate_unique(&primary)?; + create_inner( + store, + &src, + shard, + shards, + node, + block_offset, + activate, + replace, + ) + .await + .map_err(|e| anyhow!("cannot copy {src}: {e}")) +} + pub fn activate(store: Arc, deployment: String, shard: String) -> Result<(), Error> { let shard = Shard::new(shard)?; let deployment = From ccd65e777ca322816172862eaa198fc478b792b1 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Sun, 23 Mar 2025 13:14:54 -0700 Subject: [PATCH 075/150] graphman: Fix table status indicator for 'copy status' With recent changes, the status shown was '>' (in progress) for all tables that hadn't finished yet, not just the one being worked on --- node/src/manager/commands/copy.rs | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/node/src/manager/commands/copy.rs b/node/src/manager/commands/copy.rs index d3280823e76..9ca80bc9b20 100644 --- a/node/src/manager/commands/copy.rs +++ b/node/src/manager/commands/copy.rs @@ -255,13 +255,11 @@ pub fn list(pools: HashMap) -> Result<(), Error> { } pub fn status(pools: HashMap, dst: &DeploymentSearch) -> Result<(), Error> { + const CHECK: &str = "✓"; + use catalog::active_copies as ac; use catalog::deployment_schemas as ds; - fn done(ts: &Option) -> String { - ts.map(|_| "✓").unwrap_or(".").to_string() - } - fn duration(start: &UtcDateTime, end: &Option) -> String { let start = *start; let end = *end; @@ -314,7 +312,7 @@ pub fn status(pools: HashMap, dst: &DeploymentSearch) -> }; let progress = match &state.finished_at { - Some(_) => done(&state.finished_at), + Some(_) => CHECK.to_string(), None => { let target: i64 = tables.iter().map(|table| table.target_vid).sum(); let next: i64 = tables.iter().map(|table| table.next_vid).sum(); @@ -363,13 +361,15 @@ pub fn status(pools: HashMap, dst: &DeploymentSearch) -> ); println!("{:-<74}", "-"); for table in tables { - let status = if table.next_vid > 0 && table.next_vid < table.target_vid { - ">".to_string() - } else if table.target_vid < 0 { + let status = match &table.finished_at { + // table finished + Some(_) => CHECK, // empty source table - "✓".to_string() - } else { - done(&table.finished_at) + None if table.target_vid < 0 => CHECK, + // copying in progress + None if table.duration_ms > 0 => ">", + // not started + None => ".", }; println!( "{} {:<28} | {:>8} | {:>8} | {:>8} | {:>8}", From dde111cf6cd61d44b61716e583d029b6295fb451 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Sun, 23 Mar 2025 17:10:34 -0700 Subject: [PATCH 076/150] store: Create postponed indexes non-concurrently for copy/graft At the point where we create the postponed indexes during copying, nothing else is writing to the subgraph and we can't be blocking a writer with a normal 'create index'. Since concurrent index creation has to wait for all previous transactions in the database to finish, the concurrent creation can significantly slow down index creation and therefore how long the copy takes. --- store/postgres/src/copy.rs | 8 ++++++-- store/postgres/src/relational/ddl.rs | 9 +++++++-- store/postgres/src/relational/ddl_tests.rs | 2 +- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/store/postgres/src/copy.rs b/store/postgres/src/copy.rs index 5a31acfb959..d82bc33e4a8 100644 --- a/store/postgres/src/copy.rs +++ b/store/postgres/src/copy.rs @@ -730,7 +730,7 @@ impl Connection { &table.src.name.to_string(), &table.dst, true, - true, + false, )?; for (_, sql) in arr { @@ -748,7 +748,11 @@ impl Connection { .iter() .map(|c| c.name.to_string()) .collect_vec(); - for sql in table.dst.create_postponed_indexes(orig_colums).into_iter() { + for sql in table + .dst + .create_postponed_indexes(orig_colums, false) + .into_iter() + { let query = sql_query(sql); query.execute(conn)?; } diff --git a/store/postgres/src/relational/ddl.rs b/store/postgres/src/relational/ddl.rs index 55e116272d1..980bca2b9fd 100644 --- a/store/postgres/src/relational/ddl.rs +++ b/store/postgres/src/relational/ddl.rs @@ -269,7 +269,11 @@ impl Table { (method, index_expr) } - pub(crate) fn create_postponed_indexes(&self, skip_colums: Vec) -> Vec { + pub(crate) fn create_postponed_indexes( + &self, + skip_colums: Vec, + concurrently: bool, + ) -> Vec { let mut indexing_queries = vec![]; let columns = self.columns_to_index(); @@ -281,8 +285,9 @@ impl Table { && column.name.as_str() != "id" && !skip_colums.contains(&column.name.to_string()) { + let conc = if concurrently { "concurrently " } else { "" }; let sql = format!( - "create index concurrently if not exists attr_{table_index}_{column_index}_{table_name}_{column_name}\n on {qname} using {method}({index_expr});\n", + "create index {conc}if not exists attr_{table_index}_{column_index}_{table_name}_{column_name}\n on {qname} using {method}({index_expr});\n", table_index = self.position, table_name = self.name, column_name = column.name, diff --git a/store/postgres/src/relational/ddl_tests.rs b/store/postgres/src/relational/ddl_tests.rs index 86e9f232d49..bb1dcc67f46 100644 --- a/store/postgres/src/relational/ddl_tests.rs +++ b/store/postgres/src/relational/ddl_tests.rs @@ -158,7 +158,7 @@ fn generate_postponed_indexes() { let layout = test_layout(THING_GQL); let table = layout.table(&SqlName::from("Scalar")).unwrap(); let skip_colums = vec!["id".to_string()]; - let query_vec = table.create_postponed_indexes(skip_colums); + let query_vec = table.create_postponed_indexes(skip_colums, true); assert!(query_vec.len() == 7); let queries = query_vec.join(" "); check_eqv(THING_POSTPONED_INDEXES, &queries) From f898defe28ee8f2a509dc1f4e51effb1ab44bb12 Mon Sep 17 00:00:00 2001 From: encalypto Date: Mon, 24 Mar 2025 16:49:13 +0000 Subject: [PATCH 077/150] Redact URLs in RPC error log messages (#5902) --- chain/ethereum/src/ethereum_adapter.rs | 20 ++++++++++++++ graph/src/util/futures.rs | 36 +++++++++++++++++++++++--- 2 files changed, 52 insertions(+), 4 deletions(-) diff --git a/chain/ethereum/src/ethereum_adapter.rs b/chain/ethereum/src/ethereum_adapter.rs index 7173c069c65..e0714c24f02 100644 --- a/chain/ethereum/src/ethereum_adapter.rs +++ b/chain/ethereum/src/ethereum_adapter.rs @@ -147,6 +147,7 @@ impl EthereumAdapter { let retry_log_message = format!("trace_filter RPC call for block range: [{}..{}]", from, to); retry(retry_log_message, &logger) + .redact_log_urls(true) .limit(ENV_VARS.request_retries) .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) .run(move || { @@ -295,6 +296,7 @@ impl EthereumAdapter { let eth_adapter = self.clone(); let retry_log_message = format!("eth_getLogs RPC call for block range: [{}..{}]", from, to); retry(retry_log_message, &logger) + .redact_log_urls(true) .when(move |res: &Result<_, web3::error::Error>| match res { Ok(_) => false, Err(e) => !too_many_logs_fingerprints @@ -511,6 +513,7 @@ impl EthereumAdapter { let retry_log_message = format!("eth_getCode RPC call for block {}", block_ptr); retry(retry_log_message, &logger) + .redact_log_urls(true) .when(|result| match result { Ok(_) => false, Err(_) => true, @@ -546,6 +549,7 @@ impl EthereumAdapter { let retry_log_message = format!("eth_getBalance RPC call for block {}", block_ptr); retry(retry_log_message, &logger) + .redact_log_urls(true) .when(|result| match result { Ok(_) => false, Err(_) => true, @@ -586,6 +590,7 @@ impl EthereumAdapter { let block_id = self.block_ptr_to_id(&block_ptr); let retry_log_message = format!("eth_call RPC call for block {}", block_ptr); retry(retry_log_message, &logger) + .redact_log_urls(true) .limit(ENV_VARS.request_retries) .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) .run(move || { @@ -765,6 +770,7 @@ impl EthereumAdapter { stream::iter_ok::<_, Error>(ids.into_iter().map(move |hash| { let web3 = web3.clone(); retry(format!("load block {}", hash), &logger) + .redact_log_urls(true) .limit(ENV_VARS.request_retries) .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) .run(move || { @@ -799,6 +805,7 @@ impl EthereumAdapter { async move { retry(format!("load block {}", number), &logger) + .redact_log_urls(true) .limit(ENV_VARS.request_retries) .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) .run(move || { @@ -856,6 +863,7 @@ impl EthereumAdapter { stream::iter_ok::<_, Error>(block_nums.into_iter().map(move |block_num| { let web3 = web3.clone(); retry(format!("load block ptr {}", block_num), &logger) + .redact_log_urls(true) .when(|res| !res.is_ok() && !detect_null_block(res)) .no_limit() .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) @@ -1140,6 +1148,7 @@ impl EthereumAdapter { let web3 = self.web3.clone(); u64::try_from( retry("chain_id RPC call", &logger) + .redact_log_urls(true) .no_limit() .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) .run(move || { @@ -1175,6 +1184,7 @@ impl EthereumAdapterTrait for EthereumAdapter { let metrics = self.metrics.clone(); let provider = self.provider().to_string(); let net_version_future = retry("net_version RPC call", &logger) + .redact_log_urls(true) .no_limit() .timeout_secs(20) .run(move || { @@ -1203,6 +1213,7 @@ impl EthereumAdapterTrait for EthereumAdapter { ENV_VARS.genesis_block_number ); let gen_block_hash_future = retry(retry_log_message, &logger) + .redact_log_urls(true) .no_limit() .timeout_secs(30) .run(move || { @@ -1254,6 +1265,7 @@ impl EthereumAdapterTrait for EthereumAdapter { let web3 = self.web3.clone(); Box::new( retry("eth_getBlockByNumber(latest) no txs RPC call", logger) + .redact_log_urls(true) .no_limit() .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) .run(move || { @@ -1288,6 +1300,7 @@ impl EthereumAdapterTrait for EthereumAdapter { let web3 = self.web3.clone(); Box::new( retry("eth_getBlockByNumber(latest) with txs RPC call", logger) + .redact_log_urls(true) .no_limit() .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) .run(move || { @@ -1345,6 +1358,7 @@ impl EthereumAdapterTrait for EthereumAdapter { ); Box::new( retry(retry_log_message, &logger) + .redact_log_urls(true) .limit(ENV_VARS.request_retries) .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) .run(move || { @@ -1376,6 +1390,7 @@ impl EthereumAdapterTrait for EthereumAdapter { ); Box::new( retry(retry_log_message, &logger) + .redact_log_urls(true) .no_limit() .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) .run(move || { @@ -1458,6 +1473,7 @@ impl EthereumAdapterTrait for EthereumAdapter { ); Box::new( retry(retry_log_message, logger) + .redact_log_urls(true) .no_limit() .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) .run(move || { @@ -1525,6 +1541,7 @@ impl EthereumAdapterTrait for EthereumAdapter { let web3 = self.web3.clone(); let logger = logger.clone(); let res = retry(retry_log_message, &logger) + .redact_log_urls(true) .when(|res| !res.is_ok() && !detect_null_block(res)) .no_limit() .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) @@ -2279,6 +2296,7 @@ async fn fetch_transaction_receipts_in_batch_with_retry( block_hash ); retry(retry_log_message, &logger) + .redact_log_urls(true) .limit(ENV_VARS.request_retries) .no_logging() .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) @@ -2406,6 +2424,7 @@ async fn fetch_block_receipts_with_retry( // Perform the retry operation let receipts_option = retry(retry_log_message, &logger) + .redact_log_urls(true) .limit(ENV_VARS.request_retries) .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) .run(move || web3.eth().block_receipts(BlockId::Hash(block_hash)).boxed()) @@ -2450,6 +2469,7 @@ async fn fetch_transaction_receipt_with_retry( transaction_hash ); retry(retry_log_message, &logger) + .redact_log_urls(true) .limit(ENV_VARS.request_retries) .timeout_secs(ENV_VARS.json_rpc_timeout.as_secs()) .run(move || web3.eth().transaction_receipt(transaction_hash).boxed()) diff --git a/graph/src/util/futures.rs b/graph/src/util/futures.rs index d742457dcd1..7c49806c53a 100644 --- a/graph/src/util/futures.rs +++ b/graph/src/util/futures.rs @@ -1,5 +1,7 @@ use crate::ext::futures::FutureExtension; use futures03::{Future, FutureExt, TryFutureExt}; +use lazy_static::lazy_static; +use regex::Regex; use slog::{debug, trace, warn, Logger}; use std::fmt::Debug; use std::marker::PhantomData; @@ -61,6 +63,7 @@ pub fn retry(operation_name: impl ToString, logger: &Logger) -> RetryConfi log_after: 1, warn_after: 10, limit: RetryConfigProperty::Unknown, + redact_log_urls: false, phantom_item: PhantomData, phantom_error: PhantomData, } @@ -75,6 +78,7 @@ pub struct RetryConfig { limit: RetryConfigProperty, phantom_item: PhantomData, phantom_error: PhantomData, + redact_log_urls: bool, } impl RetryConfig @@ -125,6 +129,12 @@ where self } + /// Redact alphanumeric URLs from log messages. + pub fn redact_log_urls(mut self, redact_log_urls: bool) -> Self { + self.redact_log_urls = redact_log_urls; + self + } + /// Set how long (in seconds) to wait for an attempt to complete before giving up on that /// attempt. pub fn timeout_secs(self, timeout_secs: u64) -> RetryConfigWithTimeout { @@ -173,6 +183,7 @@ where let log_after = self.inner.log_after; let warn_after = self.inner.warn_after; let limit_opt = self.inner.limit.unwrap(&operation_name, "limit"); + let redact_log_urls = self.inner.redact_log_urls; let timeout = self.timeout; trace!(logger, "Run with retry: {}", operation_name); @@ -184,6 +195,7 @@ where log_after, warn_after, limit_opt, + redact_log_urls, move || { try_it() .timeout(timeout) @@ -214,6 +226,7 @@ impl RetryConfigNoTimeout { let log_after = self.inner.log_after; let warn_after = self.inner.warn_after; let limit_opt = self.inner.limit.unwrap(&operation_name, "limit"); + let redact_log_urls = self.inner.redact_log_urls; trace!(logger, "Run with retry: {}", operation_name); @@ -224,6 +237,7 @@ impl RetryConfigNoTimeout { log_after, warn_after, limit_opt, + redact_log_urls, // No timeout, so all errors are inner errors move || try_it().map_err(TimeoutError::Inner), ) @@ -265,6 +279,7 @@ fn run_retry( log_after: u64, warn_after: u64, limit_opt: Option, + redact_log_urls: bool, mut try_it_with_timeout: F, ) -> impl Future>> + Send where @@ -311,25 +326,38 @@ where // If needs retry if condition.check(&result) { + let result_str = || { + if redact_log_urls { + lazy_static! { + static ref RE: Regex = + Regex::new(r#"https?://[a-zA-Z0-9\-\._:/\?#&=]+"#).unwrap(); + } + let e = format!("{result:?}"); + RE.replace_all(&e, "[REDACTED]").into_owned() + } else { + format!("{result:?}") + } + }; + if attempt_count >= warn_after { // This looks like it would be nice to de-duplicate, but if we try // to use log! slog complains about requiring a const for the log level // See also b05e1594-e408-4047-aefb-71fc60d70e8f warn!( logger, - "Trying again after {} failed (attempt #{}) with result {:?}", + "Trying again after {} failed (attempt #{}) with result {}", &operation_name, attempt_count, - result + result_str(), ); } else if attempt_count >= log_after { // See also b05e1594-e408-4047-aefb-71fc60d70e8f debug!( logger, - "Trying again after {} failed (attempt #{}) with result {:?}", + "Trying again after {} failed (attempt #{}) with result {}", &operation_name, attempt_count, - result + result_str(), ); } From 09579fbbd3b85185ce0f96cfd1c3cd5e39ec939b Mon Sep 17 00:00:00 2001 From: Zoran Cvetkov <36600146+zorancv@users.noreply.github.com> Date: Wed, 26 Mar 2025 19:22:04 +0200 Subject: [PATCH 078/150] store: fix grafting bug --- store/postgres/src/relational_queries.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/store/postgres/src/relational_queries.rs b/store/postgres/src/relational_queries.rs index 19f9400c470..c6567c5d4f7 100644 --- a/store/postgres/src/relational_queries.rs +++ b/store/postgres/src/relational_queries.rs @@ -4809,7 +4809,7 @@ impl<'a> QueryFragment for CopyEntityBatchQuery<'a> { fn walk_ast<'b>(&'b self, mut out: AstPass<'_, 'b, Pg>) -> QueryResult<()> { out.unsafe_to_cache_prepared(); - let has_vid_seq = self.src.object.has_vid_seq(); + let has_vid_seq = self.dst.object.has_vid_seq(); // Construct a query // insert into {dst}({columns}) From eba48445745cd05ac959538e03b0a8de64cea6c9 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Tue, 25 Mar 2025 13:12:39 -0700 Subject: [PATCH 079/150] store: Do not emit an index in block$ twice If an immutable entity type has a `block` column, we would emit the index on the block$ twice, making deployment fail --- store/postgres/src/relational/ddl_tests.rs | 80 ++++++++++++++++++++++ store/postgres/src/relational/index.rs | 30 ++++++-- 2 files changed, 103 insertions(+), 7 deletions(-) diff --git a/store/postgres/src/relational/ddl_tests.rs b/store/postgres/src/relational/ddl_tests.rs index bb1dcc67f46..53106be2b1a 100644 --- a/store/postgres/src/relational/ddl_tests.rs +++ b/store/postgres/src/relational/ddl_tests.rs @@ -352,6 +352,74 @@ fn can_copy_from() { ); } +/// Check that we do not create the index on `block$` twice. There was a bug +/// that if an immutable entity type had a `block` field and index creation +/// was postponed, we would emit the index on `block$` twice, once from +/// `Table.create_time_travel_indexes` and once through +/// `IndexList.indexes_for_table` +#[test] +fn postponed_indexes_with_block_column() { + fn index_list() -> IndexList { + // To generate this list, print the output of `layout.as_ddl(None)`, run + // that in Postgres and do `select indexdef from pg_indexes where + // schemaname = 'sgd0815'` + const INDEX_DEFS: &[&str] = &[ + "CREATE UNIQUE INDEX data_pkey ON sgd0815.data USING btree (vid)", + "CREATE UNIQUE INDEX data_id_key ON sgd0815.data USING btree (id)", + "CREATE INDEX data_block ON sgd0815.data USING btree (block$)", + "CREATE INDEX attr_1_0_data_block ON sgd0815.data USING btree (block, \"block$\")", + ]; + + let mut indexes: HashMap> = HashMap::new(); + indexes.insert( + "data".to_string(), + INDEX_DEFS + .iter() + .map(|def| CreateIndex::parse(def.to_string())) + .collect(), + ); + IndexList { indexes } + } + // Names of the two indexes we are interested in. Not the leading space + // to guard a little against overlapping names + const BLOCK_IDX: &str = " data_block"; + const ATTR_IDX: &str = " attr_1_0_data_block"; + + let layout = test_layout(BLOCK_GQL); + + // Create everything + let sql = layout.as_ddl(None).unwrap(); + assert!(sql.contains(BLOCK_IDX)); + assert!(sql.contains(ATTR_IDX)); + + // Defer attribute indexes + let sql = layout.as_ddl(Some(index_list())).unwrap(); + assert!(sql.contains(BLOCK_IDX)); + assert!(!sql.contains(ATTR_IDX)); + // This used to be duplicated + let count = sql.matches(BLOCK_IDX).count(); + assert_eq!(1, count); + + let table = layout.table(&SqlName::from("Data")).unwrap(); + let sql = table.create_postponed_indexes(vec![], false); + assert_eq!(1, sql.len()); + assert!(!sql[0].contains(BLOCK_IDX)); + assert!(sql[0].contains(ATTR_IDX)); + + let dst_nsp = Namespace::new("sgd2".to_string()).unwrap(); + let arr = index_list() + .indexes_for_table(&dst_nsp, &table.name.to_string(), &table, true, false) + .unwrap(); + assert_eq!(1, arr.len()); + assert!(!arr[0].1.contains(BLOCK_IDX)); + assert!(arr[0].1.contains(ATTR_IDX)); + + let arr = index_list() + .indexes_for_table(&dst_nsp, &table.name.to_string(), &table, false, false) + .unwrap(); + assert_eq!(0, arr.len()); +} + const THING_GQL: &str = r#" type Thing @entity { id: ID! @@ -1109,3 +1177,15 @@ on "sgd0815"."stats_3_day" using btree("volume"); create index stats_3_day_dims on "sgd0815"."stats_3_day"(group_2, group_1, timestamp); "#; + +const BLOCK_GQL: &str = r#" +type Block @entity(immutable: true) { + id: ID! + number: Int! +} + +type Data @entity(immutable: true) { + id: ID! + block: Block! +} +"#; diff --git a/store/postgres/src/relational/index.rs b/store/postgres/src/relational/index.rs index 4f72e773ee6..77e7c2c400d 100644 --- a/store/postgres/src/relational/index.rs +++ b/store/postgres/src/relational/index.rs @@ -123,7 +123,7 @@ impl Display for Expr { Expr::Column(s) => write!(f, "{s}")?, Expr::Prefix(s, _) => write!(f, "{s}")?, Expr::Vid => write!(f, "vid")?, - Expr::Block => write!(f, "block")?, + Expr::Block => write!(f, "{BLOCK_COLUMN}")?, Expr::BlockRange => write!(f, "block_range")?, Expr::BlockRangeLower => write!(f, "lower(block_range)")?, Expr::BlockRangeUpper => write!(f, "upper(block_range)")?, @@ -488,12 +488,29 @@ impl CreateIndex { && columns[1] == Expr::BlockRange } Method::Brin => false, - Method::BTree | Method::Gin => { + Method::Gin => { + // 'using gin()' columns.len() == 1 && columns[0].is_attribute() && cond.is_none() && with.is_none() } + Method::BTree => { + match columns.len() { + 1 => { + // 'using btree()' + columns[0].is_attribute() && cond.is_none() && with.is_none() + } + 2 => { + // 'using btree(, block$)' + columns[0].is_attribute() + && columns[1] == Expr::Block + && cond.is_none() + && with.is_none() + } + _ => false, + } + } Method::Unknown(_) => false, } } @@ -537,6 +554,7 @@ impl CreateIndex { None, ), dummy(false, BTree, &[Expr::BlockRangeUpper], Some(Cond::Closed)), + dummy(false, BTree, &[Expr::Block], None), ] }; } @@ -630,7 +648,7 @@ impl CreateIndex { } pub fn fields_exist_in_dest<'a>(&self, dest_table: &'a Table) -> bool { - fn column_exists<'a>(it: &mut impl Iterator, column_name: &String) -> bool { + fn column_exists<'a>(it: &mut impl Iterator, column_name: &str) -> bool { it.any(|c| *c == *column_name) } @@ -667,9 +685,7 @@ impl CreateIndex { } Expr::Vid => (), Expr::Block => { - if !column_exists(cols, &"block".to_string()) { - return false; - } + return dest_table.immutable; } Expr::Unknown(expression) => { if some_column_contained( @@ -776,7 +792,7 @@ impl IndexList { // First we check if the fields do exist in the destination subgraph. // In case of grafting that is not given. if ci.fields_exist_in_dest(dest_table) - // Then we check if the index is one of the default indexes not based on + // Then we check if the index is one of the default indexes not based on // the attributes. Those will be created anyway and we should skip them. && !ci.is_default_non_attr_index() // Then ID based indexes in the immutable tables are also created initially From b27cc722e5ac0c0bcf5ef4e0bae7ade35e27203f Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Tue, 25 Mar 2025 17:10:06 -0700 Subject: [PATCH 080/150] store: IndexList.indexes_for_table: split concurrent and if_not_exists At the end of copying, we do not want to create indexes concurrently, but we do want the creation to be idempotent, i.e., have a 'if not exists' clause --- store/postgres/src/copy.rs | 1 + store/postgres/src/relational/ddl.rs | 9 ++++++++- store/postgres/src/relational/ddl_tests.rs | 18 ++++++++++++++++-- store/postgres/src/relational/index.rs | 7 ++++--- 4 files changed, 29 insertions(+), 6 deletions(-) diff --git a/store/postgres/src/copy.rs b/store/postgres/src/copy.rs index d82bc33e4a8..d92064c7f5c 100644 --- a/store/postgres/src/copy.rs +++ b/store/postgres/src/copy.rs @@ -731,6 +731,7 @@ impl Connection { &table.dst, true, false, + true, )?; for (_, sql) in arr { diff --git a/store/postgres/src/relational/ddl.rs b/store/postgres/src/relational/ddl.rs index 980bca2b9fd..e85281a5899 100644 --- a/store/postgres/src/relational/ddl.rs +++ b/store/postgres/src/relational/ddl.rs @@ -408,7 +408,14 @@ impl Table { if index_def.is_some() && ENV_VARS.postpone_attribute_index_creation { let arr = index_def .unwrap() - .indexes_for_table(&self.nsp, &self.name.to_string(), &self, false, false) + .indexes_for_table( + &self.nsp, + &self.name.to_string(), + &self, + false, + false, + false, + ) .map_err(|_| fmt::Error)?; for (_, sql) in arr { writeln!(out, "{};", sql).expect("properly formated index statements") diff --git a/store/postgres/src/relational/ddl_tests.rs b/store/postgres/src/relational/ddl_tests.rs index 53106be2b1a..c9e44854e8f 100644 --- a/store/postgres/src/relational/ddl_tests.rs +++ b/store/postgres/src/relational/ddl_tests.rs @@ -408,14 +408,28 @@ fn postponed_indexes_with_block_column() { let dst_nsp = Namespace::new("sgd2".to_string()).unwrap(); let arr = index_list() - .indexes_for_table(&dst_nsp, &table.name.to_string(), &table, true, false) + .indexes_for_table( + &dst_nsp, + &table.name.to_string(), + &table, + true, + false, + false, + ) .unwrap(); assert_eq!(1, arr.len()); assert!(!arr[0].1.contains(BLOCK_IDX)); assert!(arr[0].1.contains(ATTR_IDX)); let arr = index_list() - .indexes_for_table(&dst_nsp, &table.name.to_string(), &table, false, false) + .indexes_for_table( + &dst_nsp, + &table.name.to_string(), + &table, + false, + false, + false, + ) .unwrap(); assert_eq!(0, arr.len()); } diff --git a/store/postgres/src/relational/index.rs b/store/postgres/src/relational/index.rs index 77e7c2c400d..5776bf8f01f 100644 --- a/store/postgres/src/relational/index.rs +++ b/store/postgres/src/relational/index.rs @@ -784,7 +784,8 @@ impl IndexList { table_name: &String, dest_table: &Table, postponed: bool, - concurrent_if_not_exist: bool, + concurrent: bool, + if_not_exists: bool, ) -> Result, String)>, Error> { let mut arr = vec![]; if let Some(vec) = self.indexes.get(table_name) { @@ -805,7 +806,7 @@ impl IndexList { { if let Ok(sql) = ci .with_nsp(namespace.to_string())? - .to_sql(concurrent_if_not_exist, concurrent_if_not_exist) + .to_sql(concurrent, if_not_exists) { arr.push((ci.name(), sql)) } @@ -829,7 +830,7 @@ impl IndexList { let namespace = &layout.catalog.site.namespace; for table in layout.tables.values() { for (ind_name, create_query) in - self.indexes_for_table(namespace, &table.name.to_string(), table, true, true)? + self.indexes_for_table(namespace, &table.name.to_string(), table, true, true, true)? { if let Some(index_name) = ind_name { let table_name = table.name.clone(); From 6aa489ccb7f5d78e9ba35dd670844390fd32626f Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Wed, 26 Mar 2025 12:14:00 -0700 Subject: [PATCH 081/150] store: Make test postponed_indexes_with_block_column more specific Test that the create index statements do/do not contain 'if not exists' --- store/postgres/src/relational/ddl_tests.rs | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/store/postgres/src/relational/ddl_tests.rs b/store/postgres/src/relational/ddl_tests.rs index c9e44854e8f..b15a40cecfb 100644 --- a/store/postgres/src/relational/ddl_tests.rs +++ b/store/postgres/src/relational/ddl_tests.rs @@ -380,6 +380,15 @@ fn postponed_indexes_with_block_column() { ); IndexList { indexes } } + + fn cr(index: &str) -> String { + format!("create index{}", index) + } + + fn cre(index: &str) -> String { + format!("create index if not exists{}", index) + } + // Names of the two indexes we are interested in. Not the leading space // to guard a little against overlapping names const BLOCK_IDX: &str = " data_block"; @@ -389,12 +398,12 @@ fn postponed_indexes_with_block_column() { // Create everything let sql = layout.as_ddl(None).unwrap(); - assert!(sql.contains(BLOCK_IDX)); - assert!(sql.contains(ATTR_IDX)); + assert!(sql.contains(&cr(BLOCK_IDX))); + assert!(sql.contains(&cr(ATTR_IDX))); // Defer attribute indexes let sql = layout.as_ddl(Some(index_list())).unwrap(); - assert!(sql.contains(BLOCK_IDX)); + assert!(sql.contains(&cr(BLOCK_IDX))); assert!(!sql.contains(ATTR_IDX)); // This used to be duplicated let count = sql.matches(BLOCK_IDX).count(); @@ -404,7 +413,7 @@ fn postponed_indexes_with_block_column() { let sql = table.create_postponed_indexes(vec![], false); assert_eq!(1, sql.len()); assert!(!sql[0].contains(BLOCK_IDX)); - assert!(sql[0].contains(ATTR_IDX)); + assert!(sql[0].contains(&cre(ATTR_IDX))); let dst_nsp = Namespace::new("sgd2".to_string()).unwrap(); let arr = index_list() @@ -419,7 +428,7 @@ fn postponed_indexes_with_block_column() { .unwrap(); assert_eq!(1, arr.len()); assert!(!arr[0].1.contains(BLOCK_IDX)); - assert!(arr[0].1.contains(ATTR_IDX)); + assert!(arr[0].1.contains(&cr(ATTR_IDX))); let arr = index_list() .indexes_for_table( From 0e2ee0ae0b3f011c918187fd409c2c309671516f Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Wed, 26 Mar 2025 12:14:53 -0700 Subject: [PATCH 082/150] store: Address logic error in CreateIndex.fields_exist_in_dest Do not short-circuit checking other columns in the check for Block --- store/postgres/src/relational/index.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/store/postgres/src/relational/index.rs b/store/postgres/src/relational/index.rs index 5776bf8f01f..efa82e901f0 100644 --- a/store/postgres/src/relational/index.rs +++ b/store/postgres/src/relational/index.rs @@ -685,7 +685,9 @@ impl CreateIndex { } Expr::Vid => (), Expr::Block => { - return dest_table.immutable; + if !dest_table.immutable { + return false; + } } Expr::Unknown(expression) => { if some_column_contained( From f2c8a261d3b9733badd50c07a8b59aba2ddc6f25 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Mon, 24 Mar 2025 16:21:34 -0700 Subject: [PATCH 083/150] store: Refactor handling of manifest indexes for ds copying --- store/postgres/src/dynds/private.rs | 73 ++++++++++++++++++----------- 1 file changed, 46 insertions(+), 27 deletions(-) diff --git a/store/postgres/src/dynds/private.rs b/store/postgres/src/dynds/private.rs index e8e7f4ce992..f48192b45d4 100644 --- a/store/postgres/src/dynds/private.rs +++ b/store/postgres/src/dynds/private.rs @@ -1,4 +1,4 @@ -use std::ops::Bound; +use std::{collections::HashMap, ops::Bound}; use diesel::{ pg::sql_types, @@ -252,32 +252,11 @@ impl DataSourcesTable { .order_by(&self.vid) .load::(conn)?; + let manifest_map = + ManifestIdxMap::new(src_manifest_idx_and_name, dst_manifest_idx_and_name); let mut count = 0; - for (block_range, src_manifest_idx, param, context, causality_region, done_at) in src_tuples - { - let name = &src_manifest_idx_and_name - .iter() - .find(|(idx, _)| idx == &src_manifest_idx) - .with_context(|| { - anyhow!( - "the source {} does not have a template with index {}", - self.namespace, - src_manifest_idx - ) - })? - .1; - let dst_manifest_idx = dst_manifest_idx_and_name - .iter() - .find(|(_, n)| n == name) - .with_context(|| { - anyhow!( - "the destination {} is missing a template with name {}. The source {} created one at block {:?}", - dst.namespace, - name, self.namespace, block_range.0 - ) - })? - .0; - + for (block_range, src_idx, param, context, causality_region, done_at) in src_tuples { + let dst_idx = manifest_map.dst_idx(src_idx)?; let query = format!( "\ insert into {dst}(block_range, manifest_idx, param, context, causality_region, done_at) @@ -293,7 +272,7 @@ impl DataSourcesTable { count += sql_query(query) .bind::(target_block) .bind::, _>(block_range) - .bind::(dst_manifest_idx) + .bind::(dst_idx) .bind::, _>(param) .bind::, _>(context) .bind::(causality_region) @@ -361,3 +340,43 @@ impl DataSourcesTable { .optional()?) } } + +/// Map src manifest indexes to dst manifest indexes. If the +/// destination is missing an entry, put `None` as the value for the +/// source index +struct ManifestIdxMap<'a> { + map: HashMap, &'a String)>, +} + +impl<'a> ManifestIdxMap<'a> { + fn new(src: &'a [(i32, String)], dst: &'a [(i32, String)]) -> Self { + let map = src + .iter() + .map(|(src_idx, src_name)| { + ( + *src_idx, + ( + dst.iter() + .find(|(_, dst_name)| src_name == dst_name) + .map(|(dst_idx, _)| *dst_idx), + src_name, + ), + ) + }) + .collect(); + ManifestIdxMap { map } + } + + fn dst_idx(&self, src_idx: i32) -> Result { + let (dst_idx, name) = self.map.get(&src_idx).with_context(|| { + anyhow!("the source does not have a template with index {}", src_idx) + })?; + let dst_idx = dst_idx.with_context(|| { + anyhow!( + "the destination does not have a template with name {}", + name + ) + })?; + Ok(dst_idx) + } +} From a20cc8ad67e763b7798c0cd680fc0bd75a19e524 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Mon, 24 Mar 2025 18:05:59 -0700 Subject: [PATCH 084/150] store: Copy private data sources in batches For large numbers of data sources, the existing RBAR behavior can be very slow --- store/postgres/src/dynds/private.rs | 147 ++++++++++++++++------- store/postgres/src/relational_queries.rs | 2 +- 2 files changed, 106 insertions(+), 43 deletions(-) diff --git a/store/postgres/src/dynds/private.rs b/store/postgres/src/dynds/private.rs index f48192b45d4..50a433df006 100644 --- a/store/postgres/src/dynds/private.rs +++ b/store/postgres/src/dynds/private.rs @@ -1,8 +1,9 @@ use std::{collections::HashMap, ops::Bound}; use diesel::{ - pg::sql_types, + pg::{sql_types, Pg}, prelude::*, + query_builder::{AstPass, QueryFragment, QueryId}, sql_query, sql_types::{Binary, Bool, Integer, Jsonb, Nullable}, PgConnection, QueryDsl, RunQueryDsl, @@ -16,7 +17,7 @@ use graph::{ prelude::{serde_json, BlockNumber, StoreError}, }; -use crate::primary::Namespace; +use crate::{primary::Namespace, relational_queries::POSTGRES_MAX_PARAMETERS}; type DynTable = diesel_dynamic_schema::Table; type DynColumn = diesel_dynamic_schema::Column; @@ -226,16 +227,12 @@ impl DataSourcesTable { return Ok(count as usize); } - type Tuple = ( - (Bound, Bound), - i32, - Option>, - Option, - i32, - Option, - ); + let manifest_map = + ManifestIdxMap::new(src_manifest_idx_and_name, dst_manifest_idx_and_name); - let src_tuples = self + // Load all data sources that were created up to and including + // `target_block` and transform them ready for insertion + let dss: Vec<_> = self .table .clone() .filter( @@ -250,34 +247,18 @@ impl DataSourcesTable { &self.done_at, )) .order_by(&self.vid) - .load::(conn)?; + .load::(conn)? + .into_iter() + .map(|ds| ds.src_to_dst(target_block, &manifest_map)) + .collect::>()?; - let manifest_map = - ManifestIdxMap::new(src_manifest_idx_and_name, dst_manifest_idx_and_name); + // Split all dss into chunks so that we never use more than + // `POSTGRES_MAX_PARAMETERS` bind variables per chunk + let chunk_size = POSTGRES_MAX_PARAMETERS / CopyDsQuery::BIND_PARAMS; let mut count = 0; - for (block_range, src_idx, param, context, causality_region, done_at) in src_tuples { - let dst_idx = manifest_map.dst_idx(src_idx)?; - let query = format!( - "\ - insert into {dst}(block_range, manifest_idx, param, context, causality_region, done_at) - values(case - when upper($2) <= $1 then $2 - else int4range(lower($2), null) - end, - $3, $4, $5, $6, $7) - ", - dst = dst.qname - ); - - count += sql_query(query) - .bind::(target_block) - .bind::, _>(block_range) - .bind::(dst_idx) - .bind::, _>(param) - .bind::, _>(context) - .bind::(causality_region) - .bind::, _>(done_at) - .execute(conn)?; + for chunk in dss.chunks(chunk_size) { + let query = CopyDsQuery::new(dst, chunk)?; + count += query.execute(conn)?; } // If the manifest idxes remained constant, we can test that both tables have the same @@ -344,12 +325,12 @@ impl DataSourcesTable { /// Map src manifest indexes to dst manifest indexes. If the /// destination is missing an entry, put `None` as the value for the /// source index -struct ManifestIdxMap<'a> { - map: HashMap, &'a String)>, +struct ManifestIdxMap { + map: HashMap, String)>, } -impl<'a> ManifestIdxMap<'a> { - fn new(src: &'a [(i32, String)], dst: &'a [(i32, String)]) -> Self { +impl ManifestIdxMap { + fn new(src: &[(i32, String)], dst: &[(i32, String)]) -> Self { let map = src .iter() .map(|(src_idx, src_name)| { @@ -359,7 +340,7 @@ impl<'a> ManifestIdxMap<'a> { dst.iter() .find(|(_, dst_name)| src_name == dst_name) .map(|(dst_idx, _)| *dst_idx), - src_name, + src_name.to_string(), ), ) }) @@ -380,3 +361,85 @@ impl<'a> ManifestIdxMap<'a> { Ok(dst_idx) } } + +#[derive(Queryable)] +struct DsForCopy { + block_range: (Bound, Bound), + idx: i32, + param: Option>, + context: Option, + causality_region: i32, + done_at: Option, +} + +impl DsForCopy { + fn src_to_dst( + mut self, + target_block: BlockNumber, + map: &ManifestIdxMap, + ) -> Result { + // unclamp block range if it ends beyond target block + match self.block_range.1 { + Bound::Included(block) if block > target_block => self.block_range.1 = Bound::Unbounded, + _ => { /* use block range as is */ } + } + // Translate manifest index + self.idx = map.dst_idx(self.idx)?; + Ok(self) + } +} + +struct CopyDsQuery<'a> { + dst: &'a DataSourcesTable, + dss: &'a [DsForCopy], +} + +impl<'a> CopyDsQuery<'a> { + const BIND_PARAMS: usize = 6; + + fn new(dst: &'a DataSourcesTable, dss: &'a [DsForCopy]) -> Result { + Ok(CopyDsQuery { dst, dss }) + } +} + +impl<'a> QueryFragment for CopyDsQuery<'a> { + fn walk_ast<'b>(&'b self, mut out: AstPass<'_, 'b, Pg>) -> QueryResult<()> { + out.unsafe_to_cache_prepared(); + out.push_sql("insert into "); + out.push_sql(&self.dst.qname); + out.push_sql( + "(block_range, manifest_idx, param, context, causality_region, done_at) values ", + ); + let mut first = true; + for ds in self.dss.iter() { + if first { + first = false; + } else { + out.push_sql(", "); + } + out.push_sql("("); + out.push_bind_param::, _>(&ds.block_range)?; + out.push_sql(", "); + out.push_bind_param::(&ds.idx)?; + out.push_sql(", "); + out.push_bind_param::, _>(&ds.param)?; + out.push_sql(", "); + out.push_bind_param::, _>(&ds.context)?; + out.push_sql(", "); + out.push_bind_param::(&ds.causality_region)?; + out.push_sql(", "); + out.push_bind_param::, _>(&ds.done_at)?; + out.push_sql(")"); + } + + Ok(()) + } +} + +impl<'a> QueryId for CopyDsQuery<'a> { + type QueryId = (); + + const HAS_STATIC_QUERY_ID: bool = false; +} + +impl<'a, Conn> RunQueryDsl for CopyDsQuery<'a> {} diff --git a/store/postgres/src/relational_queries.rs b/store/postgres/src/relational_queries.rs index c6567c5d4f7..028f6044c34 100644 --- a/store/postgres/src/relational_queries.rs +++ b/store/postgres/src/relational_queries.rs @@ -53,7 +53,7 @@ use crate::{ const BASE_SQL_COLUMNS: [&str; 2] = ["id", "vid"]; /// The maximum number of bind variables that can be used in a query -const POSTGRES_MAX_PARAMETERS: usize = u16::MAX as usize; // 65535 +pub(crate) const POSTGRES_MAX_PARAMETERS: usize = u16::MAX as usize; // 65535 const SORT_KEY_COLUMN: &str = "sort_key$"; From 6e6ea3b0e370cdaa865e48d90ac819cecab3c6a4 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Mon, 24 Mar 2025 20:48:19 -0700 Subject: [PATCH 085/150] store: Provide more detail in errors from private data source copy --- store/postgres/src/dynds/private.rs | 30 ++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/store/postgres/src/dynds/private.rs b/store/postgres/src/dynds/private.rs index 50a433df006..ebfd109b206 100644 --- a/store/postgres/src/dynds/private.rs +++ b/store/postgres/src/dynds/private.rs @@ -1,4 +1,4 @@ -use std::{collections::HashMap, ops::Bound}; +use std::{collections::HashMap, i32, ops::Bound}; use diesel::{ pg::{sql_types, Pg}, @@ -249,7 +249,7 @@ impl DataSourcesTable { .order_by(&self.vid) .load::(conn)? .into_iter() - .map(|ds| ds.src_to_dst(target_block, &manifest_map)) + .map(|ds| ds.src_to_dst(target_block, &manifest_map, &self.namespace, &dst.namespace)) .collect::>()?; // Split all dss into chunks so that we never use more than @@ -348,14 +348,23 @@ impl ManifestIdxMap { ManifestIdxMap { map } } - fn dst_idx(&self, src_idx: i32) -> Result { + fn dst_idx( + &self, + src_idx: i32, + src_nsp: &Namespace, + src_created: BlockNumber, + dst_nsp: &Namespace, + ) -> Result { let (dst_idx, name) = self.map.get(&src_idx).with_context(|| { - anyhow!("the source does not have a template with index {}", src_idx) + anyhow!( + "the source {src_nsp} does not have a template with \ + index {src_idx} but created one at block {src_created}" + ) })?; let dst_idx = dst_idx.with_context(|| { anyhow!( - "the destination does not have a template with name {}", - name + "the destination {dst_nsp} is missing a template with \ + name {name}. The source {src_nsp} created one at block {src_created}" ) })?; Ok(dst_idx) @@ -377,6 +386,8 @@ impl DsForCopy { mut self, target_block: BlockNumber, map: &ManifestIdxMap, + src_nsp: &Namespace, + dst_nsp: &Namespace, ) -> Result { // unclamp block range if it ends beyond target block match self.block_range.1 { @@ -384,7 +395,12 @@ impl DsForCopy { _ => { /* use block range as is */ } } // Translate manifest index - self.idx = map.dst_idx(self.idx)?; + let src_created = match self.block_range.0 { + Bound::Included(block) => block, + Bound::Excluded(block) => block + 1, + Bound::Unbounded => i32::MAX, + }; + self.idx = map.dst_idx(self.idx, src_nsp, src_created, dst_nsp)?; Ok(self) } } From aa43630155ccf46c1c994183132209145fc2ac15 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Thu, 27 Mar 2025 16:57:13 -0700 Subject: [PATCH 086/150] store: Make ManifestIdxMap::new a little more efficient --- store/postgres/src/dynds/private.rs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/store/postgres/src/dynds/private.rs b/store/postgres/src/dynds/private.rs index ebfd109b206..e22d58f4fae 100644 --- a/store/postgres/src/dynds/private.rs +++ b/store/postgres/src/dynds/private.rs @@ -331,17 +331,14 @@ struct ManifestIdxMap { impl ManifestIdxMap { fn new(src: &[(i32, String)], dst: &[(i32, String)]) -> Self { + let dst_idx_map: HashMap<&String, i32> = + HashMap::from_iter(dst.iter().map(|(idx, name)| (name, *idx))); let map = src .iter() .map(|(src_idx, src_name)| { ( *src_idx, - ( - dst.iter() - .find(|(_, dst_name)| src_name == dst_name) - .map(|(dst_idx, _)| *dst_idx), - src_name.to_string(), - ), + (dst_idx_map.get(src_name).copied(), src_name.to_string()), ) }) .collect(); From 776afa14c324614abc84a72587f199cd7681153a Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Thu, 27 Mar 2025 17:05:31 -0700 Subject: [PATCH 087/150] store: Fix handling of bounds in DsForCopy::src_to_dst --- store/postgres/src/dynds/private.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/store/postgres/src/dynds/private.rs b/store/postgres/src/dynds/private.rs index e22d58f4fae..243a7dc5a57 100644 --- a/store/postgres/src/dynds/private.rs +++ b/store/postgres/src/dynds/private.rs @@ -389,13 +389,16 @@ impl DsForCopy { // unclamp block range if it ends beyond target block match self.block_range.1 { Bound::Included(block) if block > target_block => self.block_range.1 = Bound::Unbounded, + Bound::Excluded(block) if block - 1 > target_block => { + self.block_range.1 = Bound::Unbounded + } _ => { /* use block range as is */ } } // Translate manifest index let src_created = match self.block_range.0 { Bound::Included(block) => block, Bound::Excluded(block) => block + 1, - Bound::Unbounded => i32::MAX, + Bound::Unbounded => 0, }; self.idx = map.dst_idx(self.idx, src_nsp, src_created, dst_nsp)?; Ok(self) From 5ff19746bfe08497c316a9ce59dc93c501eae131 Mon Sep 17 00:00:00 2001 From: Krishnanand V P <44740264+incrypto32@users.noreply.github.com> Date: Mon, 31 Mar 2025 12:42:20 +0400 Subject: [PATCH 088/150] Subgraph Composition: Option to force rpc to fetch block ptrs (#5876) * chain/ethereum: Add parallel block fetching with configurable batch size when using firehose with composable subgraphs * ethereum: Add option to force RPC for block pointer lookups This adds GRAPH_ETHEREUM_FORCE_RPC_FOR_BLOCK_PTRS env var which when enabled forces the use of RPC instead of Firehose for loading block pointers by numbers, with Firehose fallback. Useful for composable subgraphs. * graph: change log level for get_block_by_number * graph: Add get_block_number_with_retry method for firehose endpoint * Address review comments --- chain/ethereum/src/chain.rs | 91 ++++++++++++++++----- chain/ethereum/src/env.rs | 7 ++ graph/src/env/mod.rs | 6 ++ graph/src/firehose/endpoints.rs | 140 +++++++++++++++++++++----------- node/src/chain.rs | 1 + 5 files changed, 175 insertions(+), 70 deletions(-) diff --git a/chain/ethereum/src/chain.rs b/chain/ethereum/src/chain.rs index f632ee36d93..117e3033b18 100644 --- a/chain/ethereum/src/chain.rs +++ b/chain/ethereum/src/chain.rs @@ -17,7 +17,7 @@ use graph::prelude::{ EthereumCallCache, LightEthereumBlock, LightEthereumBlockExt, MetricsRegistry, }; use graph::schema::InputSchema; -use graph::slog::{debug, error, trace}; +use graph::slog::{debug, error, trace, warn}; use graph::substreams::Clock; use graph::{ blockchain::{ @@ -257,6 +257,7 @@ pub struct EthereumAdapterSelector { client: Arc>, registry: Arc, chain_store: Arc, + eth_adapters: Arc, } impl EthereumAdapterSelector { @@ -265,12 +266,14 @@ impl EthereumAdapterSelector { client: Arc>, registry: Arc, chain_store: Arc, + eth_adapters: Arc, ) -> Self { Self { logger_factory, client, registry, chain_store, + eth_adapters, } } } @@ -296,6 +299,7 @@ impl TriggersAdapterSelector for EthereumAdapterSelector { chain_store: self.chain_store.cheap_clone(), unified_api_version, capabilities: *capabilities, + eth_adapters: self.eth_adapters.cheap_clone(), }; Ok(Arc::new(adapter)) } @@ -739,6 +743,7 @@ pub struct TriggersAdapter { chain_client: Arc>, capabilities: NodeCapabilities, unified_api_version: UnifiedMappingApiVersion, + eth_adapters: Arc, } /// Fetches blocks from the cache based on block numbers, excluding duplicates @@ -784,12 +789,34 @@ async fn fetch_unique_blocks_from_cache( "Loading {} block(s) not in the block cache", missing_blocks.len() ); - debug!(logger, "Missing blocks {:?}", missing_blocks); + trace!(logger, "Missing blocks {:?}", missing_blocks.len()); } (blocks, missing_blocks) } +// This is used to load blocks from the RPC. +async fn load_blocks_with_rpc( + logger: &Logger, + adapter: Arc, + chain_store: Arc, + block_numbers: BTreeSet, +) -> Result> { + let logger_clone = logger.clone(); + load_blocks( + logger, + chain_store, + block_numbers, + |missing_numbers| async move { + adapter + .load_block_ptrs_by_numbers_rpc(logger_clone, missing_numbers) + .try_collect() + .await + }, + ) + .await +} + /// Fetches blocks by their numbers, first attempting to load from cache. /// Missing blocks are retrieved from an external source, with all blocks sorted and converted to `BlockFinality` format. async fn load_blocks( @@ -847,6 +874,37 @@ impl TriggersAdapterTrait for TriggersAdapter { ) -> Result> { match &*self.chain_client { ChainClient::Firehose(endpoints) => { + // If the force_rpc_for_block_ptrs flag is set, we will use the RPC to load the blocks + // even if the firehose is available. If no adapter is available, we will log an error. + // And then fallback to the firehose. + if ENV_VARS.force_rpc_for_block_ptrs { + trace!( + logger, + "Loading blocks from RPC (force_rpc_for_block_ptrs is set)"; + "block_numbers" => format!("{:?}", block_numbers) + ); + match self.eth_adapters.cheapest_with(&self.capabilities).await { + Ok(adapter) => { + match load_blocks_with_rpc( + &logger, + adapter, + self.chain_store.clone(), + block_numbers.clone(), + ) + .await + { + Ok(blocks) => return Ok(blocks), + Err(e) => { + warn!(logger, "Error loading blocks from RPC: {}", e); + } + } + } + Err(e) => { + warn!(logger, "Error getting cheapest adapter: {}", e); + } + } + } + trace!( logger, "Loading blocks from firehose"; @@ -884,29 +942,16 @@ impl TriggersAdapterTrait for TriggersAdapter { .await } - ChainClient::Rpc(client) => { + ChainClient::Rpc(eth_adapters) => { trace!( logger, "Loading blocks from RPC"; "block_numbers" => format!("{:?}", block_numbers) ); - let adapter = client.cheapest_with(&self.capabilities).await?; - let chain_store = self.chain_store.clone(); - let logger_clone = logger.clone(); - - load_blocks( - &logger, - chain_store, - block_numbers, - |missing_numbers| async move { - adapter - .load_block_ptrs_by_numbers_rpc(logger_clone, missing_numbers) - .try_collect() - .await - }, - ) - .await + let adapter = eth_adapters.cheapest_with(&self.capabilities).await?; + load_blocks_with_rpc(&logger, adapter, self.chain_store.clone(), block_numbers) + .await } } } @@ -973,10 +1018,12 @@ impl TriggersAdapterTrait for TriggersAdapter { ChainClient::Firehose(endpoints) => { let endpoint = endpoints.endpoint().await?; let block = endpoint - .get_block_by_number::(ptr.number as u64, &self.logger) + .get_block_by_number_with_retry::(ptr.number as u64, &self.logger) .await - .map_err(|e| anyhow!("Failed to fetch block from firehose: {}", e))?; - + .context(format!( + "Failed to fetch block {} from firehose", + ptr.number + ))?; Ok(block.hash() == ptr.hash) } ChainClient::Rpc(adapter) => { diff --git a/chain/ethereum/src/env.rs b/chain/ethereum/src/env.rs index bc7223dbc07..027a26b623f 100644 --- a/chain/ethereum/src/env.rs +++ b/chain/ethereum/src/env.rs @@ -91,6 +91,10 @@ pub struct EnvVars { /// This is a comma separated list of chain ids for which the gas field will not be set /// when calling `eth_call`. pub eth_call_no_gas: Vec, + /// Set by the flag `GRAPH_ETHEREUM_FORCE_RPC_FOR_BLOCK_PTRS`. On by default. + /// When enabled, forces the use of RPC instead of Firehose for loading block pointers by numbers. + /// This is used in composable subgraphs. Firehose can be slow for loading block pointers by numbers. + pub force_rpc_for_block_ptrs: bool, } // This does not print any values avoid accidentally leaking any sensitive env vars @@ -141,6 +145,7 @@ impl From for EnvVars { .filter(|s| !s.is_empty()) .map(str::to_string) .collect(), + force_rpc_for_block_ptrs: x.force_rpc_for_block_ptrs.0, } } } @@ -192,4 +197,6 @@ struct Inner { genesis_block_number: u64, #[envconfig(from = "GRAPH_ETH_CALL_NO_GAS", default = "421613,421614")] eth_call_no_gas: String, + #[envconfig(from = "GRAPH_ETHEREUM_FORCE_RPC_FOR_BLOCK_PTRS", default = "true")] + force_rpc_for_block_ptrs: EnvVarBoolean, } diff --git a/graph/src/env/mod.rs b/graph/src/env/mod.rs index 4383ce17b5c..01d09365c3b 100644 --- a/graph/src/env/mod.rs +++ b/graph/src/env/mod.rs @@ -247,6 +247,9 @@ pub struct EnvVars { /// Set by the environment variable `GRAPH_FIREHOSE_FETCH_BLOCK_TIMEOUT_SECS`. /// The default value is 60 seconds. pub firehose_block_fetch_timeout: u64, + /// Set by the environment variable `GRAPH_FIREHOSE_BLOCK_BATCH_SIZE`. + /// The default value is 10. + pub firehose_block_batch_size: usize, } impl EnvVars { @@ -339,6 +342,7 @@ impl EnvVars { block_write_capacity: inner.block_write_capacity.0, firehose_block_fetch_retry_limit: inner.firehose_block_fetch_retry_limit, firehose_block_fetch_timeout: inner.firehose_block_fetch_timeout, + firehose_block_batch_size: inner.firehose_block_fetch_batch_size, }) } @@ -506,6 +510,8 @@ struct Inner { firehose_block_fetch_retry_limit: usize, #[envconfig(from = "GRAPH_FIREHOSE_FETCH_BLOCK_TIMEOUT_SECS", default = "60")] firehose_block_fetch_timeout: u64, + #[envconfig(from = "GRAPH_FIREHOSE_FETCH_BLOCK_BATCH_SIZE", default = "10")] + firehose_block_fetch_batch_size: usize, } #[derive(Clone, Debug)] diff --git a/graph/src/firehose/endpoints.rs b/graph/src/firehose/endpoints.rs index 825f3ddbd20..0ec95c3e2c5 100644 --- a/graph/src/firehose/endpoints.rs +++ b/graph/src/firehose/endpoints.rs @@ -13,8 +13,9 @@ use crate::{ prelude::{anyhow, debug, DeploymentHash}, substreams_rpc, }; +use anyhow::Context; use async_trait::async_trait; -use futures03::StreamExt; +use futures03::{StreamExt, TryStreamExt}; use http::uri::{Scheme, Uri}; use itertools::Itertools; use slog::{error, info, trace, Logger}; @@ -443,15 +444,47 @@ impl FirehoseEndpoint { } } - pub async fn get_block_by_number( - &self, - number: u64, + pub async fn get_block_by_ptr_with_retry( + self: Arc, + ptr: &BlockPtr, logger: &Logger, ) -> Result where M: prost::Message + BlockchainBlock + Default + 'static, { - debug!( + let retry_log_message = format!("get_block_by_ptr for block {}", ptr); + let endpoint = self.cheap_clone(); + let logger = logger.cheap_clone(); + let ptr_for_retry = ptr.clone(); + + retry(retry_log_message, &logger) + .limit(ENV_VARS.firehose_block_fetch_retry_limit) + .timeout_secs(ENV_VARS.firehose_block_fetch_timeout) + .run(move || { + let endpoint = endpoint.cheap_clone(); + let logger = logger.cheap_clone(); + let ptr = ptr_for_retry.clone(); + async move { + endpoint + .get_block_by_ptr::(&ptr, &logger) + .await + .context(format!( + "Failed to fetch block by ptr {} from firehose", + ptr + )) + } + }) + .await + .map_err(move |e| { + anyhow::anyhow!("Failed to fetch block by ptr {} from firehose: {}", ptr, e) + }) + } + + async fn get_block_by_number(&self, number: u64, logger: &Logger) -> Result + where + M: prost::Message + BlockchainBlock + Default + 'static, + { + trace!( logger, "Connecting to firehose to retrieve block for number {}", number; "provider" => self.provider.as_str(), @@ -473,6 +506,44 @@ impl FirehoseEndpoint { } } + pub async fn get_block_by_number_with_retry( + self: Arc, + number: u64, + logger: &Logger, + ) -> Result + where + M: prost::Message + BlockchainBlock + Default + 'static, + { + let retry_log_message = format!("get_block_by_number for block {}", number); + let endpoint = self.cheap_clone(); + let logger = logger.cheap_clone(); + + retry(retry_log_message, &logger) + .limit(ENV_VARS.firehose_block_fetch_retry_limit) + .timeout_secs(ENV_VARS.firehose_block_fetch_timeout) + .run(move || { + let endpoint = endpoint.cheap_clone(); + let logger = logger.cheap_clone(); + async move { + endpoint + .get_block_by_number::(number, &logger) + .await + .context(format!( + "Failed to fetch block by number {} from firehose", + number + )) + } + }) + .await + .map_err(|e| { + anyhow::anyhow!( + "Failed to fetch block by number {} from firehose: {}", + number, + e + ) + }) + } + pub async fn load_blocks_by_numbers( self: Arc, numbers: Vec, @@ -481,51 +552,24 @@ impl FirehoseEndpoint { where M: prost::Message + BlockchainBlock + Default + 'static, { - let mut blocks = Vec::with_capacity(numbers.len()); - - for number in numbers { - let provider_name = self.provider.as_str(); + let logger = logger.clone(); + let logger_for_error = logger.clone(); + + let blocks_stream = futures03::stream::iter(numbers) + .map(move |number| { + let e = self.cheap_clone(); + let l = logger.clone(); + async move { e.get_block_by_number_with_retry::(number, &l).await } + }) + .buffered(ENV_VARS.firehose_block_batch_size); - trace!( - logger, - "Loading block for block number {}", number; - "provider" => provider_name, + let blocks = blocks_stream.try_collect::>().await.map_err(|e| { + error!( + logger_for_error, + "Failed to load blocks from firehose: {}", e; ); - - let retry_log_message = format!("get_block_by_number for block {}", number); - let endpoint_for_retry = self.cheap_clone(); - - let logger_for_retry = logger.clone(); - let logger_for_error = logger.clone(); - - let block = retry(retry_log_message, &logger_for_retry) - .limit(ENV_VARS.firehose_block_fetch_retry_limit) - .timeout_secs(ENV_VARS.firehose_block_fetch_timeout) - .run(move || { - let e = endpoint_for_retry.cheap_clone(); - let l = logger_for_retry.clone(); - async move { e.get_block_by_number::(number, &l).await } - }) - .await; - - match block { - Ok(block) => { - blocks.push(block); - } - Err(e) => { - error!( - logger_for_error, - "Failed to load block number {}: {}", number, e; - "provider" => provider_name, - ); - return Err(anyhow::format_err!( - "failed to load block number {}: {}", - number, - e - )); - } - } - } + anyhow::format_err!("failed to load blocks from firehose: {}", e) + })?; Ok(blocks) } diff --git a/node/src/chain.rs b/node/src/chain.rs index 00785d11876..239db116e55 100644 --- a/node/src/chain.rs +++ b/node/src/chain.rs @@ -441,6 +441,7 @@ pub async fn networks_as_chains( client.clone(), metrics_registry.clone(), chain_store.clone(), + eth_adapters.clone(), ); let call_cache = chain_store.cheap_clone(); From 127d15c12560e4974d4c41a4778ba4b7f221410a Mon Sep 17 00:00:00 2001 From: Krishnanand V P <44740264+incrypto32@users.noreply.github.com> Date: Mon, 31 Mar 2025 13:39:56 +0400 Subject: [PATCH 089/150] Validate if the source subgraph graft chain contains any incompatible spec version (#5911) * graph: Validate if the graft chain contains any incompatible spec version for composed subgraphs * graph: allow subgraphs with prune set to never to be sourcable * graph: remove the pruning check completely for source subgraphs * Address review comments --- graph/src/data_source/subgraph.rs | 81 ++++++++++++++++++++++++------- 1 file changed, 64 insertions(+), 17 deletions(-) diff --git a/graph/src/data_source/subgraph.rs b/graph/src/data_source/subgraph.rs index 9e120a4c82c..2dd3a35c571 100644 --- a/graph/src/data_source/subgraph.rs +++ b/graph/src/data_source/subgraph.rs @@ -270,6 +270,55 @@ impl UnresolvedDataSource { .map(Arc::new) } + /// Recursively verifies that all grafts in the chain meet the minimum spec version requirement for a subgraph source + async fn verify_graft_chain_sourcable( + manifest: Arc>, + resolver: &Arc, + logger: &Logger, + graft_chain: &mut Vec, + ) -> Result<(), Error> { + // Add current manifest to graft chain + graft_chain.push(manifest.id.to_string()); + + // Check if current manifest meets spec version requirement + if manifest.spec_version < SPEC_VERSION_1_3_0 { + return Err(anyhow!( + "Subgraph with a spec version {} is not supported for a subgraph source, minimum supported version is {}. Graft chain: {}", + manifest.spec_version, + SPEC_VERSION_1_3_0, + graft_chain.join(" -> ") + )); + } + + // If there's a graft, recursively verify it + if let Some(graft) = &manifest.graft { + let graft_raw = resolver + .cat(logger, &graft.base.to_ipfs_link()) + .await + .context("Failed to resolve graft base manifest")?; + + let graft_raw: serde_yaml::Mapping = serde_yaml::from_slice(&graft_raw) + .context("Failed to parse graft base manifest as YAML")?; + + let graft_manifest = + UnresolvedSubgraphManifest::::parse(graft.base.clone(), graft_raw) + .context("Failed to parse graft base manifest")? + .resolve(resolver, logger, LATEST_VERSION.clone()) + .await + .context("Failed to resolve graft base manifest")?; + + Box::pin(Self::verify_graft_chain_sourcable( + Arc::new(graft_manifest), + resolver, + logger, + graft_chain, + )) + .await?; + } + + Ok(()) + } + #[allow(dead_code)] pub(super) async fn resolve( self, @@ -286,15 +335,6 @@ impl UnresolvedDataSource { let kind = self.kind.clone(); let source_manifest = self.resolve_source_manifest::(resolver, logger).await?; let source_spec_version = &source_manifest.spec_version; - - if source_manifest - .data_sources - .iter() - .any(|ds| matches!(ds, crate::data_source::DataSource::Subgraph(_))) - { - return Err(anyhow!("Nested subgraph data sources are not supported.")); - } - if source_spec_version < &SPEC_VERSION_1_3_0 { return Err(anyhow!( "Source subgraph manifest spec version {} is not supported, minimum supported version is {}", @@ -303,15 +343,22 @@ impl UnresolvedDataSource { )); } - let pruning_enabled = match source_manifest.indexer_hints.as_ref() { - None => false, - Some(hints) => hints.prune.is_some(), - }; + // Verify the entire graft chain meets spec version requirements + let mut graft_chain = Vec::new(); + Self::verify_graft_chain_sourcable( + source_manifest.clone(), + resolver, + logger, + &mut graft_chain, + ) + .await?; - if pruning_enabled { - return Err(anyhow!( - "Pruning is enabled for source subgraph, which is not supported" - )); + if source_manifest + .data_sources + .iter() + .any(|ds| matches!(ds, crate::data_source::DataSource::Subgraph(_))) + { + return Err(anyhow!("Nested subgraph data sources are not supported.")); } let mapping_entities: Vec = self From 38c94bb60e64bef5d531bb27d62fbaa9442ee560 Mon Sep 17 00:00:00 2001 From: Krishnanand V P <44740264+incrypto32@users.noreply.github.com> Date: Mon, 31 Mar 2025 13:54:21 +0400 Subject: [PATCH 090/150] Do not allow mutable entities in entity handlers (#5909) * graph: do not allow mutable entities in entity handlers of composed subgraphs * runtime: Remove ToAscPtr implementation for entity trigger * tests: Update subgraph composition integration tests to work with immutable entities * store: Update composition tests to work with immutable entity check --- graph/src/data_source/subgraph.rs | 11 ++- runtime/wasm/src/module/mod.rs | 12 +-- runtime/wasm/src/to_from/external.rs | 37 +-------- .../tests/chain/ethereum/manifest.rs | 77 ++++++++++++++++++- tests/docker-compose.yml | 4 +- .../src/mapping.ts | 30 ++++---- .../subgraph.yaml | 4 +- .../source-subgraph-a/schema.graphql | 2 +- .../source-subgraph-b/schema.graphql | 2 +- .../source-subgraph/schema.graphql | 5 +- .../source-subgraph/src/mapping.ts | 34 +------- .../subgraph-data-sources/src/mapping.ts | 36 ++++----- .../subgraph-data-sources/subgraph.yaml | 4 +- tests/tests/integration_tests.rs | 73 ++++-------------- 14 files changed, 143 insertions(+), 188 deletions(-) diff --git a/graph/src/data_source/subgraph.rs b/graph/src/data_source/subgraph.rs index 2dd3a35c571..d8ef847aee4 100644 --- a/graph/src/data_source/subgraph.rs +++ b/graph/src/data_source/subgraph.rs @@ -239,7 +239,16 @@ impl UnresolvedDataSource { None => { return Err(anyhow!("Entity {} not found in source manifest", entity)); } - Some(TypeKind::Object) => {} + Some(TypeKind::Object) => { + // Check if the entity is immutable + let entity_type = source_manifest.schema.entity_type(entity)?; + if !entity_type.is_immutable() { + return Err(anyhow!( + "Entity {} is not immutable and cannot be used as a mapping entity", + entity + )); + } + } } } Ok(()) diff --git a/runtime/wasm/src/module/mod.rs b/runtime/wasm/src/module/mod.rs index 4b01b3a5fd8..b911542ffe5 100644 --- a/runtime/wasm/src/module/mod.rs +++ b/runtime/wasm/src/module/mod.rs @@ -70,23 +70,13 @@ impl ToAscPtr for offchain::TriggerData { } } -impl ToAscPtr for subgraph::TriggerData { - fn to_asc_ptr( - self, - heap: &mut H, - gas: &GasCounter, - ) -> Result, HostExportError> { - asc_new(heap, &self.entity, gas).map(|ptr| ptr.erase()) - } -} - impl ToAscPtr for subgraph::MappingEntityTrigger { fn to_asc_ptr( self, heap: &mut H, gas: &GasCounter, ) -> Result, HostExportError> { - asc_new(heap, &self.data.entity, gas).map(|ptr| ptr.erase()) + asc_new(heap, &self.data.entity.entity.sorted_ref(), gas).map(|ptr| ptr.erase()) } } diff --git a/runtime/wasm/src/to_from/external.rs b/runtime/wasm/src/to_from/external.rs index 9bbe0298abc..6bb7122613f 100644 --- a/runtime/wasm/src/to_from/external.rs +++ b/runtime/wasm/src/to_from/external.rs @@ -1,13 +1,11 @@ use ethabi; -use graph::blockchain::block_stream::{EntityOperationKind, EntitySourceOperation}; use graph::data::store::scalar::Timestamp; use graph::data::value::Word; use graph::prelude::{BigDecimal, BigInt}; use graph::runtime::gas::GasCounter; use graph::runtime::{ - asc_get, asc_new, AscIndexId, AscPtr, AscType, AscValue, HostExportError, IndexForAscTypeId, - ToAscObj, + asc_get, asc_new, AscIndexId, AscPtr, AscType, AscValue, HostExportError, ToAscObj, }; use graph::{data::store, runtime::DeterministicHostError}; use graph::{prelude::serde_json, runtime::FromAscObj}; @@ -474,39 +472,6 @@ pub enum AscSubgraphEntityOp { Delete, } -#[derive(AscType)] -pub struct AscEntityTrigger { - pub entity_op: AscSubgraphEntityOp, - pub entity_type: AscPtr, - pub entity: AscPtr, - pub vid: i64, -} - -impl ToAscObj for EntitySourceOperation { - fn to_asc_obj( - &self, - heap: &mut H, - gas: &GasCounter, - ) -> Result { - let entity_op = match self.entity_op { - EntityOperationKind::Create => AscSubgraphEntityOp::Create, - EntityOperationKind::Modify => AscSubgraphEntityOp::Modify, - EntityOperationKind::Delete => AscSubgraphEntityOp::Delete, - }; - - Ok(AscEntityTrigger { - entity_op, - entity_type: asc_new(heap, &self.entity_type.as_str(), gas)?, - entity: asc_new(heap, &self.entity.sorted_ref(), gas)?, - vid: self.vid, - }) - } -} - -impl AscIndexId for AscEntityTrigger { - const INDEX_ASC_TYPE_ID: IndexForAscTypeId = IndexForAscTypeId::AscEntityTrigger; -} - impl ToAscObj> for serde_yaml::Value { fn to_asc_obj( &self, diff --git a/store/test-store/tests/chain/ethereum/manifest.rs b/store/test-store/tests/chain/ethereum/manifest.rs index 9d094ae5817..02f4e1413f9 100644 --- a/store/test-store/tests/chain/ethereum/manifest.rs +++ b/store/test-store/tests/chain/ethereum/manifest.rs @@ -47,9 +47,10 @@ specVersion: 1.3.0 "; const SOURCE_SUBGRAPH_SCHEMA: &str = " -type TestEntity @entity { id: ID! } -type User @entity { id: ID! } -type Profile @entity { id: ID! } +type TestEntity @entity(immutable: true) { id: ID! } +type MutableEntity @entity { id: ID! } +type User @entity(immutable: true) { id: ID! } +type Profile @entity(immutable: true) { id: ID! } type TokenData @entity(timeseries: true) { id: Int8! @@ -1761,6 +1762,7 @@ specVersion: 1.3.0 let result = try_resolve_manifest(yaml, SPEC_VERSION_1_3_0).await; assert!(result.is_err()); let err = result.unwrap_err(); + println!("Error: {}", err); assert!(err .to_string() .contains("Subgraph datasources cannot be used alongside onchain datasources")); @@ -1857,3 +1859,72 @@ specVersion: 1.3.0 } }) } + +#[tokio::test] +async fn subgraph_ds_manifest_mutable_entities_should_fail() { + let yaml = " +schema: + file: + /: /ipfs/Qmschema +dataSources: + - name: SubgraphSource + kind: subgraph + entities: + - Gravatar + network: mainnet + source: + address: 'QmSource' + startBlock: 9562480 + mapping: + apiVersion: 0.0.6 + language: wasm/assemblyscript + entities: + - TestEntity + file: + /: /ipfs/Qmmapping + handlers: + - handler: handleEntity + entity: MutableEntity # This is a mutable entity and should fail +specVersion: 1.3.0 +"; + + let result = try_resolve_manifest(yaml, SPEC_VERSION_1_3_0).await; + assert!(result.is_err()); + let err = result.unwrap_err(); + assert!(err + .to_string() + .contains("Entity MutableEntity is not immutable and cannot be used as a mapping entity")); +} + +#[tokio::test] +async fn subgraph_ds_manifest_immutable_entities_should_succeed() { + let yaml = " +schema: + file: + /: /ipfs/Qmschema +dataSources: + - name: SubgraphSource + kind: subgraph + entities: + - Gravatar + network: mainnet + source: + address: 'QmSource' + startBlock: 9562480 + mapping: + apiVersion: 0.0.6 + language: wasm/assemblyscript + entities: + - TestEntity + file: + /: /ipfs/Qmmapping + handlers: + - handler: handleEntity + entity: User # This is an immutable entity and should succeed +specVersion: 1.3.0 +"; + + let result = try_resolve_manifest(yaml, SPEC_VERSION_1_3_0).await; + + assert!(result.is_ok()); +} diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml index 9f05a680e7c..f45360fd367 100644 --- a/tests/docker-compose.yml +++ b/tests/docker-compose.yml @@ -1,7 +1,7 @@ version: '3' services: ipfs: - image: docker.io/ipfs/kubo:v0.17.0 + image: docker.io/ipfs/kubo:v0.34.1 ports: - '127.0.0.1:3001:5001' postgres: @@ -20,7 +20,7 @@ services: POSTGRES_DB: graph-node POSTGRES_INITDB_ARGS: "-E UTF8 --locale=C" anvil: - image: ghcr.io/foundry-rs/foundry:latest + image: ghcr.io/foundry-rs/foundry:stable ports: - '3021:8545' command: "'anvil --host 0.0.0.0 --gas-limit 100000000000 --base-fee 1 --block-time 5 --mnemonic \"test test test test test test test test test test test junk\"'" diff --git a/tests/integration-tests/multiple-subgraph-datasources/src/mapping.ts b/tests/integration-tests/multiple-subgraph-datasources/src/mapping.ts index 649d92d3f5f..373ddd7e99e 100644 --- a/tests/integration-tests/multiple-subgraph-datasources/src/mapping.ts +++ b/tests/integration-tests/multiple-subgraph-datasources/src/mapping.ts @@ -1,28 +1,26 @@ import { dataSource, EntityTrigger, log } from '@graphprotocol/graph-ts' import { AggregatedData } from '../generated/schema' -import { SourceAData } from '../generated/subgraph-QmPWnNsD4m8T9EEF1ec5d8wetFxrMebggLj1efFHzdnZhx' -import { SourceBData } from '../generated/subgraph-Qma4Rk2D1w6mFiP15ZtHHx7eWkqFR426RWswreLiDanxej' +import { SourceAData } from '../generated/subgraph-QmYHp1bPEf7EoYBpEtJUpZv1uQHYQfWE4AhvR6frjB1Huj' +import { SourceBData } from '../generated/subgraph-QmYBEzastJi7bsa722ac78tnZa6xNnV9vvweerY4kVyJtq' -export function handleSourceAData(data: EntityTrigger): void { - let aggregated = AggregatedData.load(data.data.id) - if (!aggregated) { - aggregated = new AggregatedData(data.data.id) - aggregated.sourceA = data.data.data - aggregated.first = 'sourceA' - } else { - aggregated.sourceA = data.data.data - } + +// We know this handler will run first since its defined first in the manifest +// So we dont need to check if the Aggregated data exists +export function handleSourceAData(data: SourceAData): void { + let aggregated = new AggregatedData(data.id) + aggregated.sourceA = data.data + aggregated.first = 'sourceA' aggregated.save() } -export function handleSourceBData(data: EntityTrigger): void { - let aggregated = AggregatedData.load(data.data.id) +export function handleSourceBData(data: SourceBData): void { + let aggregated = AggregatedData.load(data.id) if (!aggregated) { - aggregated = new AggregatedData(data.data.id) - aggregated.sourceB = data.data.data + aggregated = new AggregatedData(data.id) + aggregated.sourceB = data.data aggregated.first = 'sourceB' } else { - aggregated.sourceB = data.data.data + aggregated.sourceB = data.data } aggregated.save() } diff --git a/tests/integration-tests/multiple-subgraph-datasources/subgraph.yaml b/tests/integration-tests/multiple-subgraph-datasources/subgraph.yaml index 296777c578c..4dc4fc7a9b6 100644 --- a/tests/integration-tests/multiple-subgraph-datasources/subgraph.yaml +++ b/tests/integration-tests/multiple-subgraph-datasources/subgraph.yaml @@ -6,7 +6,7 @@ dataSources: name: SourceA network: test source: - address: 'QmPWnNsD4m8T9EEF1ec5d8wetFxrMebggLj1efFHzdnZhx' + address: 'QmYHp1bPEf7EoYBpEtJUpZv1uQHYQfWE4AhvR6frjB1Huj' startBlock: 0 mapping: apiVersion: 0.0.7 @@ -22,7 +22,7 @@ dataSources: name: SourceB network: test source: - address: 'Qma4Rk2D1w6mFiP15ZtHHx7eWkqFR426RWswreLiDanxej' + address: 'QmYBEzastJi7bsa722ac78tnZa6xNnV9vvweerY4kVyJtq' startBlock: 0 mapping: apiVersion: 0.0.7 diff --git a/tests/integration-tests/source-subgraph-a/schema.graphql b/tests/integration-tests/source-subgraph-a/schema.graphql index 10be822d900..2348c9b5c57 100644 --- a/tests/integration-tests/source-subgraph-a/schema.graphql +++ b/tests/integration-tests/source-subgraph-a/schema.graphql @@ -1,4 +1,4 @@ -type SourceAData @entity { +type SourceAData @entity(immutable: true) { id: ID! data: String! blockNumber: BigInt! diff --git a/tests/integration-tests/source-subgraph-b/schema.graphql b/tests/integration-tests/source-subgraph-b/schema.graphql index 9a84bdcbba3..0b012273112 100644 --- a/tests/integration-tests/source-subgraph-b/schema.graphql +++ b/tests/integration-tests/source-subgraph-b/schema.graphql @@ -1,4 +1,4 @@ -type SourceBData @entity { +type SourceBData @entity(immutable: true) { id: ID! data: String! blockNumber: BigInt! diff --git a/tests/integration-tests/source-subgraph/schema.graphql b/tests/integration-tests/source-subgraph/schema.graphql index 15bb2a33921..4fab5be71b9 100644 --- a/tests/integration-tests/source-subgraph/schema.graphql +++ b/tests/integration-tests/source-subgraph/schema.graphql @@ -1,11 +1,10 @@ -type Block @entity { +type Block @entity(immutable: true) { id: ID! number: BigInt! hash: Bytes! - testMessage: String } -type Block2 @entity { +type Block2 @entity(immutable: true) { id: ID! number: BigInt! hash: Bytes! diff --git a/tests/integration-tests/source-subgraph/src/mapping.ts b/tests/integration-tests/source-subgraph/src/mapping.ts index ad27c43c2a3..119fb9b912b 100644 --- a/tests/integration-tests/source-subgraph/src/mapping.ts +++ b/tests/integration-tests/source-subgraph/src/mapping.ts @@ -1,6 +1,5 @@ import { ethereum, log, store } from '@graphprotocol/graph-ts'; import { Block, Block2 } from '../generated/schema'; -import { BigInt } from '@graphprotocol/graph-ts'; export function handleBlock(block: ethereum.Block): void { log.info('handleBlock {}', [block.number.toString()]); @@ -21,37 +20,6 @@ export function handleBlock(block: ethereum.Block): void { let blockEntity3 = new Block2(id3); blockEntity3.number = block.number; blockEntity3.hash = block.hash; + blockEntity3.testMessage = block.number.toString().concat('-message'); blockEntity3.save(); - - if (block.number.equals(BigInt.fromI32(1))) { - let id = 'TEST'; - let entity = new Block(id); - entity.number = block.number; - entity.hash = block.hash; - entity.testMessage = 'Created at block 1'; - log.info('Created entity at block 1', []); - entity.save(); - } - - if (block.number.equals(BigInt.fromI32(2))) { - let id = 'TEST'; - let blockEntity1 = Block.load(id); - if (blockEntity1) { - // Update the block entity - blockEntity1.testMessage = 'Updated at block 2'; - log.info('Updated entity at block 2', []); - blockEntity1.save(); - } - } - - if (block.number.equals(BigInt.fromI32(3))) { - let id = 'TEST'; - let blockEntity1 = Block.load(id); - if (blockEntity1) { - blockEntity1.testMessage = 'Deleted at block 3'; - log.info('Deleted entity at block 3', []); - blockEntity1.save(); - store.remove('Block', id); - } - } } diff --git a/tests/integration-tests/subgraph-data-sources/src/mapping.ts b/tests/integration-tests/subgraph-data-sources/src/mapping.ts index 45ecbd41076..9062970361a 100644 --- a/tests/integration-tests/subgraph-data-sources/src/mapping.ts +++ b/tests/integration-tests/subgraph-data-sources/src/mapping.ts @@ -1,26 +1,26 @@ -import { Entity, log, store, BigInt, EntityTrigger, EntityOp } from '@graphprotocol/graph-ts'; -import { Block } from '../generated/subgraph-QmVz1Pt7NhgCkz4gfavmNrMhojnMT9hW81QDqVjy56ZMUP'; +import { log, store } from '@graphprotocol/graph-ts'; +import { Block, Block2 } from '../generated/subgraph-QmWi3H11QFE2PiWx6WcQkZYZdA5UasaBptUJqGn54MFux5'; import { MirrorBlock } from '../generated/schema'; -export function handleEntity(trigger: EntityTrigger): void { - let blockEntity = trigger.data; - let id = blockEntity.id; +export function handleEntity(block: Block): void { + let id = block.id; - if (trigger.operation === EntityOp.Remove) { - log.info('Removing block entity with id: {}', [id]); - store.remove('MirrorBlock', id); - return; - } + let blockEntity = loadOrCreateMirrorBlock(id); + blockEntity.number = block.number; + blockEntity.hash = block.hash; - let block = loadOrCreateMirrorBlock(id); - block.number = blockEntity.number; - block.hash = blockEntity.hash; - - if (blockEntity.testMessage) { - block.testMessage = blockEntity.testMessage; - } + blockEntity.save(); +} + +export function handleEntity2(block: Block2): void { + let id = block.id; + + let blockEntity = loadOrCreateMirrorBlock(id); + blockEntity.number = block.number; + blockEntity.hash = block.hash; + blockEntity.testMessage = block.testMessage; - block.save(); + blockEntity.save(); } export function loadOrCreateMirrorBlock(id: string): MirrorBlock { diff --git a/tests/integration-tests/subgraph-data-sources/subgraph.yaml b/tests/integration-tests/subgraph-data-sources/subgraph.yaml index 3fdc76ac089..92dc7140514 100644 --- a/tests/integration-tests/subgraph-data-sources/subgraph.yaml +++ b/tests/integration-tests/subgraph-data-sources/subgraph.yaml @@ -6,7 +6,7 @@ dataSources: name: Contract network: test source: - address: 'QmVz1Pt7NhgCkz4gfavmNrMhojnMT9hW81QDqVjy56ZMUP' + address: 'QmWi3H11QFE2PiWx6WcQkZYZdA5UasaBptUJqGn54MFux5' startBlock: 0 mapping: apiVersion: 0.0.7 @@ -16,6 +16,6 @@ dataSources: handlers: - handler: handleEntity entity: Block - - handler: handleEntity + - handler: handleEntity2 entity: Block2 file: ./src/mapping.ts diff --git a/tests/tests/integration_tests.rs b/tests/tests/integration_tests.rs index d10df25698b..5c6ab96968d 100644 --- a/tests/tests/integration_tests.rs +++ b/tests/tests/integration_tests.rs @@ -523,79 +523,34 @@ async fn subgraph_data_sources(ctx: TestContext) -> anyhow::Result<()> { assert!(subgraph.healthy); let expected_response = json!({ "mirrorBlocks": [ - { "id": "1-v1", "number": "1" }, - { "id": "1-v2", "number": "1" }, - { "id": "1-v3", "number": "1" }, - { "id": "2-v1", "number": "2" }, - { "id": "2-v2", "number": "2" }, - { "id": "2-v3", "number": "2" }, - { "id": "3-v1", "number": "3" }, - { "id": "3-v2", "number": "3" }, - { "id": "3-v3", "number": "3" }, - { "id": "4-v1", "number": "4" }, - { "id": "4-v2", "number": "4" }, - { "id": "4-v3", "number": "4" }, - { "id": "5-v1", "number": "5" }, - { "id": "5-v2", "number": "5" }, - { "id": "5-v3", "number": "5" }, - { "id": "6-v1", "number": "6" }, - { "id": "6-v2", "number": "6" }, - { "id": "6-v3", "number": "6" }, - { "id": "7-v1", "number": "7" }, - { "id": "7-v2", "number": "7" }, - { "id": "7-v3", "number": "7" }, - { "id": "8-v1", "number": "8" }, - { "id": "8-v2", "number": "8" }, - { "id": "8-v3", "number": "8" }, - { "id": "9-v1", "number": "9" }, - { "id": "9-v2", "number": "9" }, - { "id": "9-v3", "number": "9" }, - { "id": "10-v1", "number": "10" }, - { "id": "10-v2", "number": "10" }, - { "id": "10-v3", "number": "10" }, + { "id": "1-v1", "number": "1", "testMessage": null }, + { "id": "1-v2", "number": "1", "testMessage": null }, + { "id": "1-v3", "number": "1", "testMessage": "1-message" }, + { "id": "2-v1", "number": "2", "testMessage": null }, + { "id": "2-v2", "number": "2", "testMessage": null }, + { "id": "2-v3", "number": "2", "testMessage": "2-message" }, + { "id": "3-v1", "number": "3", "testMessage": null }, + { "id": "3-v2", "number": "3", "testMessage": null }, + { "id": "3-v3", "number": "3", "testMessage": "3-message" }, ] }); query_succeeds( - "Blocks should be right", + "Query all blocks with testMessage", &subgraph, - "{ mirrorBlocks(where: {number_lte: 10}, orderBy: number) { id, number } }", + "{ mirrorBlocks(where: {number_lte: 3}, orderBy: number) { id, number, testMessage } }", expected_response, ) .await?; let expected_response = json!({ - "mirrorBlock": { "id": "TEST", "number": "1", "testMessage": "Created at block 1" }, + "mirrorBlock": { "id": "1-v3", "number": "1", "testMessage": "1-message" }, }); query_succeeds( - "Blocks should be right", + "Query specific block with testMessage", &subgraph, - "{ mirrorBlock(id: \"TEST\", block: {number: 1}) { id, number, testMessage } }", - expected_response, - ) - .await?; - - let expected_response = json!({ - "mirrorBlock": { "id": "TEST", "number": "1", "testMessage": "Updated at block 2" }, - }); - - query_succeeds( - "Blocks should be right", - &subgraph, - "{ mirrorBlock(id: \"TEST\", block: {number: 2}) { id, number, testMessage } }", - expected_response, - ) - .await?; - - let expected_response = json!({ - "mirrorBlock": null, - }); - - query_succeeds( - "Blocks should be right", - &subgraph, - "{ mirrorBlock(id: \"TEST\", block: {number: 3}) { id, number, testMessage } }", + "{ mirrorBlock(id: \"1-v3\") { id, number, testMessage } }", expected_response, ) .await?; From 7f494ce544a8bde536888b9e031abc4d50d4f1b0 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Wed, 26 Mar 2025 12:37:47 -0700 Subject: [PATCH 091/150] graph: Introduce StoreError::StatementTimeout --- graph/src/components/store/err.rs | 44 ++++++++++++++++--------------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/graph/src/components/store/err.rs b/graph/src/components/store/err.rs index 3aa65c3ecb2..6af676f8e52 100644 --- a/graph/src/components/store/err.rs +++ b/graph/src/components/store/err.rs @@ -74,6 +74,8 @@ pub enum StoreError { UnsupportedFilter(String, String), #[error("writing {0} entities at block {1} failed: {2} Query: {3}")] WriteFailure(String, BlockNumber, String, String), + #[error("database query timed out")] + StatementTimeout, } // Convenience to report a constraint violation @@ -133,25 +135,29 @@ impl Clone for StoreError { Self::WriteFailure(arg0, arg1, arg2, arg3) => { Self::WriteFailure(arg0.clone(), arg1.clone(), arg2.clone(), arg3.clone()) } + Self::StatementTimeout => Self::StatementTimeout, } } } impl StoreError { - fn database_unavailable(e: &DieselError) -> Option { - // When the error is caused by a closed connection, treat the error - // as 'database unavailable'. When this happens during indexing, the - // indexing machinery will retry in that case rather than fail the - // subgraph - if let DieselError::DatabaseError(_, info) = e { - if info - .message() - .contains("server closed the connection unexpectedly") - { - return Some(Self::DatabaseUnavailable); - } + fn from_diesel_error(e: &DieselError) -> Option { + const CONN_CLOSE: &str = "server closed the connection unexpectedly"; + const STMT_TIMEOUT: &str = "canceling statement due to statement timeout"; + let DieselError::DatabaseError(_, info) = e else { + return None; + }; + if info.message().contains(CONN_CLOSE) { + // When the error is caused by a closed connection, treat the error + // as 'database unavailable'. When this happens during indexing, the + // indexing machinery will retry in that case rather than fail the + // subgraph + Some(StoreError::DatabaseUnavailable) + } else if info.message().contains(STMT_TIMEOUT) { + Some(StoreError::StatementTimeout) + } else { + None } - None } pub fn write_failure( @@ -160,19 +166,15 @@ impl StoreError { block: BlockNumber, query: String, ) -> Self { - match Self::database_unavailable(&error) { - Some(e) => return e, - None => StoreError::WriteFailure(entity.to_string(), block, error.to_string(), query), - } + Self::from_diesel_error(&error).unwrap_or_else(|| { + StoreError::WriteFailure(entity.to_string(), block, error.to_string(), query) + }) } } impl From for StoreError { fn from(e: DieselError) -> Self { - match Self::database_unavailable(&e) { - Some(e) => return e, - None => StoreError::Unknown(e.into()), - } + Self::from_diesel_error(&e).unwrap_or_else(|| StoreError::Unknown(e.into())) } } From 3c18373109d16dadab2d9c52fdd5f81cf3de6dbe Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Wed, 26 Mar 2025 13:09:46 -0700 Subject: [PATCH 092/150] graph, store: Add a timeout for each batch of data copying When the data distribution in a table is bad (e.g., if it contains huge arrays after a certain point) it can causes batches to run for many hours. We now set a timeout and reset the batch size to 1 when that happens so that we can slowly inch to a more reasonable batch size. --- docs/environment-variables.md | 8 ++++ graph/src/env/store.rs | 7 ++++ store/postgres/src/copy.rs | 65 ++++++++++++++++++++++++++++++- store/postgres/src/vid_batcher.rs | 4 ++ 4 files changed, 82 insertions(+), 2 deletions(-) diff --git a/docs/environment-variables.md b/docs/environment-variables.md index 8b395680e6a..f174e0e2e54 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -223,6 +223,14 @@ those. copying or grafting should take. This limits how long transactions for such long running operations will be, and therefore helps control bloat in other tables. Value is in seconds and defaults to 180s. +- `GRAPH_STORE_BATCH_TIMEOUT`: How long a batch operation during copying or + grafting is allowed to take at most. This is meant to guard against + batches that are catastrophically big and should be set to a small + multiple of `GRAPH_STORE_BATCH_TARGET_DURATION`, like 10 times that + value, and needs to be at least 2 times that value when set. If this + timeout is hit, the batch size is reset to 1 so we can be sure that + batches stay below `GRAPH_STORE_BATCH_TARGET_DURATION` and the smaller + batch is retried. Value is in seconds and defaults to unlimited. - `GRAPH_START_BLOCK`: block hash:block number where the forked subgraph will start indexing at. - `GRAPH_FORK_BASE`: api url for where the graph node will fork from, use `https://api.thegraph.com/subgraphs/id/` for the hosted service. diff --git a/graph/src/env/store.rs b/graph/src/env/store.rs index 3b4e50ec87d..6d2f383133e 100644 --- a/graph/src/env/store.rs +++ b/graph/src/env/store.rs @@ -81,6 +81,10 @@ pub struct EnvVarsStore { /// The default is 180s. pub batch_target_duration: Duration, + /// Cancel and reset a batch copy operation if it takes longer than + /// this. Set by `GRAPH_STORE_BATCH_TIMEOUT`. Unlimited by default + pub batch_timeout: Option, + /// Prune tables where we will remove at least this fraction of entity /// versions by rebuilding the table. Set by /// `GRAPH_STORE_HISTORY_REBUILD_THRESHOLD`. The default is 0.5 @@ -168,6 +172,7 @@ impl From for EnvVarsStore { connection_idle_timeout: Duration::from_secs(x.connection_idle_timeout_in_secs), write_queue_size: x.write_queue_size, batch_target_duration: Duration::from_secs(x.batch_target_duration_in_secs), + batch_timeout: x.batch_timeout_in_secs.map(Duration::from_secs), rebuild_threshold: x.rebuild_threshold.0, delete_threshold: x.delete_threshold.0, history_slack_factor: x.history_slack_factor.0, @@ -222,6 +227,8 @@ pub struct InnerStore { write_queue_size: usize, #[envconfig(from = "GRAPH_STORE_BATCH_TARGET_DURATION", default = "180")] batch_target_duration_in_secs: u64, + #[envconfig(from = "GRAPH_STORE_BATCH_TIMEOUT")] + batch_timeout_in_secs: Option, #[envconfig(from = "GRAPH_STORE_HISTORY_REBUILD_THRESHOLD", default = "0.5")] rebuild_threshold: ZeroToOneF64, #[envconfig(from = "GRAPH_STORE_HISTORY_DELETE_THRESHOLD", default = "0.05")] diff --git a/store/postgres/src/copy.rs b/store/postgres/src/copy.rs index d92064c7f5c..80a596b7f2c 100644 --- a/store/postgres/src/copy.rs +++ b/store/postgres/src/copy.rs @@ -19,6 +19,7 @@ use std::{ }; use diesel::{ + connection::SimpleConnection as _, dsl::sql, insert_into, r2d2::{ConnectionManager, PooledConnection}, @@ -27,7 +28,7 @@ use diesel::{ }; use graph::{ constraint_violation, - prelude::{info, o, warn, BlockNumber, BlockPtr, Logger, StoreError}, + prelude::{info, lazy_static, o, warn, BlockNumber, BlockPtr, Logger, StoreError, ENV_VARS}, schema::EntityType, }; use itertools::Itertools; @@ -54,6 +55,13 @@ const ACCEPTABLE_REPLICATION_LAG: Duration = Duration::from_secs(30); /// the lag again const REPLICATION_SLEEP: Duration = Duration::from_secs(10); +lazy_static! { + static ref STATEMENT_TIMEOUT: Option = ENV_VARS + .store + .batch_timeout + .map(|duration| format!("set local statement_timeout={}", duration.as_millis())); +} + table! { subgraphs.copy_state(dst) { // deployment_schemas.id @@ -509,6 +517,22 @@ impl TableState { Ok(Status::Finished) } + + fn set_batch_size(&mut self, conn: &mut PgConnection, size: usize) -> Result<(), StoreError> { + use copy_table_state as cts; + + self.batcher.set_batch_size(size); + + update( + cts::table + .filter(cts::dst.eq(self.dst_site.id)) + .filter(cts::entity_type.eq(self.dst.object.as_str())), + ) + .set(cts::batch_size.eq(self.batcher.batch_size() as i64)) + .execute(conn)?; + + Ok(()) + } } // A helper for logging progress while data is being copied @@ -711,7 +735,44 @@ impl Connection { } } - let status = self.transaction(|conn| table.copy_batch(conn))?; + let status = { + loop { + match self.transaction(|conn| { + if let Some(timeout) = STATEMENT_TIMEOUT.as_ref() { + conn.batch_execute(timeout)?; + } + table.copy_batch(conn) + }) { + Ok(status) => { + break status; + } + Err(StoreError::StatementTimeout) => { + warn!( + logger, + "Current batch took longer than GRAPH_STORE_BATCH_TIMEOUT seconds. Retrying with a smaller batch size." + ); + } + Err(e) => { + return Err(e); + } + } + // We hit a timeout. Reset the batch size to 1. + // That's small enough that we will make _some_ + // progress, assuming the timeout is set to a + // reasonable value (several minutes) + // + // Our estimation of batch sizes is generally good + // and stays within the prescribed bounds, but there + // are cases where proper estimation of the batch + // size is nearly impossible since the size of the + // rows in the table jumps sharply at some point + // that is hard to predict. This mechanism ensures + // that if our estimation is wrong, the consequences + // aren't too severe. + self.transaction(|conn| table.set_batch_size(conn, 1))?; + } + }; + if status == Status::Cancelled { return Ok(status); } diff --git a/store/postgres/src/vid_batcher.rs b/store/postgres/src/vid_batcher.rs index 2a1c30e7889..ef5948efd06 100644 --- a/store/postgres/src/vid_batcher.rs +++ b/store/postgres/src/vid_batcher.rs @@ -233,6 +233,10 @@ impl VidBatcher { } } } + + pub(crate) fn set_batch_size(&mut self, size: usize) { + self.batch_size.size = size as i64; + } } #[derive(Copy, Clone, QueryableByName)] From cfc4d8d7913dae64e3b24613b5a22e84773551f8 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Fri, 28 Mar 2025 09:48:37 -0700 Subject: [PATCH 093/150] graph: Check that BATCH_TIMEOUT is big enough A value that's too small will just needlessly cause timeouts and slow down copies. --- graph/src/env/mod.rs | 4 ++-- graph/src/env/store.rs | 16 +++++++++++++--- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/graph/src/env/mod.rs b/graph/src/env/mod.rs index 01d09365c3b..48fa0ba4688 100644 --- a/graph/src/env/mod.rs +++ b/graph/src/env/mod.rs @@ -253,11 +253,11 @@ pub struct EnvVars { } impl EnvVars { - pub fn from_env() -> Result { + pub fn from_env() -> Result { let inner = Inner::init_from_env()?; let graphql = InnerGraphQl::init_from_env()?.into(); let mapping_handlers = InnerMappingHandlers::init_from_env()?.into(); - let store = InnerStore::init_from_env()?.into(); + let store = InnerStore::init_from_env()?.try_into()?; // The default reorganization (reorg) threshold is set to 250. // For testing purposes, we need to set this threshold to 0 because: diff --git a/graph/src/env/store.rs b/graph/src/env/store.rs index 6d2f383133e..4fb30f58079 100644 --- a/graph/src/env/store.rs +++ b/graph/src/env/store.rs @@ -142,9 +142,11 @@ impl fmt::Debug for EnvVarsStore { } } -impl From for EnvVarsStore { - fn from(x: InnerStore) -> Self { - Self { +impl TryFrom for EnvVarsStore { + type Error = anyhow::Error; + + fn try_from(x: InnerStore) -> Result { + let vars = Self { chain_head_watcher_timeout: Duration::from_secs(x.chain_head_watcher_timeout_in_secs), query_stats_refresh_interval: Duration::from_secs( x.query_stats_refresh_interval_in_secs, @@ -184,7 +186,15 @@ impl From for EnvVarsStore { last_rollup_from_poi: x.last_rollup_from_poi, insert_extra_cols: x.insert_extra_cols, fdw_fetch_size: x.fdw_fetch_size, + }; + if let Some(timeout) = vars.batch_timeout { + if timeout < 2 * vars.batch_target_duration { + bail!( + "GRAPH_STORE_BATCH_TIMEOUT must be greater than 2*GRAPH_STORE_BATCH_TARGET_DURATION" + ); + } } + Ok(vars) } } From a5ac766655e0eb9d6823b3e0ffd7c35d6c6cd999 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Fri, 28 Mar 2025 10:04:52 -0700 Subject: [PATCH 094/150] store: Factor the loop for copying a table into a method --- store/postgres/src/copy.rs | 149 +++++++++++++++++++++---------------- 1 file changed, 83 insertions(+), 66 deletions(-) diff --git a/store/postgres/src/copy.rs b/store/postgres/src/copy.rs index 80a596b7f2c..6e909906c43 100644 --- a/store/postgres/src/copy.rs +++ b/store/postgres/src/copy.rs @@ -710,75 +710,12 @@ impl Connection { progress.start(); for table in state.tables.iter_mut().filter(|table| !table.finished()) { - while !table.finished() { - // It is important that this check happens outside the write - // transaction so that we do not hold on to locks acquired - // by the check - if table.is_cancelled(&mut self.conn)? { + match self.copy_table(logger, &mut progress, table)? { + Status::Finished => { /* Move on to the next table */ } + Status::Cancelled => { return Ok(Status::Cancelled); } - - // Pause copying if replication is lagging behind to avoid - // overloading replicas - let mut lag = catalog::replication_lag(&mut self.conn)?; - if lag > MAX_REPLICATION_LAG { - loop { - info!(&self.logger, - "Replicas are lagging too much; pausing copying for {}s to allow them to catch up", - REPLICATION_SLEEP.as_secs(); - "lag_s" => lag.as_secs()); - std::thread::sleep(REPLICATION_SLEEP); - lag = catalog::replication_lag(&mut self.conn)?; - if lag <= ACCEPTABLE_REPLICATION_LAG { - break; - } - } - } - - let status = { - loop { - match self.transaction(|conn| { - if let Some(timeout) = STATEMENT_TIMEOUT.as_ref() { - conn.batch_execute(timeout)?; - } - table.copy_batch(conn) - }) { - Ok(status) => { - break status; - } - Err(StoreError::StatementTimeout) => { - warn!( - logger, - "Current batch took longer than GRAPH_STORE_BATCH_TIMEOUT seconds. Retrying with a smaller batch size." - ); - } - Err(e) => { - return Err(e); - } - } - // We hit a timeout. Reset the batch size to 1. - // That's small enough that we will make _some_ - // progress, assuming the timeout is set to a - // reasonable value (several minutes) - // - // Our estimation of batch sizes is generally good - // and stays within the prescribed bounds, but there - // are cases where proper estimation of the batch - // size is nearly impossible since the size of the - // rows in the table jumps sharply at some point - // that is hard to predict. This mechanism ensures - // that if our estimation is wrong, the consequences - // aren't too severe. - self.transaction(|conn| table.set_batch_size(conn, 1))?; - } - }; - - if status == Status::Cancelled { - return Ok(status); - } - progress.update(&table.dst.object, &table.batcher); } - progress.table_finished(&table.batcher); } // Create indexes for all the attributes that were postponed at the start of @@ -828,6 +765,86 @@ impl Connection { Ok(Status::Finished) } + fn copy_table( + &mut self, + logger: &Logger, + progress: &mut CopyProgress<'_>, + table: &mut TableState, + ) -> Result { + use Status::*; + + while !table.finished() { + // It is important that this check happens outside the write + // transaction so that we do not hold on to locks acquired + // by the check + if table.is_cancelled(&mut self.conn)? { + return Ok(Cancelled); + } + + // Pause copying if replication is lagging behind to avoid + // overloading replicas + let mut lag = catalog::replication_lag(&mut self.conn)?; + if lag > MAX_REPLICATION_LAG { + loop { + info!(&self.logger, + "Replicas are lagging too much; pausing copying for {}s to allow them to catch up", + REPLICATION_SLEEP.as_secs(); + "lag_s" => lag.as_secs()); + std::thread::sleep(REPLICATION_SLEEP); + lag = catalog::replication_lag(&mut self.conn)?; + if lag <= ACCEPTABLE_REPLICATION_LAG { + break; + } + } + } + + let status = { + loop { + match self.transaction(|conn| { + if let Some(timeout) = STATEMENT_TIMEOUT.as_ref() { + conn.batch_execute(timeout)?; + } + table.copy_batch(conn) + }) { + Ok(status) => { + break status; + } + Err(StoreError::StatementTimeout) => { + warn!( + logger, + "Current batch took longer than GRAPH_STORE_BATCH_TIMEOUT seconds. Retrying with a smaller batch size." + ); + } + Err(e) => { + return Err(e); + } + } + // We hit a timeout. Reset the batch size to 1. + // That's small enough that we will make _some_ + // progress, assuming the timeout is set to a + // reasonable value (several minutes) + // + // Our estimation of batch sizes is generally good + // and stays within the prescribed bounds, but there + // are cases where proper estimation of the batch + // size is nearly impossible since the size of the + // rows in the table jumps sharply at some point + // that is hard to predict. This mechanism ensures + // that if our estimation is wrong, the consequences + // aren't too severe. + self.transaction(|conn| table.set_batch_size(conn, 1))?; + } + }; + + if status == Cancelled { + return Ok(Cancelled); + } + progress.update(&table.dst.object, &table.batcher); + } + progress.table_finished(&table.batcher); + Ok(Finished) + } + /// Copy the data for the subgraph `src` to the subgraph `dst`. The /// schema for both subgraphs must have already been set up. The /// `target_block` must be far enough behind the chain head so that the From 2efe3a47bc72abd5a25147cd3b70a48f80ef022a Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Fri, 28 Mar 2025 10:40:06 -0700 Subject: [PATCH 095/150] store: Make copy_table a free-standing function --- store/postgres/src/copy.rs | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/store/postgres/src/copy.rs b/store/postgres/src/copy.rs index 6e909906c43..758ec98cedf 100644 --- a/store/postgres/src/copy.rs +++ b/store/postgres/src/copy.rs @@ -122,7 +122,6 @@ pub enum Status { Cancelled, } -#[allow(dead_code)] struct CopyState { src: Arc, dst: Arc, @@ -710,7 +709,7 @@ impl Connection { progress.start(); for table in state.tables.iter_mut().filter(|table| !table.finished()) { - match self.copy_table(logger, &mut progress, table)? { + match Self::copy_table(&mut self.conn, logger, &mut progress, table)? { Status::Finished => { /* Move on to the next table */ } Status::Cancelled => { return Ok(Status::Cancelled); @@ -766,7 +765,7 @@ impl Connection { } fn copy_table( - &mut self, + conn: &mut PgConnection, logger: &Logger, progress: &mut CopyProgress<'_>, table: &mut TableState, @@ -777,21 +776,21 @@ impl Connection { // It is important that this check happens outside the write // transaction so that we do not hold on to locks acquired // by the check - if table.is_cancelled(&mut self.conn)? { + if table.is_cancelled(conn)? { return Ok(Cancelled); } // Pause copying if replication is lagging behind to avoid // overloading replicas - let mut lag = catalog::replication_lag(&mut self.conn)?; + let mut lag = catalog::replication_lag(conn)?; if lag > MAX_REPLICATION_LAG { loop { - info!(&self.logger, + info!(logger, "Replicas are lagging too much; pausing copying for {}s to allow them to catch up", REPLICATION_SLEEP.as_secs(); "lag_s" => lag.as_secs()); std::thread::sleep(REPLICATION_SLEEP); - lag = catalog::replication_lag(&mut self.conn)?; + lag = catalog::replication_lag(conn)?; if lag <= ACCEPTABLE_REPLICATION_LAG { break; } @@ -800,7 +799,7 @@ impl Connection { let status = { loop { - match self.transaction(|conn| { + match conn.transaction(|conn| { if let Some(timeout) = STATEMENT_TIMEOUT.as_ref() { conn.batch_execute(timeout)?; } @@ -832,7 +831,7 @@ impl Connection { // that is hard to predict. This mechanism ensures // that if our estimation is wrong, the consequences // aren't too severe. - self.transaction(|conn| table.set_batch_size(conn, 1))?; + conn.transaction(|conn| table.set_batch_size(conn, 1))?; } }; From 9cfafa31bfbf2b83061dab67d1192dd5d4d7f83f Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Fri, 28 Mar 2025 12:29:01 -0700 Subject: [PATCH 096/150] store: Asyncify subgraph start, and thereby copying --- store/postgres/src/copy.rs | 9 ++++++--- store/postgres/src/deployment_store.rs | 4 ++-- store/postgres/src/writable.rs | 6 ++++-- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/store/postgres/src/copy.rs b/store/postgres/src/copy.rs index 758ec98cedf..2582237a827 100644 --- a/store/postgres/src/copy.rs +++ b/store/postgres/src/copy.rs @@ -698,7 +698,10 @@ impl Connection { Ok(()) } - pub fn copy_data_internal(&mut self, index_list: IndexList) -> Result { + pub async fn copy_data_internal( + &mut self, + index_list: IndexList, + ) -> Result { let src = self.src.clone(); let dst = self.dst.clone(); let target_block = self.target_block.clone(); @@ -860,7 +863,7 @@ impl Connection { /// lower(v1.block_range) => v2.vid > v1.vid` and we can therefore stop /// the copying of each table as soon as we hit `max_vid = max { v.vid | /// lower(v.block_range) <= target_block.number }`. - pub fn copy_data(&mut self, index_list: IndexList) -> Result { + pub async fn copy_data(&mut self, index_list: IndexList) -> Result { // We require sole access to the destination site, and that we get a // consistent view of what has been copied so far. In general, that // is always true. It can happen though that this function runs when @@ -874,7 +877,7 @@ impl Connection { "Obtaining copy lock (this might take a long time if another process is still copying)" ); advisory_lock::lock_copying(&mut self.conn, self.dst.site.as_ref())?; - let res = self.copy_data_internal(index_list); + let res = self.copy_data_internal(index_list).await; advisory_lock::unlock_copying(&mut self.conn, self.dst.site.as_ref())?; if matches!(res, Ok(Status::Cancelled)) { warn!(&self.logger, "Copying was cancelled and is incomplete"); diff --git a/store/postgres/src/deployment_store.rs b/store/postgres/src/deployment_store.rs index 01f705158d3..248ba5a5473 100644 --- a/store/postgres/src/deployment_store.rs +++ b/store/postgres/src/deployment_store.rs @@ -1491,7 +1491,7 @@ impl DeploymentStore { /// to the graph point, so that calling this needlessly with `Some(..)` /// will remove any progress that might have been made since the last /// time the deployment was started. - pub(crate) fn start_subgraph( + pub(crate) async fn start_subgraph( &self, logger: &Logger, site: Arc, @@ -1528,7 +1528,7 @@ impl DeploymentStore { src_manifest_idx_and_name, dst_manifest_idx_and_name, )?; - let status = copy_conn.copy_data(index_list)?; + let status = copy_conn.copy_data(index_list).await?; if status == crate::copy::Status::Cancelled { return Err(StoreError::Canceled); } diff --git a/store/postgres/src/writable.rs b/store/postgres/src/writable.rs index 07d116790c0..26e559bcbc9 100644 --- a/store/postgres/src/writable.rs +++ b/store/postgres/src/writable.rs @@ -220,8 +220,10 @@ impl SyncStore { } None => None, }; - self.writable - .start_subgraph(logger, self.site.clone(), graft_base)?; + graph::block_on( + self.writable + .start_subgraph(logger, self.site.clone(), graft_base), + )?; self.store.primary_conn()?.copy_finished(self.site.as_ref()) }) } From d6e337c2ddeb954c57624ac0e84d353b698d24ac Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Fri, 28 Mar 2025 12:52:01 -0700 Subject: [PATCH 097/150] store: Do not access conn in copy::Connection directly We leave one place where it is accessed, but in all other places we go through copy::Connection::transaction; the reason will become apparent in the next commit(s) --- store/postgres/src/copy.rs | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/store/postgres/src/copy.rs b/store/postgres/src/copy.rs index 2582237a827..3bcfd6ecb03 100644 --- a/store/postgres/src/copy.rs +++ b/store/postgres/src/copy.rs @@ -13,7 +13,6 @@ //! `graph-node` was restarted while the copy was running. use std::{ convert::TryFrom, - ops::DerefMut, sync::Arc, time::{Duration, Instant}, }; @@ -28,7 +27,9 @@ use diesel::{ }; use graph::{ constraint_violation, - prelude::{info, lazy_static, o, warn, BlockNumber, BlockPtr, Logger, StoreError, ENV_VARS}, + prelude::{ + info, lazy_static, o, warn, BlockNumber, BlockPtr, CheapClone, Logger, StoreError, ENV_VARS, + }, schema::EntityType, }; use itertools::Itertools; @@ -623,8 +624,8 @@ pub struct Connection { src: Arc, dst: Arc, target_block: BlockPtr, - src_manifest_idx_and_name: Vec<(i32, String)>, - dst_manifest_idx_and_name: Vec<(i32, String)>, + src_manifest_idx_and_name: Arc>, + dst_manifest_idx_and_name: Arc>, } impl Connection { @@ -661,6 +662,8 @@ impl Connection { } false })?; + let src_manifest_idx_and_name = Arc::new(src_manifest_idx_and_name); + let dst_manifest_idx_and_name = Arc::new(dst_manifest_idx_and_name); Ok(Self { logger, conn, @@ -683,15 +686,16 @@ impl Connection { /// has a private data sources table. The copying is done in its own /// transaction. fn copy_private_data_sources(&mut self, state: &CopyState) -> Result<(), StoreError> { + let src_manifest_idx_and_name = self.src_manifest_idx_and_name.cheap_clone(); + let dst_manifest_idx_and_name = self.dst_manifest_idx_and_name.cheap_clone(); if state.src.site.schema_version.private_data_sources() { - let conn = &mut self.conn; - conn.transaction(|conn| { + self.transaction(|conn| { DataSourcesTable::new(state.src.site.namespace.clone()).copy_to( conn, &DataSourcesTable::new(state.dst.site.namespace.clone()), state.target_block.number, - &self.src_manifest_idx_and_name, - &self.dst_manifest_idx_and_name, + &src_manifest_idx_and_name, + &dst_manifest_idx_and_name, ) })?; } @@ -723,7 +727,6 @@ impl Connection { // Create indexes for all the attributes that were postponed at the start of // the copy/graft operations. // First recreate the indexes that existed in the original subgraph. - let conn = self.conn.deref_mut(); for table in state.tables.iter() { let arr = index_list.indexes_for_table( &self.dst.site.namespace, @@ -736,7 +739,7 @@ impl Connection { for (_, sql) in arr { let query = sql_query(format!("{};", sql)); - query.execute(conn)?; + self.transaction(|conn| query.execute(conn).map_err(StoreError::from))?; } } @@ -755,7 +758,7 @@ impl Connection { .into_iter() { let query = sql_query(sql); - query.execute(conn)?; + self.transaction(|conn| query.execute(conn).map_err(StoreError::from))?; } } @@ -876,9 +879,10 @@ impl Connection { &self.logger, "Obtaining copy lock (this might take a long time if another process is still copying)" ); - advisory_lock::lock_copying(&mut self.conn, self.dst.site.as_ref())?; + let dst_site = self.dst.site.cheap_clone(); + self.transaction(|conn| advisory_lock::lock_copying(conn, &dst_site))?; let res = self.copy_data_internal(index_list).await; - advisory_lock::unlock_copying(&mut self.conn, self.dst.site.as_ref())?; + self.transaction(|conn| advisory_lock::unlock_copying(conn, &dst_site))?; if matches!(res, Ok(Status::Cancelled)) { warn!(&self.logger, "Copying was cancelled and is incomplete"); } From 6c8016aeafac4ef3f081b8b9696c4f8d1cda7771 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Fri, 28 Mar 2025 13:02:42 -0700 Subject: [PATCH 098/150] store: Allow running copy_table without holding a reference to self --- store/postgres/src/copy.rs | 32 +++++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/store/postgres/src/copy.rs b/store/postgres/src/copy.rs index 3bcfd6ecb03..ea954af2cf8 100644 --- a/store/postgres/src/copy.rs +++ b/store/postgres/src/copy.rs @@ -56,6 +56,8 @@ const ACCEPTABLE_REPLICATION_LAG: Duration = Duration::from_secs(30); /// the lag again const REPLICATION_SLEEP: Duration = Duration::from_secs(10); +type PooledPgConnection = PooledConnection>; + lazy_static! { static ref STATEMENT_TIMEOUT: Option = ENV_VARS .store @@ -620,7 +622,17 @@ pub struct Connection { /// The connection pool for the shard that will contain the destination /// of the copy logger: Logger, - conn: PooledConnection>, + /// We always have one database connection to make sure that copy jobs, + /// once started, can eventually finished so that we don't have + /// different copy jobs that are all half done and have to wait for + /// other jobs to finish + /// + /// This is an `Option` because we need to take this connection out of + /// `self` at some point to spawn a background task to copy an + /// individual table. Except for that case, this will always be + /// `Some(..)`. Most code shouldn't access `self.conn` directly, but use + /// `self.transaction` + conn: Option, src: Arc, dst: Arc, target_block: BlockPtr, @@ -662,6 +674,7 @@ impl Connection { } false })?; + let conn = Some(conn); let src_manifest_idx_and_name = Arc::new(src_manifest_idx_and_name); let dst_manifest_idx_and_name = Arc::new(dst_manifest_idx_and_name); Ok(Self { @@ -679,7 +692,12 @@ impl Connection { where F: FnOnce(&mut PgConnection) -> Result, { - self.conn.transaction(|conn| f(conn)) + let Some(conn) = self.conn.as_mut() else { + return Err(constraint_violation!( + "copy connection has been handed to background task but not returned yet" + )); + }; + conn.transaction(|conn| f(conn)) } /// Copy private data sources if the source uses a schema version that @@ -716,7 +734,15 @@ impl Connection { progress.start(); for table in state.tables.iter_mut().filter(|table| !table.finished()) { - match Self::copy_table(&mut self.conn, logger, &mut progress, table)? { + // Take self.conn to decouple it from self, copy the table and + // put the connection back + let mut conn = self.conn.take().ok_or_else(|| { + constraint_violation!("copy connection is not where it is supposed to be") + })?; + let res = Self::copy_table(&mut conn, logger, &mut progress, table); + self.conn = Some(conn); + + match res? { Status::Finished => { /* Move on to the next table */ } Status::Cancelled => { return Ok(Status::Cancelled); From 2665dca412d7856310b6ff633a4bd57b7ec19520 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Fri, 28 Mar 2025 13:12:28 -0700 Subject: [PATCH 099/150] store: Remove lifetime from CopyProgress --- store/postgres/src/copy.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/store/postgres/src/copy.rs b/store/postgres/src/copy.rs index ea954af2cf8..8ff63bdf42f 100644 --- a/store/postgres/src/copy.rs +++ b/store/postgres/src/copy.rs @@ -538,8 +538,8 @@ impl TableState { } // A helper for logging progress while data is being copied -struct CopyProgress<'a> { - logger: &'a Logger, +struct CopyProgress { + logger: Logger, last_log: Instant, src: Arc, dst: Arc, @@ -547,8 +547,8 @@ struct CopyProgress<'a> { target_vid: i64, } -impl<'a> CopyProgress<'a> { - fn new(logger: &'a Logger, state: &CopyState) -> Self { +impl CopyProgress { + fn new(logger: Logger, state: &CopyState) -> Self { let target_vid: i64 = state .tables .iter() @@ -730,7 +730,7 @@ impl Connection { let mut state = self.transaction(|conn| CopyState::new(conn, src, dst, target_block))?; let logger = &self.logger.clone(); - let mut progress = CopyProgress::new(logger, &state); + let mut progress = CopyProgress::new(self.logger.cheap_clone(), &state); progress.start(); for table in state.tables.iter_mut().filter(|table| !table.finished()) { @@ -799,7 +799,7 @@ impl Connection { fn copy_table( conn: &mut PgConnection, logger: &Logger, - progress: &mut CopyProgress<'_>, + progress: &mut CopyProgress, table: &mut TableState, ) -> Result { use Status::*; From e09f6bb98768b36626dfe55df6d69da7b8507700 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Fri, 28 Mar 2025 13:33:37 -0700 Subject: [PATCH 100/150] store: Change CopyProgress to interior mutability --- store/postgres/src/copy.rs | 45 +++++++++++++++++++++++++++----------- 1 file changed, 32 insertions(+), 13 deletions(-) diff --git a/store/postgres/src/copy.rs b/store/postgres/src/copy.rs index 8ff63bdf42f..88b79e162b7 100644 --- a/store/postgres/src/copy.rs +++ b/store/postgres/src/copy.rs @@ -13,7 +13,10 @@ //! `graph-node` was restarted while the copy was running. use std::{ convert::TryFrom, - sync::Arc, + sync::{ + atomic::{AtomicI64, Ordering}, + Arc, Mutex, + }, time::{Duration, Instant}, }; @@ -540,10 +543,11 @@ impl TableState { // A helper for logging progress while data is being copied struct CopyProgress { logger: Logger, - last_log: Instant, + last_log: Arc>, src: Arc, dst: Arc, - current_vid: i64, + /// The sum of all `target_vid` of tables that have finished + current_vid: AtomicI64, target_vid: i64, } @@ -559,9 +563,10 @@ impl CopyProgress { .iter() .map(|table| table.batcher.next_vid()) .sum(); + let current_vid = AtomicI64::new(current_vid); Self { logger, - last_log: Instant::now(), + last_log: Arc::new(Mutex::new(Instant::now())), src: state.src.site.clone(), dst: state.dst.site.clone(), current_vid, @@ -590,8 +595,21 @@ impl CopyProgress { } } - fn update(&mut self, entity_type: &EntityType, batcher: &VidBatcher) { - if self.last_log.elapsed() > LOG_INTERVAL { + fn update(&self, entity_type: &EntityType, batcher: &VidBatcher) { + let mut last_log = self.last_log.lock().unwrap_or_else(|err| { + // Better to clear the poison error and skip a log message than + // crash for no important reason + warn!( + self.logger, + "Lock for progress locking was poisoned, skipping a log message" + ); + let mut last_log = err.into_inner(); + *last_log = Instant::now(); + self.last_log.clear_poison(); + last_log + }); + if last_log.elapsed() > LOG_INTERVAL { + let total_current_vid = self.current_vid.load(Ordering::SeqCst) + batcher.next_vid(); info!( self.logger, "Copied {:.2}% of `{}` entities ({}/{} entity versions), {:.2}% of overall data", @@ -599,14 +617,15 @@ impl CopyProgress { entity_type, batcher.next_vid(), batcher.target_vid(), - Self::progress_pct(self.current_vid + batcher.next_vid(), self.target_vid) + Self::progress_pct(total_current_vid, self.target_vid) ); - self.last_log = Instant::now(); + *last_log = Instant::now(); } } - fn table_finished(&mut self, batcher: &VidBatcher) { - self.current_vid += batcher.next_vid(); + fn table_finished(&self, batcher: &VidBatcher) { + self.current_vid + .fetch_add(batcher.next_vid(), Ordering::SeqCst); } fn finished(&self) { @@ -730,7 +749,7 @@ impl Connection { let mut state = self.transaction(|conn| CopyState::new(conn, src, dst, target_block))?; let logger = &self.logger.clone(); - let mut progress = CopyProgress::new(self.logger.cheap_clone(), &state); + let progress = Arc::new(CopyProgress::new(self.logger.cheap_clone(), &state)); progress.start(); for table in state.tables.iter_mut().filter(|table| !table.finished()) { @@ -739,7 +758,7 @@ impl Connection { let mut conn = self.conn.take().ok_or_else(|| { constraint_violation!("copy connection is not where it is supposed to be") })?; - let res = Self::copy_table(&mut conn, logger, &mut progress, table); + let res = Self::copy_table(&mut conn, logger, progress.cheap_clone(), table); self.conn = Some(conn); match res? { @@ -799,7 +818,7 @@ impl Connection { fn copy_table( conn: &mut PgConnection, logger: &Logger, - progress: &mut CopyProgress, + progress: Arc, table: &mut TableState, ) -> Result { use Status::*; From 8d2697f6fd9e748ad138af5b31ba95d4a4828a75 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Fri, 28 Mar 2025 15:00:51 -0700 Subject: [PATCH 101/150] store: Change how we iterate over all unfinished tables --- store/postgres/src/copy.rs | 39 +++++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/store/postgres/src/copy.rs b/store/postgres/src/copy.rs index 88b79e162b7..a7b683b57a1 100644 --- a/store/postgres/src/copy.rs +++ b/store/postgres/src/copy.rs @@ -132,7 +132,8 @@ struct CopyState { src: Arc, dst: Arc, target_block: BlockPtr, - tables: Vec, + finished: Vec, + unfinished: Vec, } impl CopyState { @@ -191,11 +192,13 @@ impl CopyState { target_block: BlockPtr, ) -> Result { let tables = TableState::load(conn, src.as_ref(), dst.as_ref())?; + let (finished, unfinished) = tables.into_iter().partition(|table| table.finished()); Ok(CopyState { src, dst, target_block, - tables, + finished, + unfinished, }) } @@ -217,7 +220,7 @@ impl CopyState { )) .execute(conn)?; - let mut tables: Vec<_> = dst + let mut unfinished: Vec<_> = dst .tables .values() .filter_map(|dst_table| { @@ -235,9 +238,9 @@ impl CopyState { }) }) .collect::>()?; - tables.sort_by_key(|table| table.dst.object.to_string()); + unfinished.sort_by_key(|table| table.dst.object.to_string()); - let values = tables + let values = unfinished .iter() .map(|table| { ( @@ -255,7 +258,8 @@ impl CopyState { src, dst, target_block, - tables, + finished: Vec::new(), + unfinished, }) } @@ -299,6 +303,10 @@ impl CopyState { } Ok(()) } + + fn all_tables(&self) -> impl Iterator { + self.finished.iter().chain(self.unfinished.iter()) + } } pub(crate) fn source( @@ -554,13 +562,12 @@ struct CopyProgress { impl CopyProgress { fn new(logger: Logger, state: &CopyState) -> Self { let target_vid: i64 = state - .tables - .iter() + .all_tables() .map(|table| table.batcher.target_vid()) .sum(); let current_vid = state - .tables - .iter() + .all_tables() + .filter(|table| table.finished()) .map(|table| table.batcher.next_vid()) .sum(); let current_vid = AtomicI64::new(current_vid); @@ -752,17 +759,19 @@ impl Connection { let progress = Arc::new(CopyProgress::new(self.logger.cheap_clone(), &state)); progress.start(); - for table in state.tables.iter_mut().filter(|table| !table.finished()) { + while let Some(mut table) = state.unfinished.pop() { // Take self.conn to decouple it from self, copy the table and // put the connection back let mut conn = self.conn.take().ok_or_else(|| { constraint_violation!("copy connection is not where it is supposed to be") })?; - let res = Self::copy_table(&mut conn, logger, progress.cheap_clone(), table); + let res = Self::copy_table(&mut conn, logger, progress.cheap_clone(), &mut table); self.conn = Some(conn); match res? { - Status::Finished => { /* Move on to the next table */ } + Status::Finished => { + state.finished.push(table); + } Status::Cancelled => { return Ok(Status::Cancelled); } @@ -772,7 +781,7 @@ impl Connection { // Create indexes for all the attributes that were postponed at the start of // the copy/graft operations. // First recreate the indexes that existed in the original subgraph. - for table in state.tables.iter() { + for table in state.all_tables() { let arr = index_list.indexes_for_table( &self.dst.site.namespace, &table.src.name.to_string(), @@ -790,7 +799,7 @@ impl Connection { // Second create the indexes for the new fields. // Here we need to skip those created in the first step for the old fields. - for table in state.tables.iter() { + for table in state.all_tables() { let orig_colums = table .src .columns From 01a7eccb88399b0e8841b4e825d57e31e82e9896 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Fri, 28 Mar 2025 15:35:56 -0700 Subject: [PATCH 102/150] store: Rewrite copy loop so that we could run parallel copies For now, we still only copy one table at a time, but the code is closer to what we need to copy multiple tables concurrently --- store/postgres/src/copy.rs | 235 ++++++++++++++++++++++--------------- 1 file changed, 142 insertions(+), 93 deletions(-) diff --git a/store/postgres/src/copy.rs b/store/postgres/src/copy.rs index a7b683b57a1..74f80252c13 100644 --- a/store/postgres/src/copy.rs +++ b/store/postgres/src/copy.rs @@ -14,7 +14,7 @@ use std::{ convert::TryFrom, sync::{ - atomic::{AtomicI64, Ordering}, + atomic::{AtomicBool, AtomicI64, Ordering}, Arc, Mutex, }, time::{Duration, Instant}, @@ -30,6 +30,7 @@ use diesel::{ }; use graph::{ constraint_violation, + futures03::future::select_all, prelude::{ info, lazy_static, o, warn, BlockNumber, BlockPtr, CheapClone, Logger, StoreError, ENV_VARS, }, @@ -548,7 +549,8 @@ impl TableState { } } -// A helper for logging progress while data is being copied +// A helper for logging progress while data is being copied and +// communicating across all copy workers struct CopyProgress { logger: Logger, last_log: Arc>, @@ -557,6 +559,7 @@ struct CopyProgress { /// The sum of all `target_vid` of tables that have finished current_vid: AtomicI64, target_vid: i64, + cancelled: AtomicBool, } impl CopyProgress { @@ -578,6 +581,7 @@ impl CopyProgress { dst: state.dst.site.clone(), current_vid, target_vid, + cancelled: AtomicBool::new(false), } } @@ -641,6 +645,120 @@ impl CopyProgress { "Finished copying data into {}[{}]", self.dst.deployment, self.dst.namespace ); } + + fn cancel(&self) { + self.cancelled.store(true, Ordering::SeqCst); + } + + fn is_cancelled(&self) -> bool { + self.cancelled.load(Ordering::SeqCst) + } +} + +/// A helper to run copying of one table. We need to thread `conn` and +/// `table` from the control loop to the background worker and back again to +/// the control loop. This worker facilitates that +struct CopyTableWorker { + conn: PooledPgConnection, + table: TableState, + result: Result, +} + +impl CopyTableWorker { + fn new(conn: PooledPgConnection, table: TableState) -> Self { + Self { + conn, + table, + result: Ok(Status::Cancelled), + } + } + + async fn run(mut self, logger: Logger, progress: Arc) -> Self { + self.result = self.run_inner(logger, &progress); + self + } + + fn run_inner(&mut self, logger: Logger, progress: &CopyProgress) -> Result { + use Status::*; + + let conn = &mut self.conn; + while !self.table.finished() { + // It is important that this check happens outside the write + // transaction so that we do not hold on to locks acquired + // by the check + if self.table.is_cancelled(conn)? || progress.is_cancelled() { + progress.cancel(); + return Ok(Cancelled); + } + + // Pause copying if replication is lagging behind to avoid + // overloading replicas + let mut lag = catalog::replication_lag(conn)?; + if lag > MAX_REPLICATION_LAG { + loop { + info!(logger, + "Replicas are lagging too much; pausing copying for {}s to allow them to catch up", + REPLICATION_SLEEP.as_secs(); + "lag_s" => lag.as_secs()); + std::thread::sleep(REPLICATION_SLEEP); + lag = catalog::replication_lag(conn)?; + if lag <= ACCEPTABLE_REPLICATION_LAG { + break; + } + } + } + + let status = { + loop { + if progress.is_cancelled() { + break Cancelled; + } + + match conn.transaction(|conn| { + if let Some(timeout) = STATEMENT_TIMEOUT.as_ref() { + conn.batch_execute(timeout)?; + } + self.table.copy_batch(conn) + }) { + Ok(status) => { + break status; + } + Err(StoreError::StatementTimeout) => { + warn!( + logger, + "Current batch took longer than GRAPH_STORE_BATCH_TIMEOUT seconds. Retrying with a smaller batch size." + ); + } + Err(e) => { + return Err(e); + } + } + // We hit a timeout. Reset the batch size to 1. + // That's small enough that we will make _some_ + // progress, assuming the timeout is set to a + // reasonable value (several minutes) + // + // Our estimation of batch sizes is generally good + // and stays within the prescribed bounds, but there + // are cases where proper estimation of the batch + // size is nearly impossible since the size of the + // rows in the table jumps sharply at some point + // that is hard to predict. This mechanism ensures + // that if our estimation is wrong, the consequences + // aren't too severe. + conn.transaction(|conn| self.table.set_batch_size(conn, 1))?; + } + }; + + if status == Cancelled { + progress.cancel(); + return Ok(Cancelled); + } + progress.update(&self.table.dst.object, &self.table.batcher); + } + progress.table_finished(&self.table.batcher); + Ok(Finished) + } } /// A helper for copying subgraphs @@ -759,22 +877,33 @@ impl Connection { let progress = Arc::new(CopyProgress::new(self.logger.cheap_clone(), &state)); progress.start(); - while let Some(mut table) = state.unfinished.pop() { + let mut workers = Vec::new(); + while let Some(table) = state.unfinished.pop() { // Take self.conn to decouple it from self, copy the table and // put the connection back - let mut conn = self.conn.take().ok_or_else(|| { + let conn = self.conn.take().ok_or_else(|| { constraint_violation!("copy connection is not where it is supposed to be") })?; - let res = Self::copy_table(&mut conn, logger, progress.cheap_clone(), &mut table); - self.conn = Some(conn); - match res? { - Status::Finished => { - state.finished.push(table); - } - Status::Cancelled => { - return Ok(Status::Cancelled); - } + let worker = CopyTableWorker::new(conn, table); + let fut = Box::pin(worker.run(logger.cheap_clone(), progress.cheap_clone())); + + workers.push(fut); + let (worker, _idx, remaining) = select_all(workers).await; + workers = remaining; + + // Put the connection back into self.conn so that we can use it + // in the next iteration + self.conn = Some(worker.conn); + state.finished.push(worker.table); + + if worker.result.is_err() { + progress.cancel(); + return worker.result; + } + + if progress.is_cancelled() { + return Ok(Status::Cancelled); } } @@ -824,86 +953,6 @@ impl Connection { Ok(Status::Finished) } - fn copy_table( - conn: &mut PgConnection, - logger: &Logger, - progress: Arc, - table: &mut TableState, - ) -> Result { - use Status::*; - - while !table.finished() { - // It is important that this check happens outside the write - // transaction so that we do not hold on to locks acquired - // by the check - if table.is_cancelled(conn)? { - return Ok(Cancelled); - } - - // Pause copying if replication is lagging behind to avoid - // overloading replicas - let mut lag = catalog::replication_lag(conn)?; - if lag > MAX_REPLICATION_LAG { - loop { - info!(logger, - "Replicas are lagging too much; pausing copying for {}s to allow them to catch up", - REPLICATION_SLEEP.as_secs(); - "lag_s" => lag.as_secs()); - std::thread::sleep(REPLICATION_SLEEP); - lag = catalog::replication_lag(conn)?; - if lag <= ACCEPTABLE_REPLICATION_LAG { - break; - } - } - } - - let status = { - loop { - match conn.transaction(|conn| { - if let Some(timeout) = STATEMENT_TIMEOUT.as_ref() { - conn.batch_execute(timeout)?; - } - table.copy_batch(conn) - }) { - Ok(status) => { - break status; - } - Err(StoreError::StatementTimeout) => { - warn!( - logger, - "Current batch took longer than GRAPH_STORE_BATCH_TIMEOUT seconds. Retrying with a smaller batch size." - ); - } - Err(e) => { - return Err(e); - } - } - // We hit a timeout. Reset the batch size to 1. - // That's small enough that we will make _some_ - // progress, assuming the timeout is set to a - // reasonable value (several minutes) - // - // Our estimation of batch sizes is generally good - // and stays within the prescribed bounds, but there - // are cases where proper estimation of the batch - // size is nearly impossible since the size of the - // rows in the table jumps sharply at some point - // that is hard to predict. This mechanism ensures - // that if our estimation is wrong, the consequences - // aren't too severe. - conn.transaction(|conn| table.set_batch_size(conn, 1))?; - } - }; - - if status == Cancelled { - return Ok(Cancelled); - } - progress.update(&table.dst.object, &table.batcher); - } - progress.table_finished(&table.batcher); - Ok(Finished) - } - /// Copy the data for the subgraph `src` to the subgraph `dst`. The /// schema for both subgraphs must have already been set up. The /// `target_block` must be far enough behind the chain head so that the From 3828ed710c42cbaadf4b7add3ff268acaa6a368f Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Fri, 28 Mar 2025 16:13:17 -0700 Subject: [PATCH 103/150] store: Factor creating a worker into a helper --- store/postgres/src/copy.rs | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/store/postgres/src/copy.rs b/store/postgres/src/copy.rs index 74f80252c13..e022366a091 100644 --- a/store/postgres/src/copy.rs +++ b/store/postgres/src/copy.rs @@ -13,6 +13,8 @@ //! `graph-node` was restarted while the copy was running. use std::{ convert::TryFrom, + future::Future, + pin::Pin, sync::{ atomic::{AtomicBool, AtomicI64, Ordering}, Arc, Mutex, @@ -864,6 +866,20 @@ impl Connection { Ok(()) } + fn default_worker( + &mut self, + state: &mut CopyState, + progress: Arc, + ) -> Option>>> { + let conn = self.conn.take()?; + let table = state.unfinished.pop()?; + + let worker = CopyTableWorker::new(conn, table); + Some(Box::pin( + worker.run(self.logger.cheap_clone(), progress.cheap_clone()), + )) + } + pub async fn copy_data_internal( &mut self, index_list: IndexList, @@ -873,22 +889,15 @@ impl Connection { let target_block = self.target_block.clone(); let mut state = self.transaction(|conn| CopyState::new(conn, src, dst, target_block))?; - let logger = &self.logger.clone(); let progress = Arc::new(CopyProgress::new(self.logger.cheap_clone(), &state)); progress.start(); let mut workers = Vec::new(); - while let Some(table) = state.unfinished.pop() { - // Take self.conn to decouple it from self, copy the table and - // put the connection back - let conn = self.conn.take().ok_or_else(|| { - constraint_violation!("copy connection is not where it is supposed to be") - })?; - - let worker = CopyTableWorker::new(conn, table); - let fut = Box::pin(worker.run(logger.cheap_clone(), progress.cheap_clone())); + while !state.unfinished.is_empty() && !workers.is_empty() { + if let Some(worker) = self.default_worker(&mut state, progress.cheap_clone()) { + workers.push(worker); + } - workers.push(fut); let (worker, _idx, remaining) = select_all(workers).await; workers = remaining; From 716138ac2ccd2194bb1ba04cd9c700655c913beb Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Fri, 28 Mar 2025 16:43:06 -0700 Subject: [PATCH 104/150] store: Copy multiple tables in parallel if there are idle connections --- store/postgres/src/connection_pool.rs | 44 +++++++++++--- store/postgres/src/copy.rs | 83 ++++++++++++++++++++++++--- 2 files changed, 109 insertions(+), 18 deletions(-) diff --git a/store/postgres/src/connection_pool.rs b/store/postgres/src/connection_pool.rs index 6267a41628a..f710fd2316d 100644 --- a/store/postgres/src/connection_pool.rs +++ b/store/postgres/src/connection_pool.rs @@ -524,6 +524,15 @@ impl ConnectionPool { self.get_ready()?.get_fdw(logger, timeout) } + /// Get a connection from the pool for foreign data wrapper access if + /// one is available + pub fn try_get_fdw( + &self, + logger: &Logger, + ) -> Result>>, StoreError> { + self.get_ready()?.try_get_fdw(logger) + } + pub fn connection_detail(&self) -> Result { let pool = self.get_ready()?; ForeignServer::new(pool.shard.clone(), &pool.postgres_url).map_err(|e| e.into()) @@ -980,6 +989,23 @@ impl PoolInner { } } + /// Get the pool for fdw connections. It is an error if none is configured + fn fdw_pool( + &self, + logger: &Logger, + ) -> Result<&Pool>, StoreError> { + let pool = match &self.fdw_pool { + Some(pool) => pool, + None => { + const MSG: &str = + "internal error: trying to get fdw connection on a pool that doesn't have any"; + error!(logger, "{}", MSG); + return Err(constraint_violation!(MSG)); + } + }; + Ok(pool) + } + /// Get a connection from the pool for foreign data wrapper access; /// since that pool can be very contended, periodically log that we are /// still waiting for a connection @@ -995,15 +1021,7 @@ impl PoolInner { where F: FnMut() -> bool, { - let pool = match &self.fdw_pool { - Some(pool) => pool, - None => { - const MSG: &str = - "internal error: trying to get fdw connection on a pool that doesn't have any"; - error!(logger, "{}", MSG); - return Err(constraint_violation!(MSG)); - } - }; + let pool = self.fdw_pool(logger)?; loop { match pool.get() { Ok(conn) => return Ok(conn), @@ -1016,6 +1034,14 @@ impl PoolInner { } } + /// Get a connection from the fdw pool if one is available + pub fn try_get_fdw( + &self, + logger: &Logger, + ) -> Result>>, StoreError> { + Ok(self.fdw_pool(logger)?.try_get()) + } + pub fn connection_detail(&self) -> Result { ForeignServer::new(self.shard.clone(), &self.postgres_url).map_err(|e| e.into()) } diff --git a/store/postgres/src/copy.rs b/store/postgres/src/copy.rs index e022366a091..c8079adc52e 100644 --- a/store/postgres/src/copy.rs +++ b/store/postgres/src/copy.rs @@ -598,6 +598,16 @@ impl CopyProgress { ); } + fn start_table(&self, table: &TableState) { + info!( + self.logger, + "Starting to copy `{}` entities from {} to {}", + table.dst.object, + table.src.qualified_name, + table.dst.qualified_name + ); + } + fn progress_pct(current_vid: i64, target_vid: i64) -> f64 { // When a step is done, current_vid == target_vid + 1; don't report // more than 100% completion @@ -710,6 +720,7 @@ impl CopyTableWorker { } } + progress.start_table(&self.table); let status = { loop { if progress.is_cancelled() { @@ -779,6 +790,8 @@ pub struct Connection { /// `Some(..)`. Most code shouldn't access `self.conn` directly, but use /// `self.transaction` conn: Option, + pool: ConnectionPool, + workers: usize, src: Arc, dst: Arc, target_block: BlockPtr, @@ -826,6 +839,8 @@ impl Connection { Ok(Self { logger, conn, + pool, + workers: 5, src, dst, target_block, @@ -866,18 +881,50 @@ impl Connection { Ok(()) } + /// Create a worker using the connection in `self.conn`. This may return + /// `None` if there are no more tables that need to be copied. It is an + /// error to call this if `self.conn` is `None` fn default_worker( &mut self, state: &mut CopyState, - progress: Arc, - ) -> Option>>> { - let conn = self.conn.take()?; - let table = state.unfinished.pop()?; + progress: &Arc, + ) -> Result>>>, StoreError> { + let conn = self.conn.take().ok_or_else(|| { + constraint_violation!( + "copy connection has been handed to background task but not returned yet" + ) + })?; + let Some(table) = state.unfinished.pop() else { + return Ok(None); + }; + + let worker = CopyTableWorker::new(conn, table); + Ok(Some(Box::pin( + worker.run(self.logger.cheap_clone(), progress.cheap_clone()), + ))) + } + + /// Opportunistically create an extra worker if we have more tables to + /// copy and there are idle fdw connections. If there are no more tables + /// or no idle connections, this will return `None`. + fn extra_worker( + &mut self, + state: &mut CopyState, + progress: &Arc, + ) -> Result>>>, StoreError> { + // It's important that we get the connection before the table since + // we remove the table from the state and could drop it otherwise + let Some(conn) = self.pool.try_get_fdw(&self.logger)? else { + return Ok(None); + }; + let Some(table) = state.unfinished.pop() else { + return Ok(None); + }; let worker = CopyTableWorker::new(conn, table); - Some(Box::pin( + Ok(Some(Box::pin( worker.run(self.logger.cheap_clone(), progress.cheap_clone()), - )) + ))) } pub async fn copy_data_internal( @@ -892,17 +939,34 @@ impl Connection { let progress = Arc::new(CopyProgress::new(self.logger.cheap_clone(), &state)); progress.start(); + // Run as many copy jobs as we can in parallel, up to `self.workers` + // many. We can always start at least one worker because of the + // connection in `self.conn`. If the fdw pool has idle connections + // and there are more tables to be copied, we can start more + // workers, up to `self.workers` many let mut workers = Vec::new(); while !state.unfinished.is_empty() && !workers.is_empty() { - if let Some(worker) = self.default_worker(&mut state, progress.cheap_clone()) { + // We usually add at least one job here, except if we are out of + // tables to copy. In that case, we go through the `while` loop + // every time one of the tables we are currently copying + // finishes + if let Some(worker) = self.default_worker(&mut state, &progress)? { + workers.push(worker); + } + loop { + if workers.len() >= self.workers { + break; + } + let Some(worker) = self.extra_worker(&mut state, &progress)? else { + break; + }; workers.push(worker); } - let (worker, _idx, remaining) = select_all(workers).await; workers = remaining; // Put the connection back into self.conn so that we can use it - // in the next iteration + // in the next iteration. self.conn = Some(worker.conn); state.finished.push(worker.table); @@ -915,6 +979,7 @@ impl Connection { return Ok(Status::Cancelled); } } + debug_assert!(self.conn.is_some()); // Create indexes for all the attributes that were postponed at the start of // the copy/graft operations. From 9d260fb8bfe5ca6715376c1f03bb6a5c08484fe0 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Fri, 28 Mar 2025 16:48:52 -0700 Subject: [PATCH 105/150] all: Make number of copy workers configurable --- docs/environment-variables.md | 4 ++++ graph/src/env/store.rs | 12 ++++++++++++ store/postgres/src/copy.rs | 4 ++-- 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/docs/environment-variables.md b/docs/environment-variables.md index f174e0e2e54..46903185ccf 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -231,6 +231,10 @@ those. timeout is hit, the batch size is reset to 1 so we can be sure that batches stay below `GRAPH_STORE_BATCH_TARGET_DURATION` and the smaller batch is retried. Value is in seconds and defaults to unlimited. +- `GRAPH_STORE_BATCH_WORKERS`: The number of workers to use for batch + operations. If there are idle connectiosn, each subgraph copy operation + will use up to this many workers to copy tables in parallel. Defaults + to 1 and must be at least 1 - `GRAPH_START_BLOCK`: block hash:block number where the forked subgraph will start indexing at. - `GRAPH_FORK_BASE`: api url for where the graph node will fork from, use `https://api.thegraph.com/subgraphs/id/` for the hosted service. diff --git a/graph/src/env/store.rs b/graph/src/env/store.rs index 4fb30f58079..1bdb3c4d902 100644 --- a/graph/src/env/store.rs +++ b/graph/src/env/store.rs @@ -85,6 +85,12 @@ pub struct EnvVarsStore { /// this. Set by `GRAPH_STORE_BATCH_TIMEOUT`. Unlimited by default pub batch_timeout: Option, + /// The number of workers to use for batch operations. If there are idle + /// connectiosn, each subgraph copy operation will use up to this many + /// workers to copy tables in parallel. Defaults to 1 and must be at + /// least 1 + pub batch_workers: usize, + /// Prune tables where we will remove at least this fraction of entity /// versions by rebuilding the table. Set by /// `GRAPH_STORE_HISTORY_REBUILD_THRESHOLD`. The default is 0.5 @@ -175,6 +181,7 @@ impl TryFrom for EnvVarsStore { write_queue_size: x.write_queue_size, batch_target_duration: Duration::from_secs(x.batch_target_duration_in_secs), batch_timeout: x.batch_timeout_in_secs.map(Duration::from_secs), + batch_workers: x.batch_workers, rebuild_threshold: x.rebuild_threshold.0, delete_threshold: x.delete_threshold.0, history_slack_factor: x.history_slack_factor.0, @@ -194,6 +201,9 @@ impl TryFrom for EnvVarsStore { ); } } + if vars.batch_workers < 1 { + bail!("GRAPH_STORE_BATCH_WORKERS must be at least 1"); + } Ok(vars) } } @@ -239,6 +249,8 @@ pub struct InnerStore { batch_target_duration_in_secs: u64, #[envconfig(from = "GRAPH_STORE_BATCH_TIMEOUT")] batch_timeout_in_secs: Option, + #[envconfig(from = "GRAPH_STORE_BATCH_WORKERS", default = "1")] + batch_workers: usize, #[envconfig(from = "GRAPH_STORE_HISTORY_REBUILD_THRESHOLD", default = "0.5")] rebuild_threshold: ZeroToOneF64, #[envconfig(from = "GRAPH_STORE_HISTORY_DELETE_THRESHOLD", default = "0.05")] diff --git a/store/postgres/src/copy.rs b/store/postgres/src/copy.rs index c8079adc52e..4dbc312bc9d 100644 --- a/store/postgres/src/copy.rs +++ b/store/postgres/src/copy.rs @@ -840,7 +840,7 @@ impl Connection { logger, conn, pool, - workers: 5, + workers: ENV_VARS.store.batch_workers, src, dst, target_block, @@ -945,7 +945,7 @@ impl Connection { // and there are more tables to be copied, we can start more // workers, up to `self.workers` many let mut workers = Vec::new(); - while !state.unfinished.is_empty() && !workers.is_empty() { + while !state.unfinished.is_empty() || !workers.is_empty() { // We usually add at least one job here, except if we are out of // tables to copy. In that case, we go through the `while` loop // every time one of the tables we are currently copying From b3543bbb3d0f65aadb6f8b4d13ed7458ee8bbf0b Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Mon, 31 Mar 2025 07:59:23 -0700 Subject: [PATCH 106/150] graph: Fix typo in comment --- graph/src/env/store.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/graph/src/env/store.rs b/graph/src/env/store.rs index 1bdb3c4d902..ded0be9f144 100644 --- a/graph/src/env/store.rs +++ b/graph/src/env/store.rs @@ -86,7 +86,7 @@ pub struct EnvVarsStore { pub batch_timeout: Option, /// The number of workers to use for batch operations. If there are idle - /// connectiosn, each subgraph copy operation will use up to this many + /// connections, each subgraph copy operation will use up to this many /// workers to copy tables in parallel. Defaults to 1 and must be at /// least 1 pub batch_workers: usize, From 5d75dc6bdca05bb1fc823d7dcbef316a1da549fb Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Mon, 31 Mar 2025 18:58:29 -0700 Subject: [PATCH 107/150] store: Log 'starting to copy table' only once --- store/postgres/src/copy.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/store/postgres/src/copy.rs b/store/postgres/src/copy.rs index 4dbc312bc9d..55fed5cde2f 100644 --- a/store/postgres/src/copy.rs +++ b/store/postgres/src/copy.rs @@ -694,6 +694,7 @@ impl CopyTableWorker { use Status::*; let conn = &mut self.conn; + progress.start_table(&self.table); while !self.table.finished() { // It is important that this check happens outside the write // transaction so that we do not hold on to locks acquired @@ -720,7 +721,6 @@ impl CopyTableWorker { } } - progress.start_table(&self.table); let status = { loop { if progress.is_cancelled() { From 3369381b47de210270cb4041575fbd64222efacb Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Mon, 31 Mar 2025 19:29:12 -0700 Subject: [PATCH 108/150] graph, store: Wait for a little bit for an addl fdw connection If we don't wait at all, we can use fewer connections than are available since the pool might be below its capacity but has no idle connections open currently --- graph/src/env/store.rs | 9 +++++++++ store/postgres/src/connection_pool.rs | 17 ++++++++++++++--- store/postgres/src/copy.rs | 5 ++++- 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/graph/src/env/store.rs b/graph/src/env/store.rs index ded0be9f144..661d0356446 100644 --- a/graph/src/env/store.rs +++ b/graph/src/env/store.rs @@ -91,6 +91,12 @@ pub struct EnvVarsStore { /// least 1 pub batch_workers: usize, + /// How long to wait to get an additional connection for a batch worker. + /// This should just be big enough to allow the connection pool to + /// establish a connection. Set by `GRAPH_STORE_BATCH_WORKER_WAIT`. + /// Value is in ms and defaults to 2000ms + pub batch_worker_wait: Duration, + /// Prune tables where we will remove at least this fraction of entity /// versions by rebuilding the table. Set by /// `GRAPH_STORE_HISTORY_REBUILD_THRESHOLD`. The default is 0.5 @@ -182,6 +188,7 @@ impl TryFrom for EnvVarsStore { batch_target_duration: Duration::from_secs(x.batch_target_duration_in_secs), batch_timeout: x.batch_timeout_in_secs.map(Duration::from_secs), batch_workers: x.batch_workers, + batch_worker_wait: Duration::from_millis(x.batch_worker_wait), rebuild_threshold: x.rebuild_threshold.0, delete_threshold: x.delete_threshold.0, history_slack_factor: x.history_slack_factor.0, @@ -251,6 +258,8 @@ pub struct InnerStore { batch_timeout_in_secs: Option, #[envconfig(from = "GRAPH_STORE_BATCH_WORKERS", default = "1")] batch_workers: usize, + #[envconfig(from = "GRAPH_STORE_BATCH_WORKER_WAIT", default = "2000")] + batch_worker_wait: u64, #[envconfig(from = "GRAPH_STORE_HISTORY_REBUILD_THRESHOLD", default = "0.5")] rebuild_threshold: ZeroToOneF64, #[envconfig(from = "GRAPH_STORE_HISTORY_DELETE_THRESHOLD", default = "0.05")] diff --git a/store/postgres/src/connection_pool.rs b/store/postgres/src/connection_pool.rs index f710fd2316d..782c2a57489 100644 --- a/store/postgres/src/connection_pool.rs +++ b/store/postgres/src/connection_pool.rs @@ -529,8 +529,9 @@ impl ConnectionPool { pub fn try_get_fdw( &self, logger: &Logger, + timeout: Duration, ) -> Result>>, StoreError> { - self.get_ready()?.try_get_fdw(logger) + self.get_ready()?.try_get_fdw(logger, timeout) } pub fn connection_detail(&self) -> Result { @@ -1034,12 +1035,22 @@ impl PoolInner { } } - /// Get a connection from the fdw pool if one is available + /// Get a connection from the fdw pool if one is available. We wait for + /// `timeout` for a connection which should be set just big enough to + /// allow establishing a connection pub fn try_get_fdw( &self, logger: &Logger, + timeout: Duration, ) -> Result>>, StoreError> { - Ok(self.fdw_pool(logger)?.try_get()) + // Any error trying to get a connection is treated as "couldn't get + // a connection in time". If there is a serious error with the + // database, e.g., because it's not available, the next database + // operation will run into it and report it. + self.fdw_pool(logger)? + .get_timeout(timeout) + .map(|conn| Some(conn)) + .or_else(|_| Ok(None)) } pub fn connection_detail(&self) -> Result { diff --git a/store/postgres/src/copy.rs b/store/postgres/src/copy.rs index 55fed5cde2f..6e21b019f3d 100644 --- a/store/postgres/src/copy.rs +++ b/store/postgres/src/copy.rs @@ -914,7 +914,10 @@ impl Connection { ) -> Result>>>, StoreError> { // It's important that we get the connection before the table since // we remove the table from the state and could drop it otherwise - let Some(conn) = self.pool.try_get_fdw(&self.logger)? else { + let Some(conn) = self + .pool + .try_get_fdw(&self.logger, ENV_VARS.store.batch_worker_wait)? + else { return Ok(None); }; let Some(table) = state.unfinished.pop() else { From 5d7e4cebc46488e7b8faa3d92f33545d3c707e7b Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Mon, 31 Mar 2025 19:32:48 -0700 Subject: [PATCH 109/150] store: Sort unfinished tables in CopyState That makes the order in which tables are copied predictable --- store/postgres/src/copy.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/store/postgres/src/copy.rs b/store/postgres/src/copy.rs index 6e21b019f3d..25792bba04e 100644 --- a/store/postgres/src/copy.rs +++ b/store/postgres/src/copy.rs @@ -195,7 +195,9 @@ impl CopyState { target_block: BlockPtr, ) -> Result { let tables = TableState::load(conn, src.as_ref(), dst.as_ref())?; - let (finished, unfinished) = tables.into_iter().partition(|table| table.finished()); + let (finished, mut unfinished): (Vec<_>, Vec<_>) = + tables.into_iter().partition(|table| table.finished()); + unfinished.sort_by_key(|table| table.dst.object.to_string()); Ok(CopyState { src, dst, From 36ad6a24a2456fa5504f14b20dd59c0cffae2b40 Mon Sep 17 00:00:00 2001 From: Filipe Azevedo Date: Tue, 1 Apr 2025 20:13:47 +0100 Subject: [PATCH 110/150] fix firehose tls (#5923) --- Cargo.toml | 9 ++++++++- graph/src/firehose/endpoints.rs | 11 ++++++++--- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index a8193c3f0ed..b938992bc30 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -47,7 +47,14 @@ chrono = "0.4.38" bs58 = "0.5.1" clap = { version = "4.5.4", features = ["derive", "env"] } derivative = "2.2.0" -diesel = { version = "2.2.7", features = ["postgres", "serde_json", "numeric", "r2d2", "chrono", "i-implement-a-third-party-backend-and-opt-into-breaking-changes"] } +diesel = { version = "2.2.7", features = [ + "postgres", + "serde_json", + "numeric", + "r2d2", + "chrono", + "i-implement-a-third-party-backend-and-opt-into-breaking-changes", +] } diesel-derive-enum = { version = "2.1.0", features = ["postgres"] } diesel-dynamic-schema = { version = "0.2.3", features = ["postgres"] } diesel_derives = "2.2.3" diff --git a/graph/src/firehose/endpoints.rs b/graph/src/firehose/endpoints.rs index 0ec95c3e2c5..448eb845496 100644 --- a/graph/src/firehose/endpoints.rs +++ b/graph/src/firehose/endpoints.rs @@ -194,9 +194,14 @@ impl FirehoseEndpoint { let endpoint_builder = match uri.scheme().unwrap_or(&Scheme::HTTP).as_str() { "http" => Channel::builder(uri), - "https" => Channel::builder(uri) - .tls_config(ClientTlsConfig::new()) - .expect("TLS config on this host is invalid"), + "https" => { + let mut tls = ClientTlsConfig::new(); + tls = tls.with_native_roots(); + + Channel::builder(uri) + .tls_config(tls) + .expect("TLS config on this host is invalid") + } _ => panic!("invalid uri scheme for firehose endpoint"), }; From 7478019bc49561267b0c567caea76e1213f1f1a0 Mon Sep 17 00:00:00 2001 From: Zoran Cvetkov <36600146+zorancv@users.noreply.github.com> Date: Wed, 2 Apr 2025 19:48:04 +0300 Subject: [PATCH 111/150] Hash legacy hash value when grafting from pre 0.0.6 spec_version --- Cargo.lock | 1 + graph/Cargo.toml | 1 + .../subgraph/proof_of_indexing/online.rs | 14 +++++++++++--- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a7ea378f9e3..33815c70807 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1872,6 +1872,7 @@ dependencies = [ "serde_plain", "serde_regex", "serde_yaml", + "sha2", "slog", "slog-async", "slog-envlogger", diff --git a/graph/Cargo.toml b/graph/Cargo.toml index 163838f5d00..dc4bd6e42e9 100644 --- a/graph/Cargo.toml +++ b/graph/Cargo.toml @@ -48,6 +48,7 @@ serde_derive = { workspace = true } serde_json = { workspace = true } serde_regex = { workspace = true } serde_yaml = { workspace = true } +sha2 = "0.10.8" slog = { version = "2.7.0", features = [ "release_max_level_trace", "max_level_trace", diff --git a/graph/src/components/subgraph/proof_of_indexing/online.rs b/graph/src/components/subgraph/proof_of_indexing/online.rs index d47f08b0a8f..ebf7a65e2f9 100644 --- a/graph/src/components/subgraph/proof_of_indexing/online.rs +++ b/graph/src/components/subgraph/proof_of_indexing/online.rs @@ -9,6 +9,7 @@ use crate::{ prelude::{debug, BlockNumber, DeploymentHash, Logger, ENV_VARS}, util::stable_hash_glue::AsBytes, }; +use sha2::{Digest, Sha256}; use stable_hash::{fast::FastStableHasher, FieldAddress, StableHash, StableHasher}; use stable_hash_legacy::crypto::{Blake3SeqNo, SetHasher}; use stable_hash_legacy::prelude::{ @@ -31,6 +32,8 @@ enum Hashers { Legacy(SetHasher), } +const STABLE_HASH_LEN: usize = 32; + impl Hashers { fn new(version: ProofOfIndexingVersion) -> Self { match version { @@ -132,9 +135,14 @@ impl BlockEventStream { } Hashers::Fast(mut digest) => { if let Some(prev) = prev { - let prev = prev - .try_into() - .expect("Expected valid fast stable hash representation"); + let prev = if prev.len() == STABLE_HASH_LEN { + prev.try_into() + .expect("Expected valid fast stable hash representation") + } else { + let mut hasher = Sha256::new(); + hasher.update(prev); + hasher.finalize().into() + }; let prev = FastStableHasher::from_bytes(prev); digest.mixin(&prev); } From e51dd2478699138fd9538b5d8e5a20c1be43b62d Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Tue, 1 Apr 2025 09:49:18 -0700 Subject: [PATCH 112/150] store: Address corner case in VidBatcher --- store/postgres/src/vid_batcher.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/store/postgres/src/vid_batcher.rs b/store/postgres/src/vid_batcher.rs index ef5948efd06..b8bbaa91113 100644 --- a/store/postgres/src/vid_batcher.rs +++ b/store/postgres/src/vid_batcher.rs @@ -258,7 +258,10 @@ impl VidRange { } pub fn is_empty(&self) -> bool { - self.max == -1 + // min > max can happen when we restart a copy job that has finished + // some tables. For those, min (the next_vid) will be larger than + // max (the target_vid) + self.max == -1 || self.min > self.max } pub fn size(&self) -> usize { From 197d3d1150a44b2a40a2ab4073d3dd526d6a276b Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Tue, 1 Apr 2025 09:57:21 -0700 Subject: [PATCH 113/150] store: Never set start of VidBatcher beyond max_vid+1 --- store/postgres/src/vid_batcher.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/store/postgres/src/vid_batcher.rs b/store/postgres/src/vid_batcher.rs index b8bbaa91113..78abf84529e 100644 --- a/store/postgres/src/vid_batcher.rs +++ b/store/postgres/src/vid_batcher.rs @@ -226,7 +226,8 @@ impl VidBatcher { let duration = self.step_timer.elapsed(); let batch_size = self.batch_size.adapt(duration); - self.start = self.end + 1; + // We can't possibly copy farther than `max_vid` + self.start = (self.end + 1).min(self.max_vid + 1); self.end = ogive.next_point(self.start, batch_size as usize)?; Ok((duration, Some(res))) From a9844a6c5c5909fc293f07390ffe6d81a114d08c Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Tue, 1 Apr 2025 10:28:13 -0700 Subject: [PATCH 114/150] store: Do not mark database unavailable for failed try_get_fdw --- store/postgres/src/connection_pool.rs | 40 ++++++++++++++++++++++----- store/postgres/src/copy.rs | 8 +++--- 2 files changed, 37 insertions(+), 11 deletions(-) diff --git a/store/postgres/src/connection_pool.rs b/store/postgres/src/connection_pool.rs index 782c2a57489..31dfdfd8eb2 100644 --- a/store/postgres/src/connection_pool.rs +++ b/store/postgres/src/connection_pool.rs @@ -347,12 +347,14 @@ impl PoolName { #[derive(Clone)] struct PoolStateTracker { available: Arc, + ignore_timeout: Arc, } impl PoolStateTracker { fn new() -> Self { Self { available: Arc::new(AtomicBool::new(true)), + ignore_timeout: Arc::new(AtomicBool::new(false)), } } @@ -367,6 +369,20 @@ impl PoolStateTracker { fn is_available(&self) -> bool { self.available.load(Ordering::Relaxed) } + + fn timeout_is_ignored(&self) -> bool { + self.ignore_timeout.load(Ordering::Relaxed) + } + + fn ignore_timeout(&self, f: F) -> R + where + F: FnOnce() -> R, + { + self.ignore_timeout.store(true, Ordering::Relaxed); + let res = f(); + self.ignore_timeout.store(false, Ordering::Relaxed); + res + } } impl ConnectionPool { @@ -530,8 +546,12 @@ impl ConnectionPool { &self, logger: &Logger, timeout: Duration, - ) -> Result>>, StoreError> { - self.get_ready()?.try_get_fdw(logger, timeout) + ) -> Option>> { + let Ok(inner) = self.get_ready() else { + return None; + }; + self.state_tracker + .ignore_timeout(|| inner.try_get_fdw(logger, timeout)) } pub fn connection_detail(&self) -> Result { @@ -740,6 +760,9 @@ impl HandleEvent for EventHandler { } fn handle_timeout(&self, event: e::TimeoutEvent) { + if self.state_tracker.timeout_is_ignored() { + return; + } self.add_conn_wait_time(event.timeout()); if self.state_tracker.is_available() { error!(self.logger, "Connection checkout timed out"; @@ -1042,15 +1065,18 @@ impl PoolInner { &self, logger: &Logger, timeout: Duration, - ) -> Result>>, StoreError> { + ) -> Option>> { // Any error trying to get a connection is treated as "couldn't get // a connection in time". If there is a serious error with the // database, e.g., because it's not available, the next database // operation will run into it and report it. - self.fdw_pool(logger)? - .get_timeout(timeout) - .map(|conn| Some(conn)) - .or_else(|_| Ok(None)) + let Ok(fdw_pool) = self.fdw_pool(logger) else { + return None; + }; + let Ok(conn) = fdw_pool.get_timeout(timeout) else { + return None; + }; + Some(conn) } pub fn connection_detail(&self) -> Result { diff --git a/store/postgres/src/copy.rs b/store/postgres/src/copy.rs index 25792bba04e..6438f641eaa 100644 --- a/store/postgres/src/copy.rs +++ b/store/postgres/src/copy.rs @@ -125,7 +125,7 @@ pub fn is_source(conn: &mut PgConnection, site: &Site) -> Result Result>>>, StoreError> { let conn = self.conn.take().ok_or_else(|| { constraint_violation!( - "copy connection has been handed to background task but not returned yet" + "copy connection has been handed to background task but not returned yet (default worker)" ) })?; let Some(table) = state.unfinished.pop() else { @@ -918,7 +918,7 @@ impl Connection { // we remove the table from the state and could drop it otherwise let Some(conn) = self .pool - .try_get_fdw(&self.logger, ENV_VARS.store.batch_worker_wait)? + .try_get_fdw(&self.logger, ENV_VARS.store.batch_worker_wait) else { return Ok(None); }; From d2ef8f408bc778c20ca0d43120ad6fbc36e213a2 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Tue, 1 Apr 2025 10:50:10 -0700 Subject: [PATCH 115/150] store: Be more careful about exiting copy_data_internal early We need to be absolutely sure that when `copy_data_internal` is done, we have a connection in `self.conn` and therefore want to make it clear when we might exit early with an error --- store/postgres/src/copy.rs | 55 +++++++++++++++++++++++++++----------- 1 file changed, 39 insertions(+), 16 deletions(-) diff --git a/store/postgres/src/copy.rs b/store/postgres/src/copy.rs index 6438f641eaa..0307e10b0f5 100644 --- a/store/postgres/src/copy.rs +++ b/store/postgres/src/copy.rs @@ -890,20 +890,18 @@ impl Connection { &mut self, state: &mut CopyState, progress: &Arc, - ) -> Result>>>, StoreError> { - let conn = self.conn.take().ok_or_else(|| { - constraint_violation!( - "copy connection has been handed to background task but not returned yet (default worker)" - ) - })?; + ) -> Option>>> { + let Some(conn) = self.conn.take() else { + return None; + }; let Some(table) = state.unfinished.pop() else { - return Ok(None); + return None; }; let worker = CopyTableWorker::new(conn, table); - Ok(Some(Box::pin( + Some(Box::pin( worker.run(self.logger.cheap_clone(), progress.cheap_clone()), - ))) + )) } /// Opportunistically create an extra worker if we have more tables to @@ -913,23 +911,44 @@ impl Connection { &mut self, state: &mut CopyState, progress: &Arc, - ) -> Result>>>, StoreError> { + ) -> Option>>> { // It's important that we get the connection before the table since // we remove the table from the state and could drop it otherwise let Some(conn) = self .pool .try_get_fdw(&self.logger, ENV_VARS.store.batch_worker_wait) else { - return Ok(None); + return None; }; let Some(table) = state.unfinished.pop() else { - return Ok(None); + return None; }; let worker = CopyTableWorker::new(conn, table); - Ok(Some(Box::pin( + Some(Box::pin( worker.run(self.logger.cheap_clone(), progress.cheap_clone()), - ))) + )) + } + + /// Check that we can make progress, i.e., that we have at least one + /// worker that copies as long as there are unfinished tables. This is a + /// safety check to guard against `copy_data_internal` looping forever + /// because of some internal inconsistency + fn assert_progress(&self, num_workers: usize, state: &CopyState) -> Result<(), StoreError> { + if num_workers == 0 && !state.unfinished.is_empty() { + // Something bad happened. We should have at least one + // worker if there are still tables to copy + if self.conn.is_none() { + return Err(constraint_violation!( + "copy connection has been handed to background task but not returned yet (copy_data_internal)" + )); + } else { + return Err(constraint_violation!( + "no workers left but still tables to copy" + )); + } + } + Ok(()) } pub async fn copy_data_internal( @@ -949,24 +968,28 @@ impl Connection { // connection in `self.conn`. If the fdw pool has idle connections // and there are more tables to be copied, we can start more // workers, up to `self.workers` many + // + // The loop has to be very careful about terminating early so that + // we do not ever leave the loop with `self.conn == None` let mut workers = Vec::new(); while !state.unfinished.is_empty() || !workers.is_empty() { // We usually add at least one job here, except if we are out of // tables to copy. In that case, we go through the `while` loop // every time one of the tables we are currently copying // finishes - if let Some(worker) = self.default_worker(&mut state, &progress)? { + if let Some(worker) = self.default_worker(&mut state, &progress) { workers.push(worker); } loop { if workers.len() >= self.workers { break; } - let Some(worker) = self.extra_worker(&mut state, &progress)? else { + let Some(worker) = self.extra_worker(&mut state, &progress) else { break; }; workers.push(worker); } + self.assert_progress(workers.len(), &state)?; let (worker, _idx, remaining) = select_all(workers).await; workers = remaining; From 9af403e0f808ca4d838163e96e2b52e8af4999fe Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Tue, 1 Apr 2025 11:27:46 -0700 Subject: [PATCH 116/150] store: Spawn copy workers on the blocking pool Otherwise, they don't really run in parallel --- store/postgres/src/copy.rs | 32 ++++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/store/postgres/src/copy.rs b/store/postgres/src/copy.rs index 0307e10b0f5..e2865fc6863 100644 --- a/store/postgres/src/copy.rs +++ b/store/postgres/src/copy.rs @@ -687,9 +687,18 @@ impl CopyTableWorker { } } - async fn run(mut self, logger: Logger, progress: Arc) -> Self { - self.result = self.run_inner(logger, &progress); - self + async fn run( + mut self, + logger: Logger, + progress: Arc, + ) -> Result { + let object = self.table.dst.object.cheap_clone(); + graph::spawn_blocking_allow_panic(move || { + self.result = self.run_inner(logger, &progress); + self + }) + .await + .map_err(|e| constraint_violation!("copy worker for {} panicked: {}", object, e)) } fn run_inner(&mut self, logger: Logger, progress: &CopyProgress) -> Result { @@ -890,7 +899,7 @@ impl Connection { &mut self, state: &mut CopyState, progress: &Arc, - ) -> Option>>> { + ) -> Option>>>> { let Some(conn) = self.conn.take() else { return None; }; @@ -911,7 +920,7 @@ impl Connection { &mut self, state: &mut CopyState, progress: &Arc, - ) -> Option>>> { + ) -> Option>>>> { // It's important that we get the connection before the table since // we remove the table from the state and could drop it otherwise let Some(conn) = self @@ -990,7 +999,18 @@ impl Connection { workers.push(worker); } self.assert_progress(workers.len(), &state)?; - let (worker, _idx, remaining) = select_all(workers).await; + let (result, _idx, remaining) = select_all(workers).await; + + let worker = match result { + Ok(worker) => worker, + Err(e) => { + // This is a panic in the background task. We need to + // cancel all other tasks and return the error + progress.cancel(); + return Err(e); + } + }; + workers = remaining; // Put the connection back into self.conn so that we can use it From 5b8ba9dab37eadf4ae5d14ee8defde7c8c69c222 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Tue, 1 Apr 2025 13:52:46 -0700 Subject: [PATCH 117/150] store: Make sure we release the copy lock even if a worker panics --- store/postgres/src/copy.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/store/postgres/src/copy.rs b/store/postgres/src/copy.rs index e2865fc6863..3288e6c9c56 100644 --- a/store/postgres/src/copy.rs +++ b/store/postgres/src/copy.rs @@ -1104,10 +1104,23 @@ impl Connection { &self.logger, "Obtaining copy lock (this might take a long time if another process is still copying)" ); + let dst_site = self.dst.site.cheap_clone(); self.transaction(|conn| advisory_lock::lock_copying(conn, &dst_site))?; + let res = self.copy_data_internal(index_list).await; + + if self.conn.is_none() { + // A background worker panicked and left us without our + // dedicated connection, but we still need to release the copy + // lock; get a normal connection, not from the fdw pool for that + // as that will be much less contended. We won't be holding on + // to the connection for long as `res` will be an error and we + // will abort starting this subgraph + self.conn = Some(self.pool.get()?); + } self.transaction(|conn| advisory_lock::unlock_copying(conn, &dst_site))?; + if matches!(res, Ok(Status::Cancelled)) { warn!(&self.logger, "Copying was cancelled and is incomplete"); } From 93d78620375c3299806a6814b29f102d79ae42af Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Tue, 1 Apr 2025 18:07:33 -0700 Subject: [PATCH 118/150] store: Better error when a copy batch times out --- store/postgres/src/copy.rs | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/store/postgres/src/copy.rs b/store/postgres/src/copy.rs index 3288e6c9c56..fbf74d251cf 100644 --- a/store/postgres/src/copy.rs +++ b/store/postgres/src/copy.rs @@ -748,10 +748,19 @@ impl CopyTableWorker { break status; } Err(StoreError::StatementTimeout) => { + let timeout = ENV_VARS + .store + .batch_timeout + .map(|t| t.as_secs().to_string()) + .unwrap_or_else(|| "unlimted".to_string()); warn!( - logger, - "Current batch took longer than GRAPH_STORE_BATCH_TIMEOUT seconds. Retrying with a smaller batch size." - ); + logger, + "Current batch timed out. Retrying with a smaller batch size."; + "timeout_s" => timeout, + "table" => self.table.dst.qualified_name.as_str(), + "current_vid" => self.table.batcher.next_vid(), + "current_batch_size" => self.table.batcher.batch_size(), + ); } Err(e) => { return Err(e); From b04c3769a13ff9d2a4313d87f72b91d1c2ca0233 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Tue, 1 Apr 2025 18:10:49 -0700 Subject: [PATCH 119/150] store: Reset the end of VidBatcher when changing the size --- store/postgres/src/vid_batcher.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/store/postgres/src/vid_batcher.rs b/store/postgres/src/vid_batcher.rs index 78abf84529e..93197b5a85d 100644 --- a/store/postgres/src/vid_batcher.rs +++ b/store/postgres/src/vid_batcher.rs @@ -237,6 +237,10 @@ impl VidBatcher { pub(crate) fn set_batch_size(&mut self, size: usize) { self.batch_size.size = size as i64; + self.end = match &self.ogive { + Some(ogive) => ogive.next_point(self.start, size as usize).unwrap(), + None => self.start + size as i64, + }; } } From 02a96c2ee790bda9d3bb31ed7a2db777f82b7895 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Tue, 1 Apr 2025 20:54:22 -0700 Subject: [PATCH 120/150] graph: Reduce default fdw_fetch_size to 1000 10,000 seems too big and actually slows things down --- graph/src/env/store.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/graph/src/env/store.rs b/graph/src/env/store.rs index 661d0356446..3ecf92e0388 100644 --- a/graph/src/env/store.rs +++ b/graph/src/env/store.rs @@ -280,7 +280,7 @@ pub struct InnerStore { last_rollup_from_poi: bool, #[envconfig(from = "GRAPH_STORE_INSERT_EXTRA_COLS", default = "0")] insert_extra_cols: usize, - #[envconfig(from = "GRAPH_STORE_FDW_FETCH_SIZE", default = "10000")] + #[envconfig(from = "GRAPH_STORE_FDW_FETCH_SIZE", default = "1000")] fdw_fetch_size: usize, } From bbb7478477277a6102483799596efc8cbcbdac81 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Wed, 2 Apr 2025 11:37:29 -0700 Subject: [PATCH 121/150] store: If copying encounters an error, wait for all workers to finish --- store/postgres/src/copy.rs | 36 ++++++++++++++++++++++++++++++++---- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/store/postgres/src/copy.rs b/store/postgres/src/copy.rs index fbf74d251cf..0d9c27cf6b7 100644 --- a/store/postgres/src/copy.rs +++ b/store/postgres/src/copy.rs @@ -37,6 +37,7 @@ use graph::{ info, lazy_static, o, warn, BlockNumber, BlockPtr, CheapClone, Logger, StoreError, ENV_VARS, }, schema::EntityType, + slog::error, }; use itertools::Itertools; @@ -969,6 +970,33 @@ impl Connection { Ok(()) } + /// Wait for all workers to finish. This is called when we a worker has + /// failed with an error that forces us to abort copying + async fn cancel_workers( + &mut self, + progress: Arc, + mut workers: Vec>>>>, + ) { + progress.cancel(); + error!( + self.logger, + "copying encountered an error; waiting for all workers to finish" + ); + while !workers.is_empty() { + let (result, _, remaining) = select_all(workers).await; + workers = remaining; + match result { + Ok(worker) => { + self.conn = Some(worker.conn); + } + Err(e) => { + /* Ignore; we had an error previously */ + error!(self.logger, "copy worker panicked: {}", e); + } + } + } + } + pub async fn copy_data_internal( &mut self, index_list: IndexList, @@ -1009,30 +1037,30 @@ impl Connection { } self.assert_progress(workers.len(), &state)?; let (result, _idx, remaining) = select_all(workers).await; + workers = remaining; let worker = match result { Ok(worker) => worker, Err(e) => { // This is a panic in the background task. We need to // cancel all other tasks and return the error - progress.cancel(); + self.cancel_workers(progress, workers).await; return Err(e); } }; - workers = remaining; - // Put the connection back into self.conn so that we can use it // in the next iteration. self.conn = Some(worker.conn); state.finished.push(worker.table); if worker.result.is_err() { - progress.cancel(); + self.cancel_workers(progress, workers).await; return worker.result; } if progress.is_cancelled() { + self.cancel_workers(progress, workers).await; return Ok(Status::Cancelled); } } From f56900ed79253ea9cb97a992010112975c5829b1 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Wed, 2 Apr 2025 11:45:25 -0700 Subject: [PATCH 122/150] store: Clarify the logic of the control loop in copy_data_internal --- store/postgres/src/copy.rs | 48 ++++++++++++++++++++++++-------------- 1 file changed, 31 insertions(+), 17 deletions(-) diff --git a/store/postgres/src/copy.rs b/store/postgres/src/copy.rs index 0d9c27cf6b7..b5db91d40da 100644 --- a/store/postgres/src/copy.rs +++ b/store/postgres/src/copy.rs @@ -1035,34 +1035,48 @@ impl Connection { }; workers.push(worker); } + self.assert_progress(workers.len(), &state)?; let (result, _idx, remaining) = select_all(workers).await; workers = remaining; - let worker = match result { - Ok(worker) => worker, + // Analyze `result` and take another trip through the loop if + // everything is ok; wait for pending workers and return if + // there was an error or if copying was cancelled. + match result { Err(e) => { // This is a panic in the background task. We need to // cancel all other tasks and return the error self.cancel_workers(progress, workers).await; return Err(e); } - }; - - // Put the connection back into self.conn so that we can use it - // in the next iteration. - self.conn = Some(worker.conn); - state.finished.push(worker.table); - - if worker.result.is_err() { - self.cancel_workers(progress, workers).await; - return worker.result; - } + Ok(worker) => { + // Put the connection back into self.conn so that we can use it + // in the next iteration. + self.conn = Some(worker.conn); - if progress.is_cancelled() { - self.cancel_workers(progress, workers).await; - return Ok(Status::Cancelled); - } + match (worker.result, progress.is_cancelled()) { + (Ok(Status::Finished), false) => { + // The worker finished successfully, and nothing was + // cancelled; take another trip through the loop + state.finished.push(worker.table); + } + (Ok(Status::Finished), true) => { + state.finished.push(worker.table); + self.cancel_workers(progress, workers).await; + return Ok(Status::Cancelled); + } + (Ok(Status::Cancelled), _) => { + self.cancel_workers(progress, workers).await; + return Ok(Status::Cancelled); + } + (Err(e), _) => { + self.cancel_workers(progress, workers).await; + return Err(e); + } + } + } + }; } debug_assert!(self.conn.is_some()); From 843278aa70991063c25c63378f31bcc64c433754 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Wed, 2 Apr 2025 12:15:34 -0700 Subject: [PATCH 123/150] store: Take ownership of self in CopyConnection.copy_data This ensures that `copy_data` can't be called more than once on any instance; when copying encounters an error, it might leave the CopyConnection in an inconsistent state and should therefore not be reused Also make `copy_data_internal` private; it should never be called from the outside --- store/postgres/src/copy.rs | 7 ++----- store/postgres/src/deployment_store.rs | 2 +- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/store/postgres/src/copy.rs b/store/postgres/src/copy.rs index b5db91d40da..9e9ba187c6a 100644 --- a/store/postgres/src/copy.rs +++ b/store/postgres/src/copy.rs @@ -997,10 +997,7 @@ impl Connection { } } - pub async fn copy_data_internal( - &mut self, - index_list: IndexList, - ) -> Result { + async fn copy_data_internal(&mut self, index_list: IndexList) -> Result { let src = self.src.clone(); let dst = self.dst.clone(); let target_block = self.target_block.clone(); @@ -1142,7 +1139,7 @@ impl Connection { /// lower(v1.block_range) => v2.vid > v1.vid` and we can therefore stop /// the copying of each table as soon as we hit `max_vid = max { v.vid | /// lower(v.block_range) <= target_block.number }`. - pub async fn copy_data(&mut self, index_list: IndexList) -> Result { + pub async fn copy_data(mut self, index_list: IndexList) -> Result { // We require sole access to the destination site, and that we get a // consistent view of what has been copied so far. In general, that // is always true. It can happen though that this function runs when diff --git a/store/postgres/src/deployment_store.rs b/store/postgres/src/deployment_store.rs index 248ba5a5473..96dd5507f3e 100644 --- a/store/postgres/src/deployment_store.rs +++ b/store/postgres/src/deployment_store.rs @@ -1519,7 +1519,7 @@ impl DeploymentStore { // as adding new tables in `self`; we only need to check that tables // that actually need to be copied from the source are compatible // with the corresponding tables in `self` - let mut copy_conn = crate::copy::Connection::new( + let copy_conn = crate::copy::Connection::new( logger, self.pool.clone(), src.clone(), From 9a13debfc56edd63602295609baf93b6f7c2572a Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Thu, 3 Apr 2025 11:58:44 -0700 Subject: [PATCH 124/150] store: Check that we map all the tables that MirrorJob needs --- store/postgres/src/connection_pool.rs | 30 ++++++++++++++++++++++++- store/postgres/src/primary.rs | 32 ++++++++++++++------------- 2 files changed, 46 insertions(+), 16 deletions(-) diff --git a/store/postgres/src/connection_pool.rs b/store/postgres/src/connection_pool.rs index 31dfdfd8eb2..ace5cddd719 100644 --- a/store/postgres/src/connection_pool.rs +++ b/store/postgres/src/connection_pool.rs @@ -33,7 +33,7 @@ use std::{collections::HashMap, sync::RwLock}; use postgres::config::{Config, Host}; -use crate::primary::{self, NAMESPACE_PUBLIC}; +use crate::primary::{self, Mirror, NAMESPACE_PUBLIC}; use crate::{advisory_lock, catalog}; use crate::{Shard, PRIMARY_SHARD}; @@ -54,10 +54,36 @@ const SHARDED_TABLES: [(&str, &[&str]); 2] = [ "subgraph_error", "subgraph_manifest", "table_stats", + "subgraph", + "subgraph_version", + "subgraph_deployment_assignment", ], ), ]; +/// Make sure that the tables that `jobs::MirrorJob` wants to mirror are +/// actually mapped into the various shards. A failure here is simply a +/// coding mistake +fn check_mirrored_tables() { + for table in Mirror::PUBLIC_TABLES { + if !PRIMARY_TABLES.contains(&table) { + panic!("table {} is not in PRIMARY_TABLES", table); + } + } + + let subgraphs_tables = *SHARDED_TABLES + .iter() + .find(|(nsp, _)| *nsp == "subgraphs") + .map(|(_, tables)| tables) + .unwrap(); + + for table in Mirror::SUBGRAPHS_TABLES { + if !subgraphs_tables.contains(&table) { + panic!("table {} is not in SHARDED_TABLES[subgraphs]", table); + } + } +} + pub struct ForeignServer { pub name: String, pub shard: Shard, @@ -817,6 +843,8 @@ impl PoolInner { registry: Arc, state_tracker: PoolStateTracker, ) -> PoolInner { + check_mirrored_tables(); + let logger_store = logger.new(o!("component" => "Store")); let logger_pool = logger.new(o!("component" => "ConnectionPool")); let const_labels = { diff --git a/store/postgres/src/primary.rs b/store/postgres/src/primary.rs index ab6be9ee0ba..5ec81dcbd61 100644 --- a/store/postgres/src/primary.rs +++ b/store/postgres/src/primary.rs @@ -1839,6 +1839,20 @@ pub struct Mirror { } impl Mirror { + // The tables that we mirror + // + // `chains` needs to be mirrored before `deployment_schemas` because + // of the fk constraint on `deployment_schemas.network`. We don't + // care much about mirroring `active_copies` but it has a fk + // constraint on `deployment_schemas` and is tiny, therefore it's + // easiest to just mirror it + pub(crate) const PUBLIC_TABLES: [&str; 3] = ["chains", "deployment_schemas", "active_copies"]; + pub(crate) const SUBGRAPHS_TABLES: [&str; 3] = [ + "subgraph_deployment_assignment", + "subgraph", + "subgraph_version", + ]; + pub fn new(pools: &HashMap) -> Mirror { let primary = pools .get(&PRIMARY_SHARD) @@ -1895,18 +1909,6 @@ impl Mirror { conn: &mut PgConnection, handle: &CancelHandle, ) -> Result<(), StoreError> { - // `chains` needs to be mirrored before `deployment_schemas` because - // of the fk constraint on `deployment_schemas.network`. We don't - // care much about mirroring `active_copies` but it has a fk - // constraint on `deployment_schemas` and is tiny, therefore it's - // easiest to just mirror it - const PUBLIC_TABLES: [&str; 3] = ["chains", "deployment_schemas", "active_copies"]; - const SUBGRAPHS_TABLES: [&str; 3] = [ - "subgraph_deployment_assignment", - "subgraph", - "subgraph_version", - ]; - fn run_query(conn: &mut PgConnection, query: String) -> Result<(), StoreError> { conn.batch_execute(&query).map_err(StoreError::from) } @@ -1938,11 +1940,11 @@ impl Mirror { // Truncate all tables at once, otherwise truncation can fail // because of foreign key constraints - let tables = PUBLIC_TABLES + let tables = Self::PUBLIC_TABLES .iter() .map(|name| (NAMESPACE_PUBLIC, name)) .chain( - SUBGRAPHS_TABLES + Self::SUBGRAPHS_TABLES .iter() .map(|name| (NAMESPACE_SUBGRAPHS, name)), ) @@ -1953,7 +1955,7 @@ impl Mirror { check_cancel()?; // Repopulate `PUBLIC_TABLES` by copying their data wholesale - for table_name in PUBLIC_TABLES { + for table_name in Self::PUBLIC_TABLES { copy_table( conn, ForeignServer::PRIMARY_PUBLIC, From c765ce7a09dd93249a8541503f7c55f0a686dfc6 Mon Sep 17 00:00:00 2001 From: Krishnanand V P <44740264+incrypto32@users.noreply.github.com> Date: Mon, 7 Apr 2025 16:08:40 +0400 Subject: [PATCH 125/150] graphman: disable load management when using graphman (#5875) --- node/src/bin/manager.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/node/src/bin/manager.rs b/node/src/bin/manager.rs index 20cf93d94df..5142a2ab939 100644 --- a/node/src/bin/manager.rs +++ b/node/src/bin/manager.rs @@ -34,6 +34,7 @@ use graph_store_postgres::{ }; use itertools::Itertools; use lazy_static::lazy_static; +use std::env; use std::str::FromStr; use std::{collections::HashMap, num::ParseIntError, sync::Arc, time::Duration}; const VERSION_LABEL_KEY: &str = "version"; @@ -1030,6 +1031,9 @@ impl Context { #[tokio::main] async fn main() -> anyhow::Result<()> { + // Disable load management for graphman commands + env::set_var("GRAPH_LOAD_THRESHOLD", "0"); + let opt = Opt::parse(); Terminal::set_color_preference(&opt.color); From 8ddd068480c4754a86c6e73857204dffe1bb0232 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Fri, 4 Apr 2025 10:21:48 -0700 Subject: [PATCH 126/150] graph: Make metrics registration less noisy --- graph/src/components/metrics/registry.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/graph/src/components/metrics/registry.rs b/graph/src/components/metrics/registry.rs index e010d3a89fa..93cf51b3bd1 100644 --- a/graph/src/components/metrics/registry.rs +++ b/graph/src/components/metrics/registry.rs @@ -3,7 +3,7 @@ use std::sync::{Arc, RwLock}; use prometheus::IntGauge; use prometheus::{labels, Histogram, IntCounterVec}; -use slog::info; +use slog::debug; use crate::components::metrics::{counter_with_labels, gauge_with_labels}; use crate::prelude::Collector; @@ -133,7 +133,7 @@ impl MetricsRegistry { let mut result = self.registry.register(collector.clone()); if matches!(result, Err(PrometheusError::AlreadyReg)) { - info!(logger, "Resolving duplicate metric registration"); + debug!(logger, "Resolving duplicate metric registration"); // Since the current metric is a duplicate, // we can use it to unregister the previous registration. @@ -144,7 +144,6 @@ impl MetricsRegistry { match result { Ok(()) => { - info!(logger, "Successfully registered a new metric"); self.registered_metrics.inc(); } Err(err) => { From 13cce5330b19dc9488458342fb3c8a45c3dcb37f Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Thu, 3 Apr 2025 09:53:41 -0700 Subject: [PATCH 127/150] store: Allow creating special Namespaces --- store/postgres/src/primary.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/store/postgres/src/primary.rs b/store/postgres/src/primary.rs index 5ec81dcbd61..f329ae4bba2 100644 --- a/store/postgres/src/primary.rs +++ b/store/postgres/src/primary.rs @@ -266,6 +266,13 @@ impl Namespace { Namespace(format!("prune{id}")) } + /// A namespace that is not a deployment namespace. This is used for + /// special namespaces we use. No checking is done on `s` and the caller + /// must ensure it's a valid namespace name + pub fn special(s: impl Into) -> Self { + Namespace(s.into()) + } + pub fn as_str(&self) -> &str { &self.0 } From 3eac30c4e594875b8c136ec4cf27fab0faf5c0e9 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Thu, 3 Apr 2025 14:45:41 -0700 Subject: [PATCH 128/150] store: Factor the locale check into a method --- store/postgres/src/connection_pool.rs | 35 +++++++++++++++++---------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/store/postgres/src/connection_pool.rs b/store/postgres/src/connection_pool.rs index ace5cddd719..05868815c02 100644 --- a/store/postgres/src/connection_pool.rs +++ b/store/postgres/src/connection_pool.rs @@ -1169,24 +1169,33 @@ impl PoolInner { .and_then(|()| pool.create_cross_shard_views(coord.servers.as_ref())); result.unwrap_or_else(|err| die(&pool.logger, "migrations failed", &err)); - // Locale check - if let Err(msg) = catalog::Locale::load(&mut conn)?.suitable() { - if &self.shard == &*PRIMARY_SHARD && primary::is_empty(&mut conn)? { - die( - &pool.logger, - "Database does not use C locale. \ - Please check the graph-node documentation for how to set up the database locale", - &msg, - ); - } else { - warn!(pool.logger, "{}.\nPlease check the graph-node documentation for how to set up the database locale", msg); - } - } + self.locale_check(&pool.logger, conn)?; debug!(&pool.logger, "Setup finished"; "setup_time_s" => start.elapsed().as_secs()); Ok(()) } + fn locale_check( + &self, + logger: &Logger, + mut conn: PooledConnection>, + ) -> Result<(), StoreError> { + Ok( + if let Err(msg) = catalog::Locale::load(&mut conn)?.suitable() { + if &self.shard == &*PRIMARY_SHARD && primary::is_empty(&mut conn)? { + const MSG: &str = + "Database does not use C locale. \ + Please check the graph-node documentation for how to set up the database locale"; + + crit!(logger, "{}: {}", MSG, msg); + panic!("{}: {}", MSG, msg); + } else { + warn!(logger, "{}.\nPlease check the graph-node documentation for how to set up the database locale", msg); + } + }, + ) + } + pub(crate) async fn query_permit(&self) -> tokio::sync::OwnedSemaphorePermit { let start = Instant::now(); let permit = self.query_semaphore.cheap_clone().acquire_owned().await; From 034cba57a833fd4db619f5ca7120cd5d4e957e78 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Fri, 4 Apr 2025 10:22:18 -0700 Subject: [PATCH 129/150] graph: Allow returning values from task_spawn --- graph/src/task_spawn.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/graph/src/task_spawn.rs b/graph/src/task_spawn.rs index 09055ad5381..dd1477bb1c8 100644 --- a/graph/src/task_spawn.rs +++ b/graph/src/task_spawn.rs @@ -57,10 +57,11 @@ pub fn block_on(f: impl Future03) -> T { } /// Spawns a thread with access to the tokio runtime. Panics if the thread cannot be spawned. -pub fn spawn_thread( - name: impl Into, - f: impl 'static + FnOnce() + Send, -) -> std::thread::JoinHandle<()> { +pub fn spawn_thread(name: impl Into, f: F) -> std::thread::JoinHandle +where + F: 'static + FnOnce() -> R + Send, + R: 'static + Send, +{ let conf = std::thread::Builder::new().name(name.into()); let runtime = tokio::runtime::Handle::current(); conf.spawn(move || { From cbbd4e1680ec8e5089df6234dc8364400ad2ff31 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Fri, 4 Apr 2025 10:37:06 -0700 Subject: [PATCH 130/150] store: Remove dead code from connection_pool --- store/postgres/src/connection_pool.rs | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/store/postgres/src/connection_pool.rs b/store/postgres/src/connection_pool.rs index 05868815c02..7b4bc1bafb6 100644 --- a/store/postgres/src/connection_pool.rs +++ b/store/postgres/src/connection_pool.rs @@ -580,16 +580,6 @@ impl ConnectionPool { .ignore_timeout(|| inner.try_get_fdw(logger, timeout)) } - pub fn connection_detail(&self) -> Result { - let pool = self.get_ready()?; - ForeignServer::new(pool.shard.clone(), &pool.postgres_url).map_err(|e| e.into()) - } - - /// Check that we can connect to the database - pub fn check(&self) -> bool { - true - } - /// Setup the database for this pool. This includes configuring foreign /// data wrappers for cross-shard communication, and running any pending /// schema migrations for this database. @@ -1027,20 +1017,6 @@ impl PoolInner { self.pool.get().map_err(|_| StoreError::DatabaseUnavailable) } - pub fn get_with_timeout_warning( - &self, - logger: &Logger, - ) -> Result>, StoreError> { - loop { - match self.pool.get_timeout(ENV_VARS.store.connection_timeout) { - Ok(conn) => return Ok(conn), - Err(e) => error!(logger, "Error checking out connection, retrying"; - "error" => brief_error_msg(&e), - ), - } - } - } - /// Get the pool for fdw connections. It is an error if none is configured fn fdw_pool( &self, From f162e2f97bbeaeba8a43de3b012e29e51bd27568 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Fri, 4 Apr 2025 21:29:21 -0700 Subject: [PATCH 131/150] store: Do not manage anything about pg_stat_statements It should be up to the operator if they use it or not, and when they want to reset it --- store/postgres/src/connection_pool.rs | 5 ----- tests/src/config.rs | 1 - 2 files changed, 6 deletions(-) diff --git a/store/postgres/src/connection_pool.rs b/store/postgres/src/connection_pool.rs index 7b4bc1bafb6..e00071ef138 100644 --- a/store/postgres/src/connection_pool.rs +++ b/store/postgres/src/connection_pool.rs @@ -1364,11 +1364,6 @@ fn migrate_schema(logger: &Logger, conn: &mut PgConnection) -> Result Date: Thu, 3 Apr 2025 09:31:51 -0700 Subject: [PATCH 132/150] store: Change db setup strategy to guard better against races The current database setup code was inerently racy when several nodes were starting up as it relied on piecemeal locking of individual steps. This change completely revamps the strategy we use: setup now takes a lock on the primary, so that only one node at a time will run the setup code. --- graph/src/components/store/err.rs | 2 +- node/src/store_builder.rs | 7 +- store/postgres/src/advisory_lock.rs | 34 ++- store/postgres/src/connection_pool.rs | 291 ++++++++++++++++++-------- 4 files changed, 227 insertions(+), 107 deletions(-) diff --git a/graph/src/components/store/err.rs b/graph/src/components/store/err.rs index 6af676f8e52..76be7c311ce 100644 --- a/graph/src/components/store/err.rs +++ b/graph/src/components/store/err.rs @@ -141,7 +141,7 @@ impl Clone for StoreError { } impl StoreError { - fn from_diesel_error(e: &DieselError) -> Option { + pub fn from_diesel_error(e: &DieselError) -> Option { const CONN_CLOSE: &str = "server closed the connection unexpectedly"; const STMT_TIMEOUT: &str = "canceling statement due to statement timeout"; let DieselError::DatabaseError(_, info) = e else { diff --git a/node/src/store_builder.rs b/node/src/store_builder.rs index 7fadf6b92c2..abaf59471fd 100644 --- a/node/src/store_builder.rs +++ b/node/src/store_builder.rs @@ -1,7 +1,6 @@ use std::iter::FromIterator; use std::{collections::HashMap, sync::Arc}; -use graph::futures03::future::join_all; use graph::prelude::{o, MetricsRegistry, NodeId}; use graph::url::Url; use graph::{ @@ -62,7 +61,7 @@ impl StoreBuilder { // attempt doesn't work for all of them because the database is // unavailable, they will try again later in the normal course of // using the pool - join_all(pools.values().map(|pool| pool.setup())).await; + coord.setup_all(logger).await; let chains = HashMap::from_iter(config.chains.chains.iter().map(|(name, chain)| { let shard = ShardName::new(chain.shard.to_string()) @@ -196,8 +195,8 @@ impl StoreBuilder { Arc::new(DieselStore::new(subgraph_store, block_store)) } - /// Create a connection pool for the main database of the primary shard - /// without connecting to all the other configured databases + /// Create a connection pool for the main (non-replica) database of a + /// shard pub fn main_pool( logger: &Logger, node: &NodeId, diff --git a/store/postgres/src/advisory_lock.rs b/store/postgres/src/advisory_lock.rs index bd60d34c634..85e2cf5a4ae 100644 --- a/store/postgres/src/advisory_lock.rs +++ b/store/postgres/src/advisory_lock.rs @@ -6,7 +6,7 @@ //! has more details on advisory locks. //! //! We use the following 64 bit locks: -//! * 1,2: to synchronize on migratons +//! * 1: to synchronize on migratons //! //! We use the following 2x 32-bit locks //! * 1, n: to lock copying of the deployment with id n in the destination @@ -69,17 +69,31 @@ const COPY: Scope = Scope { id: 1 }; const WRITE: Scope = Scope { id: 2 }; const PRUNE: Scope = Scope { id: 3 }; -/// Get a lock for running migrations. Blocks until we get the lock. -pub(crate) fn lock_migration(conn: &mut PgConnection) -> Result<(), StoreError> { - sql_query("select pg_advisory_lock(1)").execute(conn)?; +/// Block until we can get the migration lock, then run `f` and unlock when +/// it is done. This is used to make sure that only one node runs setup at a +/// time. +pub(crate) async fn with_migration_lock( + conn: &mut PgConnection, + f: F, +) -> Result +where + F: FnOnce(&mut PgConnection) -> Fut, + Fut: std::future::Future>, +{ + fn execute(conn: &mut PgConnection, query: &str, msg: &str) -> Result<(), StoreError> { + sql_query(query).execute(conn).map(|_| ()).map_err(|e| { + StoreError::from_diesel_error(&e) + .unwrap_or_else(|| StoreError::Unknown(anyhow::anyhow!("{}: {}", msg, e))) + }) + } - Ok(()) -} + const LOCK: &str = "select pg_advisory_lock(1)"; + const UNLOCK: &str = "select pg_advisory_unlock(1)"; -/// Release the migration lock. -pub(crate) fn unlock_migration(conn: &mut PgConnection) -> Result<(), StoreError> { - sql_query("select pg_advisory_unlock(1)").execute(conn)?; - Ok(()) + execute(conn, LOCK, "failed to acquire migration lock")?; + let res = f(conn).await; + execute(conn, UNLOCK, "failed to release migration lock")?; + res } /// Take the lock used to keep two copy operations to run simultaneously on diff --git a/store/postgres/src/connection_pool.rs b/store/postgres/src/connection_pool.rs index e00071ef138..171fb8dbbb6 100644 --- a/store/postgres/src/connection_pool.rs +++ b/store/postgres/src/connection_pool.rs @@ -10,6 +10,8 @@ use diesel_migrations::{EmbeddedMigrations, HarnessWithOutput}; use graph::cheap_clone::CheapClone; use graph::components::store::QueryPermit; use graph::constraint_violation; +use graph::futures03::future::join_all; +use graph::futures03::FutureExt as _; use graph::prelude::tokio::time::Instant; use graph::prelude::{tokio, MetricsRegistry}; use graph::slog::warn; @@ -33,8 +35,9 @@ use std::{collections::HashMap, sync::RwLock}; use postgres::config::{Config, Host}; -use crate::primary::{self, Mirror, NAMESPACE_PUBLIC}; -use crate::{advisory_lock, catalog}; +use crate::advisory_lock::with_migration_lock; +use crate::catalog; +use crate::primary::{self, Mirror, Namespace, NAMESPACE_PUBLIC}; use crate::{Shard, PRIMARY_SHARD}; /// Tables that we map from the primary into `primary_public` in each shard @@ -479,12 +482,17 @@ impl ConnectionPool { } match &*guard { - PoolState::Created(pool, servers) => { - pool.setup(servers.clone())?; - let pool2 = pool.clone(); - *guard = PoolState::Ready(pool.clone()); - self.state_tracker.mark_available(); - Ok(pool2) + PoolState::Created(pool, coord) => { + let migrated = coord.cheap_clone().setup_bg(pool.cheap_clone())?; + + if migrated { + let pool2 = pool.clone(); + *guard = PoolState::Ready(pool.clone()); + self.state_tracker.mark_available(); + Ok(pool2) + } else { + Err(StoreError::DatabaseUnavailable) + } } PoolState::Ready(pool) => Ok(pool.clone()), PoolState::Disabled => Err(StoreError::DatabaseDisabled), @@ -580,23 +588,6 @@ impl ConnectionPool { .ignore_timeout(|| inner.try_get_fdw(logger, timeout)) } - /// Setup the database for this pool. This includes configuring foreign - /// data wrappers for cross-shard communication, and running any pending - /// schema migrations for this database. - /// - /// # Panics - /// - /// If any errors happen during the migration, the process panics - pub async fn setup(&self) { - let pool = self.clone(); - graph::spawn_blocking_allow_panic(move || { - pool.get_ready().ok(); - }) - .await - // propagate panics - .unwrap(); - } - pub(crate) async fn query_permit(&self) -> Result { let pool = match &*self.inner.lock(&self.logger) { PoolState::Created(pool, _) | PoolState::Ready(pool) => pool.clone(), @@ -1096,61 +1087,6 @@ impl PoolInner { .unwrap_or(false) } - /// Setup the database for this pool. This includes configuring foreign - /// data wrappers for cross-shard communication, and running any pending - /// schema migrations for this database. - /// - /// Returns `StoreError::DatabaseUnavailable` if we can't connect to the - /// database. Any other error causes a panic. - /// - /// # Panics - /// - /// If any errors happen during the migration, the process panics - fn setup(&self, coord: Arc) -> Result<(), StoreError> { - fn die(logger: &Logger, msg: &'static str, err: &dyn std::fmt::Display) -> ! { - crit!(logger, "{}", msg; "error" => format!("{:#}", err)); - panic!("{}: {}", msg, err); - } - - let pool = self.clone(); - let mut conn = self.get().map_err(|_| StoreError::DatabaseUnavailable)?; - - let start = Instant::now(); - - advisory_lock::lock_migration(&mut conn) - .unwrap_or_else(|err| die(&pool.logger, "failed to get migration lock", &err)); - // This code can cause a race in database setup: if pool A has had - // schema changes and pool B then tries to map tables from pool A, - // but does so before the concurrent thread running this code for - // pool B has at least finished `configure_fdw`, mapping tables will - // fail. In that case, the node must be restarted. The restart is - // guaranteed because this failure will lead to a panic in the setup - // for pool A - // - // This code can also leave the table mappings in a state where they - // have not been updated if the process is killed after migrating - // the schema but before finishing remapping in all shards. - // Addressing that would require keeping track of the need to remap - // in the database instead of just in memory - let result = pool - .configure_fdw(coord.servers.as_ref()) - .and_then(|()| pool.drop_cross_shard_views()) - .and_then(|()| migrate_schema(&pool.logger, &mut conn)); - debug!(&pool.logger, "Release migration lock"); - advisory_lock::unlock_migration(&mut conn).unwrap_or_else(|err| { - die(&pool.logger, "failed to release migration lock", &err); - }); - let result = result - .and_then(|count| coord.propagate(&pool, count)) - .and_then(|()| pool.create_cross_shard_views(coord.servers.as_ref())); - result.unwrap_or_else(|err| die(&pool.logger, "migrations failed", &err)); - - self.locale_check(&pool.logger, conn)?; - - debug!(&pool.logger, "Setup finished"; "setup_time_s" => start.elapsed().as_secs()); - Ok(()) - } - fn locale_check( &self, logger: &Logger, @@ -1199,6 +1135,28 @@ impl PoolInner { }) } + /// Do the part of database setup that only affects this pool. Those + /// steps are + /// 1. Configuring foreign servers and user mappings for talking to the + /// other shards + /// 2. Migrating the schema to the latest version + /// 3. Checking that the locale is set to C + async fn migrate( + self: Arc, + servers: &[ForeignServer], + ) -> Result<(Arc, MigrationCount), StoreError> { + self.configure_fdw(servers)?; + let mut conn = self.get()?; + let (this, count) = conn.transaction(|conn| -> Result<_, StoreError> { + let count = migrate_schema(&self.logger, conn)?; + Ok((self, count)) + })?; + + this.locale_check(&this.logger, conn)?; + + Ok((this, count)) + } + /// If this is the primary shard, drop the namespace `CROSS_SHARD_NSP` fn drop_cross_shard_views(&self) -> Result<(), StoreError> { if self.shard != *PRIMARY_SHARD { @@ -1242,14 +1200,17 @@ impl PoolInner { return Ok(()); } - info!(&self.logger, "Creating cross-shard views"); let mut conn = self.get()?; + let sharded = Namespace::special(ForeignServer::CROSS_SHARD_NSP); + if catalog::has_namespace(&mut conn, &sharded)? { + // We dropped the namespace before, but another node must have + // recreated it in the meantime so we don't need to do anything + return Ok(()); + } + info!(&self.logger, "Creating cross-shard views"); conn.transaction(|conn| { - let query = format!( - "create schema if not exists {}", - ForeignServer::CROSS_SHARD_NSP - ); + let query = format!("create schema {}", ForeignServer::CROSS_SHARD_NSP); conn.batch_execute(&query)?; for (src_nsp, src_tables) in SHARDED_TABLES { // Pairs of (shard, nsp) for all servers @@ -1458,13 +1419,7 @@ impl PoolCoordinator { if count.had_migrations() { let server = self.server(&pool.shard)?; for pool in self.pools.lock().unwrap().values() { - let mut conn = pool.get()?; - let remap_res = { - advisory_lock::lock_migration(&mut conn)?; - let res = pool.remap(server); - advisory_lock::unlock_migration(&mut conn)?; - res - }; + let remap_res = pool.remap(server); if let Err(e) = remap_res { error!(pool.logger, "Failed to map imports from {}", server.shard; "error" => e.to_string()); return Err(e); @@ -1488,4 +1443,156 @@ impl PoolCoordinator { .find(|server| &server.shard == shard) .ok_or_else(|| constraint_violation!("unknown shard {shard}")) } + + fn primary(&self) -> Result, StoreError> { + self.pools + .lock() + .unwrap() + .get(&*PRIMARY_SHARD) + .cloned() + .ok_or_else(|| { + constraint_violation!("internal error: primary shard not found in pool coordinator") + }) + } + + /// Setup all pools the coordinator knows about and return the number of + /// pools that were successfully set up. + /// + /// # Panics + /// + /// If any errors besides a database not being available happen during + /// the migration, the process panics + pub async fn setup_all(&self, logger: &Logger) -> usize { + let pools = self + .pools + .lock() + .unwrap() + .values() + .cloned() + .collect::>(); + + let res = self.setup(pools).await; + + match res { + Ok(count) => { + info!(logger, "Setup finished"; "shards" => count); + count + } + Err(e) => { + crit!(logger, "database setup failed"; "error" => format!("{e}")); + panic!("database setup failed: {}", e); + } + } + } + + /// A helper to call `setup` from a non-async context. Returns `true` if + /// the setup was actually run, i.e. if `pool` was available + fn setup_bg(self: Arc, pool: Arc) -> Result { + let migrated = graph::spawn_thread("database-setup", move || { + graph::block_on(self.setup(vec![pool.clone()])) + }) + .join() + // unwrap: propagate panics + .unwrap()?; + Ok(migrated == 1) + } + + /// Setup all pools by doing the following steps: + /// 1. Get the migration lock in the primary. This makes sure that only + /// one node runs migrations + /// 2. Remove the views in `sharded` as they might interfere with + /// running migrations + /// 3. In parallel, do the following in each pool: + /// 1. Configure fdw servers + /// 2. Run migrations in all pools in parallel + /// 4. In parallel, do the following in each pool: + /// 1. Create/update the mappings in `shard__subgraphs` and in + /// `primary_public` + /// 5. Create the views in `sharded` again + /// 6. Release the migration lock + /// + /// This method tolerates databases that are not available and will + /// simply ignore them. The returned count is the number of pools that + /// were successfully set up. + async fn setup(&self, pools: Vec>) -> Result { + type MigrationCounts = Vec<(Arc, MigrationCount)>; + + /// Filter out pools that are not available. We don't want to fail + /// because one of the pools is not available. We will just ignore + /// them and continue with the others. + fn filter_unavailable( + (pool, res): (Arc, Result), + ) -> Option> { + if let Err(StoreError::DatabaseUnavailable) = res { + error!( + pool.logger, + "migrations failed because database was unavailable" + ); + None + } else { + Some(res) + } + } + + /// Migrate all pools in parallel + async fn migrate( + pools: &[Arc], + servers: &[ForeignServer], + ) -> Result { + let futures = pools + .iter() + .map(|pool| { + pool.cheap_clone() + .migrate(servers) + .map(|res| (pool.cheap_clone(), res)) + }) + .collect::>(); + join_all(futures) + .await + .into_iter() + .filter_map(filter_unavailable) + .collect::, _>>() + } + + /// Propagate the schema changes to all other pools in parallel + async fn propagate( + this: &PoolCoordinator, + migrated: MigrationCounts, + ) -> Result { + let futures = migrated + .into_iter() + .map(|(pool, count)| async move { + let res = this.propagate(&pool, count); + (pool.cheap_clone(), res) + }) + .collect::>(); + join_all(futures) + .await + .into_iter() + .filter_map(filter_unavailable) + .collect::, _>>() + .map(|v| v.len()) + } + + let primary = self.primary()?; + + let mut pconn = primary.get().map_err(|_| StoreError::DatabaseUnavailable)?; + + // Everything here happens under the migration lock. Anything called + // from here should not try to get that lock, otherwise the process + // will deadlock + let res = with_migration_lock(&mut pconn, |_| async { + primary.drop_cross_shard_views()?; + + let migrated = migrate(&pools, self.servers.as_ref()).await?; + + let propagated = propagate(&self, migrated).await?; + + primary.create_cross_shard_views(&self.servers)?; + Ok(propagated) + }) + .await; + + res + } } From a63e607c844875b3d211b254ff5ed2dafd246df1 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Fri, 4 Apr 2025 11:41:49 -0700 Subject: [PATCH 133/150] node, store: Give the PoolCoordinator a logger --- node/src/bin/manager.rs | 2 +- node/src/store_builder.rs | 2 +- store/postgres/src/connection_pool.rs | 9 +++++++-- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/node/src/bin/manager.rs b/node/src/bin/manager.rs index 5142a2ab939..50ee9b61958 100644 --- a/node/src/bin/manager.rs +++ b/node/src/bin/manager.rs @@ -898,7 +898,7 @@ impl Context { fn primary_pool(self) -> ConnectionPool { let primary = self.config.primary_store(); - let coord = Arc::new(PoolCoordinator::new(Arc::new(vec![]))); + let coord = Arc::new(PoolCoordinator::new(&self.logger, Arc::new(vec![]))); let pool = StoreBuilder::main_pool( &self.logger, &self.node_id, diff --git a/node/src/store_builder.rs b/node/src/store_builder.rs index abaf59471fd..2d2e56dbc69 100644 --- a/node/src/store_builder.rs +++ b/node/src/store_builder.rs @@ -110,7 +110,7 @@ impl StoreBuilder { .collect::, _>>() .expect("connection url's contain enough detail"); let servers = Arc::new(servers); - let coord = Arc::new(PoolCoordinator::new(servers)); + let coord = Arc::new(PoolCoordinator::new(logger, servers)); let shards: Vec<_> = config .stores diff --git a/store/postgres/src/connection_pool.rs b/store/postgres/src/connection_pool.rs index 171fb8dbbb6..f03bd75ecb0 100644 --- a/store/postgres/src/connection_pool.rs +++ b/store/postgres/src/connection_pool.rs @@ -1336,13 +1336,16 @@ fn migrate_schema(logger: &Logger, conn: &mut PgConnection) -> Result>>, servers: Arc>, } impl PoolCoordinator { - pub fn new(servers: Arc>) -> Self { + pub fn new(logger: &Logger, servers: Arc>) -> Self { + let logger = logger.new(o!("component" => "ConnectionPool", "component" => "Coordinator")); Self { + logger, pools: Mutex::new(HashMap::new()), servers, } @@ -1581,7 +1584,9 @@ impl PoolCoordinator { // Everything here happens under the migration lock. Anything called // from here should not try to get that lock, otherwise the process // will deadlock + debug!(self.logger, "Waiting for migration lock"); let res = with_migration_lock(&mut pconn, |_| async { + debug!(self.logger, "Migration lock acquired"); primary.drop_cross_shard_views()?; let migrated = migrate(&pools, self.servers.as_ref()).await?; @@ -1592,7 +1597,7 @@ impl PoolCoordinator { Ok(propagated) }) .await; - + debug!(self.logger, "Database setup finished"); res } } From 3282af28a63038bc41a53074b088ba6146c52108 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Fri, 4 Apr 2025 12:35:14 -0700 Subject: [PATCH 134/150] store: Encapsulate mutable state tracking in PoolState Before, PoolState was just an enum and code all over the place dealt with its interior mutability. Now, we encapsulate that to simplify code using the PoolState --- store/postgres/src/connection_pool.rs | 173 ++++++++++++++++++-------- 1 file changed, 118 insertions(+), 55 deletions(-) diff --git a/store/postgres/src/connection_pool.rs b/store/postgres/src/connection_pool.rs index f03bd75ecb0..70caf4c49fc 100644 --- a/store/postgres/src/connection_pool.rs +++ b/store/postgres/src/connection_pool.rs @@ -10,6 +10,7 @@ use diesel_migrations::{EmbeddedMigrations, HarnessWithOutput}; use graph::cheap_clone::CheapClone; use graph::components::store::QueryPermit; use graph::constraint_violation; +use graph::derive::CheapClone; use graph::futures03::future::join_all; use graph::futures03::FutureExt as _; use graph::prelude::tokio::time::Instant; @@ -312,7 +313,17 @@ impl ForeignServer { /// them on idle. This is much shorter than the default of 10 minutes. const FDW_IDLE_TIMEOUT: Duration = Duration::from_secs(60); -/// A pool goes through several states, and this enum tracks what state we +enum PoolStateInner { + /// A connection pool, and all the servers for which we need to + /// establish fdw mappings when we call `setup` on the pool + Created(Arc, Arc), + /// The pool has been successfully set up + Ready(Arc), + /// The pool has been disabled by setting its size to 0 + Disabled(String), +} + +/// A pool goes through several states, and this struct tracks what state we /// are in, together with the `state_tracker` field on `ConnectionPool`. /// When first created, the pool is in state `Created`; once we successfully /// called `setup` on it, it moves to state `Ready`. During use, we use the @@ -322,20 +333,96 @@ const FDW_IDLE_TIMEOUT: Duration = Duration::from_secs(60); /// database connection. That avoids overall undesirable states like buildup /// of queries; instead of queueing them until the database is available, /// they return almost immediately with an error -enum PoolState { - /// A connection pool, and all the servers for which we need to - /// establish fdw mappings when we call `setup` on the pool - Created(Arc, Arc), - /// The pool has been successfully set up - Ready(Arc), - /// The pool has been disabled by setting its size to 0 - Disabled, +#[derive(Clone, CheapClone)] +struct PoolState { + logger: Logger, + inner: Arc>, } +impl PoolState { + fn new(logger: Logger, inner: PoolStateInner, name: String) -> Self { + let pool_name = format!("pool-{}", name); + Self { + logger, + inner: Arc::new(TimedMutex::new(inner, pool_name)), + } + } + + fn disabled(logger: Logger, name: &str) -> Self { + Self::new( + logger, + PoolStateInner::Disabled(name.to_string()), + name.to_string(), + ) + } + + fn created(pool: Arc, coord: Arc) -> Self { + let logger = pool.logger.clone(); + let name = pool.shard.to_string(); + let inner = PoolStateInner::Created(pool, coord); + Self::new(logger, inner, name) + } + + fn ready(pool: Arc) -> Self { + let logger = pool.logger.clone(); + let name = pool.shard.to_string(); + let inner = PoolStateInner::Ready(pool); + Self::new(logger, inner, name) + } + + fn set_ready(&self) { + use PoolStateInner::*; + + let mut guard = self.inner.lock(&self.logger); + match &*guard { + Created(pool, _) => *guard = Ready(pool.clone()), + Ready(_) | Disabled(_) => { /* nothing to do */ } + } + } + + /// Get a connection pool that is ready, i.e., has been through setup + /// and running migrations + fn get_ready(&self) -> Result, StoreError> { + let mut guard = self.inner.lock(&self.logger); + + use PoolStateInner::*; + match &*guard { + Created(pool, coord) => { + let migrated = coord.cheap_clone().setup_bg(pool.cheap_clone())?; + + if migrated { + let pool2 = pool.cheap_clone(); + *guard = Ready(pool.cheap_clone()); + Ok(pool2) + } else { + Err(StoreError::DatabaseUnavailable) + } + } + Ready(pool) => Ok(pool.clone()), + Disabled(name) => Err(constraint_violation!( + "tried to access disabled database pool `{}`", + name + )), + } + } + + /// Get the inner pool, regardless of whether it has been set up or not. + /// Most uses should use `get_ready` instead + fn get_unready(&self) -> Result, StoreError> { + use PoolStateInner::*; + + match &*self.inner.lock(&self.logger) { + Created(pool, _) | Ready(pool) => Ok(pool.cheap_clone()), + Disabled(name) => Err(constraint_violation!( + "tried to access disabled database pool `{}`", + name + )), + } + } +} #[derive(Clone)] pub struct ConnectionPool { - inner: Arc>, - logger: Logger, + inner: PoolState, pub shard: Shard, state_tracker: PoolStateTracker, } @@ -428,9 +515,9 @@ impl ConnectionPool { let state_tracker = PoolStateTracker::new(); let shard = Shard::new(shard_name.to_string()).expect("shard_name is a valid name for a shard"); - let pool_state = { + let inner = { if pool_size == 0 { - PoolState::Disabled + PoolState::disabled(logger.cheap_clone(), shard_name) } else { let pool = PoolInner::create( shard.clone(), @@ -443,15 +530,14 @@ impl ConnectionPool { state_tracker.clone(), ); if pool_name.is_replica() { - PoolState::Ready(Arc::new(pool)) + PoolState::ready(Arc::new(pool)) } else { - PoolState::Created(Arc::new(pool), coord) + PoolState::created(Arc::new(pool), coord) } } }; ConnectionPool { - inner: Arc::new(TimedMutex::new(pool_state, format!("pool-{}", shard_name))), - logger: logger.clone(), + inner, shard, state_tracker, } @@ -460,11 +546,7 @@ impl ConnectionPool { /// This is only used for `graphman` to ensure it doesn't run migrations /// or other setup steps pub fn skip_setup(&self) { - let mut guard = self.inner.lock(&self.logger); - match &*guard { - PoolState::Created(pool, _) => *guard = PoolState::Ready(pool.clone()), - PoolState::Ready(_) | PoolState::Disabled => { /* nothing to do */ } - } + self.inner.set_ready(); } /// Return a pool that is ready, i.e., connected to the database. If the @@ -472,7 +554,6 @@ impl ConnectionPool { /// or the pool is marked as unavailable, return /// `StoreError::DatabaseUnavailable` fn get_ready(&self) -> Result, StoreError> { - let mut guard = self.inner.lock(&self.logger); if !self.state_tracker.is_available() { // We know that trying to use this pool is pointless since the // database is not available, and will only lead to other @@ -481,21 +562,12 @@ impl ConnectionPool { return Err(StoreError::DatabaseUnavailable); } - match &*guard { - PoolState::Created(pool, coord) => { - let migrated = coord.cheap_clone().setup_bg(pool.cheap_clone())?; - - if migrated { - let pool2 = pool.clone(); - *guard = PoolState::Ready(pool.clone()); - self.state_tracker.mark_available(); - Ok(pool2) - } else { - Err(StoreError::DatabaseUnavailable) - } + match self.inner.get_ready() { + Ok(pool) => { + self.state_tracker.mark_available(); + Ok(pool) } - PoolState::Ready(pool) => Ok(pool.clone()), - PoolState::Disabled => Err(StoreError::DatabaseDisabled), + Err(e) => Err(e), } } @@ -589,12 +661,7 @@ impl ConnectionPool { } pub(crate) async fn query_permit(&self) -> Result { - let pool = match &*self.inner.lock(&self.logger) { - PoolState::Created(pool, _) | PoolState::Ready(pool) => pool.clone(), - PoolState::Disabled => { - return Err(StoreError::DatabaseDisabled); - } - }; + let pool = self.inner.get_unready()?; let start = Instant::now(); let permit = pool.query_permit().await; Ok(QueryPermit { @@ -604,10 +671,9 @@ impl ConnectionPool { } pub(crate) fn wait_stats(&self) -> Result { - match &*self.inner.lock(&self.logger) { - PoolState::Created(pool, _) | PoolState::Ready(pool) => Ok(pool.wait_stats.clone()), - PoolState::Disabled => Err(StoreError::DatabaseDisabled), - } + self.inner + .get_unready() + .map(|pool| pool.wait_stats.cheap_clone()) } /// Mirror key tables from the primary into our own schema. We do this @@ -1381,14 +1447,11 @@ impl PoolCoordinator { // yet. We remember the `PoolInner` so that later, when we have to // call `remap()`, we do not have to take this lock as that will be // already held in `get_ready()` - match &*pool.inner.lock(logger) { - PoolState::Created(inner, _) | PoolState::Ready(inner) => { - self.pools - .lock() - .unwrap() - .insert(pool.shard.clone(), inner.clone()); - } - PoolState::Disabled => { /* nothing to do */ } + if let Some(inner) = pool.inner.get_unready().ok() { + self.pools + .lock() + .unwrap() + .insert(pool.shard.clone(), inner.clone()); } } pool From 70656a378172e8aca86a8fa1c803a8d0868d9c56 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Fri, 4 Apr 2025 12:37:21 -0700 Subject: [PATCH 135/150] node, store: Rename 'PoolName' to 'PoolRole' --- node/src/store_builder.rs | 6 +++--- store/postgres/src/connection_pool.rs | 18 +++++++++--------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/node/src/store_builder.rs b/node/src/store_builder.rs index 2d2e56dbc69..27dc7d5d021 100644 --- a/node/src/store_builder.rs +++ b/node/src/store_builder.rs @@ -8,7 +8,7 @@ use graph::{ util::security::SafeDisplay, }; use graph_store_postgres::connection_pool::{ - ConnectionPool, ForeignServer, PoolCoordinator, PoolName, + ConnectionPool, ForeignServer, PoolCoordinator, PoolRole, }; use graph_store_postgres::{ BlockStore as DieselBlockStore, ChainHeadUpdateListener as PostgresChainHeadUpdateListener, @@ -224,7 +224,7 @@ impl StoreBuilder { coord.create_pool( &logger, name, - PoolName::Main, + PoolRole::Main, shard.connection.clone(), pool_size, Some(fdw_pool_size), @@ -264,7 +264,7 @@ impl StoreBuilder { coord.clone().create_pool( &logger, name, - PoolName::Replica(pool), + PoolRole::Replica(pool), replica.connection.clone(), pool_size, None, diff --git a/store/postgres/src/connection_pool.rs b/store/postgres/src/connection_pool.rs index 70caf4c49fc..fd8c26204af 100644 --- a/store/postgres/src/connection_pool.rs +++ b/store/postgres/src/connection_pool.rs @@ -435,27 +435,27 @@ impl fmt::Debug for ConnectionPool { } } -/// The name of the pool, mostly for logging, and what purpose it serves. +/// The role of the pool, mostly for logging, and what purpose it serves. /// The main pool will always be called `main`, and can be used for reading /// and writing. Replica pools can only be used for reading, and don't /// require any setup (migrations etc.) -pub enum PoolName { +pub enum PoolRole { Main, Replica(String), } -impl PoolName { +impl PoolRole { fn as_str(&self) -> &str { match self { - PoolName::Main => "main", - PoolName::Replica(name) => name, + PoolRole::Main => "main", + PoolRole::Replica(name) => name, } } fn is_replica(&self) -> bool { match self { - PoolName::Main => false, - PoolName::Replica(_) => true, + PoolRole::Main => false, + PoolRole::Replica(_) => true, } } } @@ -504,7 +504,7 @@ impl PoolStateTracker { impl ConnectionPool { fn create( shard_name: &str, - pool_name: PoolName, + pool_name: PoolRole, postgres_url: String, pool_size: u32, fdw_pool_size: Option, @@ -1421,7 +1421,7 @@ impl PoolCoordinator { self: Arc, logger: &Logger, name: &str, - pool_name: PoolName, + pool_name: PoolRole, postgres_url: String, pool_size: u32, fdw_pool_size: Option, From 67108356b0614fd128a17c21f908524b82cca884 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Fri, 4 Apr 2025 15:26:54 -0700 Subject: [PATCH 136/150] all: Filter out shards with pool size 0 Instead of dealing with disabled shards (shards that have a pool size of 0 configured), filter those shards out on startup and warn about them. The end effect is that for that configuration, users will get an error of 'unkown shard' rather than 'shard disabled'. Since configuring a shard to have no connections is kinda pathological, and leads to an error when it is used either way, the code simplification is worth the slightly less helpful error message. Removing the 'disabled' state from pools has ripple effects to quite a few other places, simplifying them a bit --- graph/src/components/store/traits.rs | 6 +- graph/src/data/query/trace.rs | 7 +- graphql/src/execution/resolver.rs | 2 +- graphql/src/introspection/resolver.rs | 2 +- graphql/src/runner.rs | 2 +- graphql/src/store/resolver.rs | 4 +- node/src/store_builder.rs | 20 ++++- server/index-node/src/resolver.rs | 4 +- store/postgres/src/block_store.rs | 6 +- store/postgres/src/connection_pool.rs | 79 +++++++------------ store/postgres/src/deployment_store.rs | 4 +- store/postgres/src/query_store.rs | 4 +- store/postgres/src/store.rs | 4 +- .../test-store/tests/graphql/introspection.rs | 6 +- 14 files changed, 67 insertions(+), 83 deletions(-) diff --git a/graph/src/components/store/traits.rs b/graph/src/components/store/traits.rs index 27cb3768e2c..73cb22269fe 100644 --- a/graph/src/components/store/traits.rs +++ b/graph/src/components/store/traits.rs @@ -655,7 +655,7 @@ pub trait QueryStore: Send + Sync { block_hash: &BlockHash, ) -> Result, Option)>, StoreError>; - fn wait_stats(&self) -> Result; + fn wait_stats(&self) -> PoolWaitStats; /// Find the current state for the subgraph deployment `id` and /// return details about it needed for executing queries @@ -668,7 +668,7 @@ pub trait QueryStore: Send + Sync { fn network_name(&self) -> &str; /// A permit should be acquired before starting query execution. - async fn query_permit(&self) -> Result; + async fn query_permit(&self) -> QueryPermit; /// Report the name of the shard in which the subgraph is stored. This /// should only be used for reporting and monitoring @@ -683,7 +683,7 @@ pub trait QueryStore: Send + Sync { #[async_trait] pub trait StatusStore: Send + Sync + 'static { /// A permit should be acquired before starting query execution. - async fn query_permit(&self) -> Result; + async fn query_permit(&self) -> QueryPermit; fn status(&self, filter: status::Filter) -> Result, StoreError>; diff --git a/graph/src/data/query/trace.rs b/graph/src/data/query/trace.rs index cf2d153dca4..256c9cdeaf6 100644 --- a/graph/src/data/query/trace.rs +++ b/graph/src/data/query/trace.rs @@ -118,11 +118,8 @@ impl Trace { } } - pub fn query_done(&mut self, dur: Duration, permit: &Result) { - let permit_dur = match permit { - Ok(permit) => permit.wait, - Err(_) => Duration::from_millis(0), - }; + pub fn query_done(&mut self, dur: Duration, permit: &QueryPermit) { + let permit_dur = permit.wait; match self { Trace::None => { /* nothing to do */ } Trace::Root { .. } => { diff --git a/graphql/src/execution/resolver.rs b/graphql/src/execution/resolver.rs index ca59e401dfc..0074eb124d8 100644 --- a/graphql/src/execution/resolver.rs +++ b/graphql/src/execution/resolver.rs @@ -18,7 +18,7 @@ use super::Query; pub trait Resolver: Sized + Send + Sync + 'static { const CACHEABLE: bool; - async fn query_permit(&self) -> Result; + async fn query_permit(&self) -> QueryPermit; /// Prepare for executing a query by prefetching as much data as possible fn prefetch( diff --git a/graphql/src/introspection/resolver.rs b/graphql/src/introspection/resolver.rs index 0f67b717c5a..765b0399695 100644 --- a/graphql/src/introspection/resolver.rs +++ b/graphql/src/introspection/resolver.rs @@ -356,7 +356,7 @@ impl Resolver for IntrospectionResolver { // see `fn as_introspection_context`, so this value is irrelevant. const CACHEABLE: bool = false; - async fn query_permit(&self) -> Result { + async fn query_permit(&self) -> QueryPermit { unreachable!() } diff --git a/graphql/src/runner.rs b/graphql/src/runner.rs index 96f30e8bc9d..d2f0bc9c96c 100644 --- a/graphql/src/runner.rs +++ b/graphql/src/runner.rs @@ -143,7 +143,7 @@ where )?; self.load_manager .decide( - &store.wait_stats().map_err(QueryExecutionError::from)?, + &store.wait_stats(), store.shard(), store.deployment_id(), query.shape_hash, diff --git a/graphql/src/store/resolver.rs b/graphql/src/store/resolver.rs index 82c40420fa6..d7032740768 100644 --- a/graphql/src/store/resolver.rs +++ b/graphql/src/store/resolver.rs @@ -256,8 +256,8 @@ impl StoreResolver { impl Resolver for StoreResolver { const CACHEABLE: bool = true; - async fn query_permit(&self) -> Result { - self.store.query_permit().await.map_err(Into::into) + async fn query_permit(&self) -> QueryPermit { + self.store.query_permit().await } fn prefetch( diff --git a/node/src/store_builder.rs b/node/src/store_builder.rs index 27dc7d5d021..5294179f8eb 100644 --- a/node/src/store_builder.rs +++ b/node/src/store_builder.rs @@ -2,6 +2,7 @@ use std::iter::FromIterator; use std::{collections::HashMap, sync::Arc}; use graph::prelude::{o, MetricsRegistry, NodeId}; +use graph::slog::warn; use graph::url::Url; use graph::{ prelude::{info, CheapClone, Logger}, @@ -115,8 +116,23 @@ impl StoreBuilder { let shards: Vec<_> = config .stores .iter() - .map(|(name, shard)| { + .filter_map(|(name, shard)| { let logger = logger.new(o!("shard" => name.to_string())); + let pool_size = shard.pool_size.size_for(node, name).unwrap_or_else(|_| { + panic!("cannot determine the pool size for store {}", name) + }); + if pool_size == 0 { + if name == PRIMARY_SHARD.as_str() { + panic!("pool size for primary shard must be greater than 0"); + } else { + warn!( + logger, + "pool size for shard {} is 0, ignoring this shard", name + ); + return None; + } + } + let conn_pool = Self::main_pool( &logger, node, @@ -137,7 +153,7 @@ impl StoreBuilder { let name = ShardName::new(name.to_string()).expect("shard names have been validated"); - (name, conn_pool, read_only_conn_pools, weights) + Some((name, conn_pool, read_only_conn_pools, weights)) }) .collect(); diff --git a/server/index-node/src/resolver.rs b/server/index-node/src/resolver.rs index a60e5d35fd9..7974afe41db 100644 --- a/server/index-node/src/resolver.rs +++ b/server/index-node/src/resolver.rs @@ -777,8 +777,8 @@ fn entity_changes_to_graphql(entity_changes: Vec) -> r::Value { impl Resolver for IndexNodeResolver { const CACHEABLE: bool = false; - async fn query_permit(&self) -> Result { - self.store.query_permit().await.map_err(Into::into) + async fn query_permit(&self) -> QueryPermit { + self.store.query_permit().await } fn prefetch( diff --git a/store/postgres/src/block_store.rs b/store/postgres/src/block_store.rs index 9af40b8d2a0..f69267fff17 100644 --- a/store/postgres/src/block_store.rs +++ b/store/postgres/src/block_store.rs @@ -319,11 +319,7 @@ impl BlockStore { } pub(crate) async fn query_permit_primary(&self) -> QueryPermit { - self.mirror - .primary() - .query_permit() - .await - .expect("the primary is never disabled") + self.mirror.primary().query_permit().await } pub fn allocate_chain( diff --git a/store/postgres/src/connection_pool.rs b/store/postgres/src/connection_pool.rs index fd8c26204af..c2f5bc95a9c 100644 --- a/store/postgres/src/connection_pool.rs +++ b/store/postgres/src/connection_pool.rs @@ -319,8 +319,6 @@ enum PoolStateInner { Created(Arc, Arc), /// The pool has been successfully set up Ready(Arc), - /// The pool has been disabled by setting its size to 0 - Disabled(String), } /// A pool goes through several states, and this struct tracks what state we @@ -348,14 +346,6 @@ impl PoolState { } } - fn disabled(logger: Logger, name: &str) -> Self { - Self::new( - logger, - PoolStateInner::Disabled(name.to_string()), - name.to_string(), - ) - } - fn created(pool: Arc, coord: Arc) -> Self { let logger = pool.logger.clone(); let name = pool.shard.to_string(); @@ -376,7 +366,7 @@ impl PoolState { let mut guard = self.inner.lock(&self.logger); match &*guard { Created(pool, _) => *guard = Ready(pool.clone()), - Ready(_) | Disabled(_) => { /* nothing to do */ } + Ready(_) => { /* nothing to do */ } } } @@ -399,24 +389,16 @@ impl PoolState { } } Ready(pool) => Ok(pool.clone()), - Disabled(name) => Err(constraint_violation!( - "tried to access disabled database pool `{}`", - name - )), } } /// Get the inner pool, regardless of whether it has been set up or not. /// Most uses should use `get_ready` instead - fn get_unready(&self) -> Result, StoreError> { + fn get_unready(&self) -> Arc { use PoolStateInner::*; match &*self.inner.lock(&self.logger) { - Created(pool, _) | Ready(pool) => Ok(pool.cheap_clone()), - Disabled(name) => Err(constraint_violation!( - "tried to access disabled database pool `{}`", - name - )), + Created(pool, _) | Ready(pool) => pool.cheap_clone(), } } } @@ -516,24 +498,20 @@ impl ConnectionPool { let shard = Shard::new(shard_name.to_string()).expect("shard_name is a valid name for a shard"); let inner = { - if pool_size == 0 { - PoolState::disabled(logger.cheap_clone(), shard_name) + let pool = PoolInner::create( + shard.clone(), + pool_name.as_str(), + postgres_url, + pool_size, + fdw_pool_size, + logger, + registry, + state_tracker.clone(), + ); + if pool_name.is_replica() { + PoolState::ready(Arc::new(pool)) } else { - let pool = PoolInner::create( - shard.clone(), - pool_name.as_str(), - postgres_url, - pool_size, - fdw_pool_size, - logger, - registry, - state_tracker.clone(), - ); - if pool_name.is_replica() { - PoolState::ready(Arc::new(pool)) - } else { - PoolState::created(Arc::new(pool), coord) - } + PoolState::created(Arc::new(pool), coord) } }; ConnectionPool { @@ -660,20 +638,18 @@ impl ConnectionPool { .ignore_timeout(|| inner.try_get_fdw(logger, timeout)) } - pub(crate) async fn query_permit(&self) -> Result { - let pool = self.inner.get_unready()?; + pub(crate) async fn query_permit(&self) -> QueryPermit { + let pool = self.inner.get_unready(); let start = Instant::now(); let permit = pool.query_permit().await; - Ok(QueryPermit { + QueryPermit { permit, wait: start.elapsed(), - }) + } } - pub(crate) fn wait_stats(&self) -> Result { - self.inner - .get_unready() - .map(|pool| pool.wait_stats.cheap_clone()) + pub(crate) fn wait_stats(&self) -> PoolWaitStats { + self.inner.get_unready().wait_stats.cheap_clone() } /// Mirror key tables from the primary into our own schema. We do this @@ -1447,12 +1423,11 @@ impl PoolCoordinator { // yet. We remember the `PoolInner` so that later, when we have to // call `remap()`, we do not have to take this lock as that will be // already held in `get_ready()` - if let Some(inner) = pool.inner.get_unready().ok() { - self.pools - .lock() - .unwrap() - .insert(pool.shard.clone(), inner.clone()); - } + let inner = pool.inner.get_unready(); + self.pools + .lock() + .unwrap() + .insert(pool.shard.clone(), inner.clone()); } pool } diff --git a/store/postgres/src/deployment_store.rs b/store/postgres/src/deployment_store.rs index 96dd5507f3e..91230d63b7b 100644 --- a/store/postgres/src/deployment_store.rs +++ b/store/postgres/src/deployment_store.rs @@ -415,7 +415,7 @@ impl DeploymentStore { Ok(conn) } - pub(crate) async fn query_permit(&self, replica: ReplicaId) -> Result { + pub(crate) async fn query_permit(&self, replica: ReplicaId) -> QueryPermit { let pool = match replica { ReplicaId::Main => &self.pool, ReplicaId::ReadOnly(idx) => &self.read_only_pools[idx], @@ -423,7 +423,7 @@ impl DeploymentStore { pool.query_permit().await } - pub(crate) fn wait_stats(&self, replica: ReplicaId) -> Result { + pub(crate) fn wait_stats(&self, replica: ReplicaId) -> PoolWaitStats { match replica { ReplicaId::Main => self.pool.wait_stats(), ReplicaId::ReadOnly(idx) => self.read_only_pools[idx].wait_stats(), diff --git a/store/postgres/src/query_store.rs b/store/postgres/src/query_store.rs index 8fc2da822e4..fe7d084030b 100644 --- a/store/postgres/src/query_store.rs +++ b/store/postgres/src/query_store.rs @@ -112,7 +112,7 @@ impl QueryStoreTrait for QueryStore { self.chain_store.block_numbers(block_hashes).await } - fn wait_stats(&self) -> Result { + fn wait_stats(&self) -> PoolWaitStats { self.store.wait_stats(self.replica_id) } @@ -137,7 +137,7 @@ impl QueryStoreTrait for QueryStore { &self.site.network } - async fn query_permit(&self) -> Result { + async fn query_permit(&self) -> QueryPermit { self.store.query_permit(self.replica_id).await } diff --git a/store/postgres/src/store.rs b/store/postgres/src/store.rs index 50a5e4b21e0..7eb428a5058 100644 --- a/store/postgres/src/store.rs +++ b/store/postgres/src/store.rs @@ -167,8 +167,8 @@ impl StatusStore for Store { .await } - async fn query_permit(&self) -> Result { + async fn query_permit(&self) -> QueryPermit { // Status queries go to the primary shard. - Ok(self.block_store.query_permit_primary().await) + self.block_store.query_permit_primary().await } } diff --git a/store/test-store/tests/graphql/introspection.rs b/store/test-store/tests/graphql/introspection.rs index 6139e673767..8bc76213e6b 100644 --- a/store/test-store/tests/graphql/introspection.rs +++ b/store/test-store/tests/graphql/introspection.rs @@ -53,15 +53,15 @@ impl Resolver for MockResolver { Ok(r::Value::Null) } - async fn query_permit(&self) -> Result { + async fn query_permit(&self) -> QueryPermit { let permit = Arc::new(tokio::sync::Semaphore::new(1)) .acquire_owned() .await .unwrap(); - Ok(QueryPermit { + QueryPermit { permit, wait: Duration::from_secs(0), - }) + } } } From eb6fae71fd9a05c38b779a97b0bb3de8f17edd7b Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Fri, 4 Apr 2025 16:15:33 -0700 Subject: [PATCH 137/150] store: Make sure we do not run setup twice for the same pool With the previous code, we would run setup initially when creating all pools, but they would not be marked as set up. On the first access to the pool we would try to run setup again, which is not needed. This change makes it so that we remember that we ran setup successfully when pools are created --- store/postgres/src/connection_pool.rs | 132 ++++++++++++++++---------- 1 file changed, 84 insertions(+), 48 deletions(-) diff --git a/store/postgres/src/connection_pool.rs b/store/postgres/src/connection_pool.rs index c2f5bc95a9c..9a6afe9a37e 100644 --- a/store/postgres/src/connection_pool.rs +++ b/store/postgres/src/connection_pool.rs @@ -373,22 +373,27 @@ impl PoolState { /// Get a connection pool that is ready, i.e., has been through setup /// and running migrations fn get_ready(&self) -> Result, StoreError> { - let mut guard = self.inner.lock(&self.logger); + // We have to be careful here that we do not hold a lock when we + // call `setup_bg`, otherwise we will deadlock + let (pool, coord) = { + let guard = self.inner.lock(&self.logger); + + use PoolStateInner::*; + match &*guard { + Created(pool, coord) => (pool.cheap_clone(), coord.cheap_clone()), + Ready(pool) => return Ok(pool.clone()), + } + }; - use PoolStateInner::*; - match &*guard { - Created(pool, coord) => { - let migrated = coord.cheap_clone().setup_bg(pool.cheap_clone())?; + // self is `Created` and needs to have setup run + coord.setup_bg(self.cheap_clone())?; - if migrated { - let pool2 = pool.cheap_clone(); - *guard = Ready(pool.cheap_clone()); - Ok(pool2) - } else { - Err(StoreError::DatabaseUnavailable) - } - } - Ready(pool) => Ok(pool.clone()), + // We just tried to set up the pool; if it is still not set up and + // we didn't have an error, it means the database is not available + if self.needs_setup() { + return Err(StoreError::DatabaseUnavailable); + } else { + Ok(pool) } } @@ -401,6 +406,16 @@ impl PoolState { Created(pool, _) | Ready(pool) => pool.cheap_clone(), } } + + fn needs_setup(&self) -> bool { + let guard = self.inner.lock(&self.logger); + + use PoolStateInner::*; + match &*guard { + Created(_, _) => true, + Ready(_) => false, + } + } } #[derive(Clone)] pub struct ConnectionPool { @@ -1186,7 +1201,7 @@ impl PoolInner { async fn migrate( self: Arc, servers: &[ForeignServer], - ) -> Result<(Arc, MigrationCount), StoreError> { + ) -> Result { self.configure_fdw(servers)?; let mut conn = self.get()?; let (this, count) = conn.transaction(|conn| -> Result<_, StoreError> { @@ -1196,7 +1211,7 @@ impl PoolInner { this.locale_check(&this.logger, conn)?; - Ok((this, count)) + Ok(count) } /// If this is the primary shard, drop the namespace `CROSS_SHARD_NSP` @@ -1379,7 +1394,7 @@ fn migrate_schema(logger: &Logger, conn: &mut PgConnection) -> Result>>, + pools: Mutex>, servers: Arc>, } @@ -1419,16 +1434,12 @@ impl PoolCoordinator { // Ignore non-writable pools (replicas), there is no need (and no // way) to coordinate schema changes with them if is_writable { - // It is safe to take this lock here since nobody has seen the pool - // yet. We remember the `PoolInner` so that later, when we have to - // call `remap()`, we do not have to take this lock as that will be - // already held in `get_ready()` - let inner = pool.inner.get_unready(); self.pools .lock() .unwrap() - .insert(pool.shard.clone(), inner.clone()); + .insert(pool.shard.clone(), pool.inner.cheap_clone()); } + pool } @@ -1460,6 +1471,7 @@ impl PoolCoordinator { if count.had_migrations() { let server = self.server(&pool.shard)?; for pool in self.pools.lock().unwrap().values() { + let pool = pool.get_unready(); let remap_res = pool.remap(server); if let Err(e) = remap_res { error!(pool.logger, "Failed to map imports from {}", server.shard; "error" => e.to_string()); @@ -1470,8 +1482,15 @@ impl PoolCoordinator { Ok(()) } + /// Return a list of all pools, regardless of whether they are ready or + /// not. pub fn pools(&self) -> Vec> { - self.pools.lock().unwrap().values().cloned().collect() + self.pools + .lock() + .unwrap() + .values() + .map(|state| state.get_unready()) + .collect::>() } pub fn servers(&self) -> Arc> { @@ -1486,14 +1505,12 @@ impl PoolCoordinator { } fn primary(&self) -> Result, StoreError> { - self.pools - .lock() - .unwrap() - .get(&*PRIMARY_SHARD) - .cloned() - .ok_or_else(|| { - constraint_violation!("internal error: primary shard not found in pool coordinator") - }) + let map = self.pools.lock().unwrap(); + let pool_state = map.get(&*&PRIMARY_SHARD).ok_or_else(|| { + constraint_violation!("internal error: primary shard not found in pool coordinator") + })?; + + Ok(pool_state.get_unready()) } /// Setup all pools the coordinator knows about and return the number of @@ -1528,7 +1545,7 @@ impl PoolCoordinator { /// A helper to call `setup` from a non-async context. Returns `true` if /// the setup was actually run, i.e. if `pool` was available - fn setup_bg(self: Arc, pool: Arc) -> Result { + fn setup_bg(self: Arc, pool: PoolState) -> Result { let migrated = graph::spawn_thread("database-setup", move || { graph::block_on(self.setup(vec![pool.clone()])) }) @@ -1555,37 +1572,43 @@ impl PoolCoordinator { /// This method tolerates databases that are not available and will /// simply ignore them. The returned count is the number of pools that /// were successfully set up. - async fn setup(&self, pools: Vec>) -> Result { - type MigrationCounts = Vec<(Arc, MigrationCount)>; + /// + /// When this method returns, the entries from `states` that were + /// successfully set up will be marked as ready. The method returns the + /// number of pools that were set up + async fn setup(&self, states: Vec) -> Result { + type MigrationCounts = Vec<(PoolState, MigrationCount)>; /// Filter out pools that are not available. We don't want to fail /// because one of the pools is not available. We will just ignore /// them and continue with the others. fn filter_unavailable( - (pool, res): (Arc, Result), - ) -> Option> { + (state, res): (PoolState, Result), + ) -> Option> { if let Err(StoreError::DatabaseUnavailable) = res { error!( - pool.logger, + state.logger, "migrations failed because database was unavailable" ); None } else { - Some(res) + Some(res.map(|count| (state, count))) } } /// Migrate all pools in parallel async fn migrate( - pools: &[Arc], + pools: &[PoolState], servers: &[ForeignServer], ) -> Result { let futures = pools .iter() - .map(|pool| { - pool.cheap_clone() + .map(|state| { + state + .get_unready() + .cheap_clone() .migrate(servers) - .map(|res| (pool.cheap_clone(), res)) + .map(|res| (state.cheap_clone(), res)) }) .collect::>(); join_all(futures) @@ -1599,26 +1622,32 @@ impl PoolCoordinator { async fn propagate( this: &PoolCoordinator, migrated: MigrationCounts, - ) -> Result { + ) -> Result, StoreError> { let futures = migrated .into_iter() - .map(|(pool, count)| async move { + .map(|(state, count)| async move { + let pool = state.get_unready(); let res = this.propagate(&pool, count); - (pool.cheap_clone(), res) + (state.cheap_clone(), res) }) .collect::>(); join_all(futures) .await .into_iter() .filter_map(filter_unavailable) + .map(|res| res.map(|(state, ())| state)) .collect::, _>>() - .map(|v| v.len()) } let primary = self.primary()?; let mut pconn = primary.get().map_err(|_| StoreError::DatabaseUnavailable)?; + let pools: Vec<_> = states + .into_iter() + .filter(|pool| pool.needs_setup()) + .collect(); + // Everything here happens under the migration lock. Anything called // from here should not try to get that lock, otherwise the process // will deadlock @@ -1636,6 +1665,13 @@ impl PoolCoordinator { }) .await; debug!(self.logger, "Database setup finished"); - res + + // Mark all pool states that we set up completely as ready + res.map(|states| { + for state in &states { + state.set_ready(); + } + states.len() + }) } } From c23ee969d1d420d0cb331020a3ba335adce27799 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Sun, 6 Apr 2025 11:14:45 -0700 Subject: [PATCH 138/150] store: Avoid running setup unnecessarily if several threads try to run it --- store/postgres/src/connection_pool.rs | 33 +++++++++++++++++++-------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/store/postgres/src/connection_pool.rs b/store/postgres/src/connection_pool.rs index 9a6afe9a37e..6ff46649494 100644 --- a/store/postgres/src/connection_pool.rs +++ b/store/postgres/src/connection_pool.rs @@ -1643,10 +1643,13 @@ impl PoolCoordinator { let mut pconn = primary.get().map_err(|_| StoreError::DatabaseUnavailable)?; - let pools: Vec<_> = states + let states: Vec<_> = states .into_iter() .filter(|pool| pool.needs_setup()) .collect(); + if states.is_empty() { + return Ok(0); + } // Everything here happens under the migration lock. Anything called // from here should not try to get that lock, otherwise the process @@ -1654,24 +1657,34 @@ impl PoolCoordinator { debug!(self.logger, "Waiting for migration lock"); let res = with_migration_lock(&mut pconn, |_| async { debug!(self.logger, "Migration lock acquired"); + + // While we were waiting for the migration lock, another thread + // might have already run this + let states: Vec<_> = states + .into_iter() + .filter(|pool| pool.needs_setup()) + .collect(); + if states.is_empty() { + debug!(self.logger, "No pools to set up"); + return Ok(0); + } + primary.drop_cross_shard_views()?; - let migrated = migrate(&pools, self.servers.as_ref()).await?; + let migrated = migrate(&states, self.servers.as_ref()).await?; let propagated = propagate(&self, migrated).await?; primary.create_cross_shard_views(&self.servers)?; - Ok(propagated) - }) - .await; - debug!(self.logger, "Database setup finished"); - // Mark all pool states that we set up completely as ready - res.map(|states| { - for state in &states { + for state in &propagated { state.set_ready(); } - states.len() + Ok(propagated.len()) }) + .await; + debug!(self.logger, "Database setup finished"); + + res } } From 4be64c16f575c9da424ac730e26a497f829683ee Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Mon, 7 Apr 2025 17:17:43 -0700 Subject: [PATCH 139/150] store: Do not create aggregate indexes twice When index creation was postponed, we would create aggregate indexes twice. They could also be postponed, but we'll leave that for another day. --- store/postgres/src/relational/ddl.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/store/postgres/src/relational/ddl.rs b/store/postgres/src/relational/ddl.rs index e85281a5899..a3c4ed6885e 100644 --- a/store/postgres/src/relational/ddl.rs +++ b/store/postgres/src/relational/ddl.rs @@ -422,8 +422,9 @@ impl Table { } } else { self.create_attribute_indexes(out)?; + self.create_aggregate_indexes(schema, out)?; } - self.create_aggregate_indexes(schema, out) + Ok(()) } pub fn exclusion_ddl(&self, out: &mut String) -> fmt::Result { From 784150edb9b2de9b77d5ba449a5cb724ea8b230b Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Mon, 7 Apr 2025 09:17:26 -0700 Subject: [PATCH 140/150] store: Use a fdw connection for copy::is_source --- store/postgres/src/copy.rs | 25 +++++++++++++++++++++---- store/postgres/src/deployment_store.rs | 3 +-- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/store/postgres/src/copy.rs b/store/postgres/src/copy.rs index 9e9ba187c6a..e0ae71eab3e 100644 --- a/store/postgres/src/copy.rs +++ b/store/postgres/src/copy.rs @@ -37,7 +37,7 @@ use graph::{ info, lazy_static, o, warn, BlockNumber, BlockPtr, CheapClone, Logger, StoreError, ENV_VARS, }, schema::EntityType, - slog::error, + slog::{debug, error}, }; use itertools::Itertools; @@ -113,16 +113,33 @@ table! { } /// Return `true` if the site is the source of a copy operation. The copy -/// operation might be just queued or in progress already -pub fn is_source(conn: &mut PgConnection, site: &Site) -> Result { +/// operation might be just queued or in progress already. This method will +/// block until a fdw connection becomes available. +pub fn is_source(logger: &Logger, pool: &ConnectionPool, site: &Site) -> Result { use active_copies as ac; + // We use a fdw connection to check if the site is being copied. If we + // used an ordinary connection and there are many calls to this method, + // postgres_fdw might open an unmanageable number of connections into + // the primary, which makes the primary run out of connections + let mut last_log = Instant::now(); + let mut conn = pool.get_fdw(&logger, || { + if last_log.elapsed() > LOG_INTERVAL { + last_log = Instant::now(); + debug!( + logger, + "Waiting for fdw connection to check if site {} is being copied", site.namespace + ); + } + false + })?; + select(diesel::dsl::exists( ac::table .filter(ac::src.eq(site.id)) .filter(ac::cancelled_at.is_null()), )) - .get_result::(conn) + .get_result::(&mut conn) .map_err(StoreError::from) } diff --git a/store/postgres/src/deployment_store.rs b/store/postgres/src/deployment_store.rs index 91230d63b7b..c78b06be46d 100644 --- a/store/postgres/src/deployment_store.rs +++ b/store/postgres/src/deployment_store.rs @@ -1235,8 +1235,7 @@ impl DeploymentStore { req: PruneRequest, ) -> Result<(), StoreError> { { - let mut conn = store.get_conn()?; - if copy::is_source(&mut conn, &site)? { + if copy::is_source(&logger, &store.pool, &site)? { debug!( logger, "Skipping pruning since this deployment is being copied" From 1a2aaf3fe07c78ec8e1b9f18022bf829a54bc48d Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Tue, 1 Apr 2025 14:07:49 -0700 Subject: [PATCH 141/150] store: Try to spawn copy workers more frequently We used to only try to spawn a new copy worker when we were finished copying a table. But that can take a long time, and in between some connections might have become available. We now check every few minutes if a connection is available and spawn a new worker if it is --- store/postgres/src/copy.rs | 113 +++++++++++++++++++++++++++++-------- 1 file changed, 89 insertions(+), 24 deletions(-) diff --git a/store/postgres/src/copy.rs b/store/postgres/src/copy.rs index e0ae71eab3e..effe2950ee2 100644 --- a/store/postgres/src/copy.rs +++ b/store/postgres/src/copy.rs @@ -32,12 +32,13 @@ use diesel::{ }; use graph::{ constraint_violation, - futures03::future::select_all, + futures03::{future::select_all, FutureExt as _}, prelude::{ info, lazy_static, o, warn, BlockNumber, BlockPtr, CheapClone, Logger, StoreError, ENV_VARS, }, schema::EntityType, slog::{debug, error}, + tokio, }; use itertools::Itertools; @@ -687,6 +688,21 @@ impl CopyProgress { } } +enum WorkerResult { + Ok(CopyTableWorker), + Err(StoreError), + Wake, +} + +impl From> for WorkerResult { + fn from(result: Result) -> Self { + match result { + Ok(worker) => WorkerResult::Ok(worker), + Err(e) => WorkerResult::Err(e), + } + } +} + /// A helper to run copying of one table. We need to thread `conn` and /// `table` from the control loop to the background worker and back again to /// the control loop. This worker facilitates that @@ -705,11 +721,7 @@ impl CopyTableWorker { } } - async fn run( - mut self, - logger: Logger, - progress: Arc, - ) -> Result { + async fn run(mut self, logger: Logger, progress: Arc) -> WorkerResult { let object = self.table.dst.object.cheap_clone(); graph::spawn_blocking_allow_panic(move || { self.result = self.run_inner(logger, &progress); @@ -717,6 +729,7 @@ impl CopyTableWorker { }) .await .map_err(|e| constraint_violation!("copy worker for {} panicked: {}", object, e)) + .into() } fn run_inner(&mut self, logger: Logger, progress: &CopyProgress) -> Result { @@ -812,6 +825,57 @@ impl CopyTableWorker { } } +/// A helper to manage the workers that are copying data. Besides the actual +/// workers it also keeps a worker that wakes us up periodically to give us +/// a chance to create more workers if there are database connections +/// available +struct Workers { + /// The list of workers that are currently running. This will always + /// include a future that wakes us up periodically + futures: Vec>>>, +} + +impl Workers { + fn new() -> Self { + Self { + futures: vec![Self::waker()], + } + } + + fn add(&mut self, worker: Pin>>) { + self.futures.push(worker); + } + + fn has_work(&self) -> bool { + self.futures.len() > 1 + } + + async fn select(&mut self) -> WorkerResult { + use WorkerResult::*; + + let futures = std::mem::take(&mut self.futures); + let (result, _idx, remaining) = select_all(futures).await; + self.futures = remaining; + match result { + Ok(_) | Err(_) => { /* nothing to do */ } + Wake => { + self.futures.push(Self::waker()); + } + } + result + } + + fn waker() -> Pin>> { + let sleep = tokio::time::sleep(ENV_VARS.store.batch_target_duration); + Box::pin(sleep.map(|()| WorkerResult::Wake)) + } + + /// Return the number of workers that are not the waker + fn len(&self) -> usize { + self.futures.len() - 1 + } +} + /// A helper for copying subgraphs pub struct Connection { /// The connection pool for the shard that will contain the destination @@ -926,7 +990,7 @@ impl Connection { &mut self, state: &mut CopyState, progress: &Arc, - ) -> Option>>>> { + ) -> Option>>> { let Some(conn) = self.conn.take() else { return None; }; @@ -947,7 +1011,7 @@ impl Connection { &mut self, state: &mut CopyState, progress: &Arc, - ) -> Option>>>> { + ) -> Option>>> { // It's important that we get the connection before the table since // we remove the table from the state and could drop it otherwise let Some(conn) = self @@ -989,19 +1053,15 @@ impl Connection { /// Wait for all workers to finish. This is called when we a worker has /// failed with an error that forces us to abort copying - async fn cancel_workers( - &mut self, - progress: Arc, - mut workers: Vec>>>>, - ) { + async fn cancel_workers(&mut self, progress: Arc, mut workers: Workers) { progress.cancel(); error!( self.logger, "copying encountered an error; waiting for all workers to finish" ); - while !workers.is_empty() { - let (result, _, remaining) = select_all(workers).await; - workers = remaining; + while workers.has_work() { + use WorkerResult::*; + let result = workers.select().await; match result { Ok(worker) => { self.conn = Some(worker.conn); @@ -1010,6 +1070,7 @@ impl Connection { /* Ignore; we had an error previously */ error!(self.logger, "copy worker panicked: {}", e); } + Wake => { /* Ignore; this is just a waker */ } } } } @@ -1031,14 +1092,14 @@ impl Connection { // // The loop has to be very careful about terminating early so that // we do not ever leave the loop with `self.conn == None` - let mut workers = Vec::new(); - while !state.unfinished.is_empty() || !workers.is_empty() { + let mut workers = Workers::new(); + while !state.unfinished.is_empty() || workers.has_work() { // We usually add at least one job here, except if we are out of // tables to copy. In that case, we go through the `while` loop // every time one of the tables we are currently copying // finishes if let Some(worker) = self.default_worker(&mut state, &progress) { - workers.push(worker); + workers.add(worker); } loop { if workers.len() >= self.workers { @@ -1047,24 +1108,24 @@ impl Connection { let Some(worker) = self.extra_worker(&mut state, &progress) else { break; }; - workers.push(worker); + workers.add(worker); } self.assert_progress(workers.len(), &state)?; - let (result, _idx, remaining) = select_all(workers).await; - workers = remaining; + let result = workers.select().await; // Analyze `result` and take another trip through the loop if // everything is ok; wait for pending workers and return if // there was an error or if copying was cancelled. + use WorkerResult as W; match result { - Err(e) => { + W::Err(e) => { // This is a panic in the background task. We need to // cancel all other tasks and return the error self.cancel_workers(progress, workers).await; return Err(e); } - Ok(worker) => { + W::Ok(worker) => { // Put the connection back into self.conn so that we can use it // in the next iteration. self.conn = Some(worker.conn); @@ -1090,6 +1151,10 @@ impl Connection { } } } + W::Wake => { + // nothing to do, just try to create more workers by + // going through the loop again + } }; } debug_assert!(self.conn.is_some()); From 16521ee8e69ab78999edecd108f2d2c06e327b9c Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Wed, 9 Apr 2025 17:45:15 -0700 Subject: [PATCH 142/150] store: Do not use a fdw connection to check active_copies Copying checks the active_copies table through the primary_public.active_copies foreign table to determine whether the copy has been cancelled and it should stop. With a large number of copies running, that causes a large number of postgres_fdw connections into the primary, which can overwhelm the primary. Instead, we now pass the connection pool for the primary into the copy code so that it can do this check without involving postgres_fdw. --- store/postgres/src/copy.rs | 73 ++++++++------------------ store/postgres/src/deployment_store.rs | 15 ++++-- store/postgres/src/primary.rs | 51 +++++++++++++++++- store/postgres/src/subgraph_store.rs | 9 +++- 4 files changed, 90 insertions(+), 58 deletions(-) diff --git a/store/postgres/src/copy.rs b/store/postgres/src/copy.rs index effe2950ee2..a20c9b2a29d 100644 --- a/store/postgres/src/copy.rs +++ b/store/postgres/src/copy.rs @@ -37,7 +37,7 @@ use graph::{ info, lazy_static, o, warn, BlockNumber, BlockPtr, CheapClone, Logger, StoreError, ENV_VARS, }, schema::EntityType, - slog::{debug, error}, + slog::error, tokio, }; use itertools::Itertools; @@ -45,7 +45,7 @@ use itertools::Itertools; use crate::{ advisory_lock, catalog, deployment, dynds::DataSourcesTable, - primary::{DeploymentId, Site}, + primary::{DeploymentId, Primary, Site}, relational::index::IndexList, vid_batcher::{VidBatcher, VidRange}, }; @@ -104,46 +104,6 @@ table! { } } -// This is the same as primary::active_copies, but mapped into each shard -table! { - primary_public.active_copies(dst) { - src -> Integer, - dst -> Integer, - cancelled_at -> Nullable, - } -} - -/// Return `true` if the site is the source of a copy operation. The copy -/// operation might be just queued or in progress already. This method will -/// block until a fdw connection becomes available. -pub fn is_source(logger: &Logger, pool: &ConnectionPool, site: &Site) -> Result { - use active_copies as ac; - - // We use a fdw connection to check if the site is being copied. If we - // used an ordinary connection and there are many calls to this method, - // postgres_fdw might open an unmanageable number of connections into - // the primary, which makes the primary run out of connections - let mut last_log = Instant::now(); - let mut conn = pool.get_fdw(&logger, || { - if last_log.elapsed() > LOG_INTERVAL { - last_log = Instant::now(); - debug!( - logger, - "Waiting for fdw connection to check if site {} is being copied", site.namespace - ); - } - false - })?; - - select(diesel::dsl::exists( - ac::table - .filter(ac::src.eq(site.id)) - .filter(ac::cancelled_at.is_null()), - )) - .get_result::(&mut conn) - .map_err(StoreError::from) -} - #[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum Status { Finished, @@ -161,6 +121,7 @@ struct CopyState { impl CopyState { fn new( conn: &mut PgConnection, + primary: Primary, src: Arc, dst: Arc, target_block: BlockPtr, @@ -199,9 +160,9 @@ impl CopyState { src.site.id )); } - Self::load(conn, src, dst, target_block) + Self::load(conn, primary, src, dst, target_block) } - None => Self::create(conn, src, dst, target_block), + None => Self::create(conn, primary.cheap_clone(), src, dst, target_block), }?; Ok(state) @@ -209,11 +170,12 @@ impl CopyState { fn load( conn: &mut PgConnection, + primary: Primary, src: Arc, dst: Arc, target_block: BlockPtr, ) -> Result { - let tables = TableState::load(conn, src.as_ref(), dst.as_ref())?; + let tables = TableState::load(conn, primary, src.as_ref(), dst.as_ref())?; let (finished, mut unfinished): (Vec<_>, Vec<_>) = tables.into_iter().partition(|table| table.finished()); unfinished.sort_by_key(|table| table.dst.object.to_string()); @@ -228,6 +190,7 @@ impl CopyState { fn create( conn: &mut PgConnection, + primary: Primary, src: Arc, dst: Arc, target_block: BlockPtr, @@ -253,6 +216,7 @@ impl CopyState { .map(|src_table| { TableState::init( conn, + primary.cheap_clone(), dst.site.clone(), &src, src_table.clone(), @@ -354,6 +318,7 @@ pub(crate) fn source( /// transformation. See `CopyEntityBatchQuery` for the details of what /// exactly that means struct TableState { + primary: Primary, src: Arc
, dst: Arc
, dst_site: Arc, @@ -364,6 +329,7 @@ struct TableState { impl TableState { fn init( conn: &mut PgConnection, + primary: Primary, dst_site: Arc, src_layout: &Layout, src: Arc
, @@ -373,6 +339,7 @@ impl TableState { let vid_range = VidRange::for_copy(conn, &src, target_block)?; let batcher = VidBatcher::load(conn, &src_layout.site.namespace, src.as_ref(), vid_range)?; Ok(Self { + primary, src, dst, dst_site, @@ -387,6 +354,7 @@ impl TableState { fn load( conn: &mut PgConnection, + primary: Primary, src_layout: &Layout, dst_layout: &Layout, ) -> Result, StoreError> { @@ -450,6 +418,7 @@ impl TableState { .with_batch_size(size as usize); Ok(TableState { + primary: primary.cheap_clone(), src, dst, dst_site: dst_layout.site.clone(), @@ -516,13 +485,8 @@ impl TableState { } fn is_cancelled(&self, conn: &mut PgConnection) -> Result { - use active_copies as ac; - let dst = self.dst_site.as_ref(); - let canceled = ac::table - .filter(ac::dst.eq(dst.id)) - .select(ac::cancelled_at.is_not_null()) - .get_result::(conn)?; + let canceled = self.primary.is_copy_cancelled(dst)?; if canceled { use copy_state as cs; @@ -893,6 +857,7 @@ pub struct Connection { /// `self.transaction` conn: Option, pool: ConnectionPool, + primary: Primary, workers: usize, src: Arc, dst: Arc, @@ -910,6 +875,7 @@ impl Connection { /// is available. pub fn new( logger: &Logger, + primary: Primary, pool: ConnectionPool, src: Arc, dst: Arc, @@ -942,6 +908,7 @@ impl Connection { logger, conn, pool, + primary, workers: ENV_VARS.store.batch_workers, src, dst, @@ -1079,7 +1046,9 @@ impl Connection { let src = self.src.clone(); let dst = self.dst.clone(); let target_block = self.target_block.clone(); - let mut state = self.transaction(|conn| CopyState::new(conn, src, dst, target_block))?; + let primary = self.primary.cheap_clone(); + let mut state = + self.transaction(|conn| CopyState::new(conn, primary, src, dst, target_block))?; let progress = Arc::new(CopyProgress::new(self.logger.cheap_clone(), &state)); progress.start(); diff --git a/store/postgres/src/deployment_store.rs b/store/postgres/src/deployment_store.rs index c78b06be46d..e497430c2bf 100644 --- a/store/postgres/src/deployment_store.rs +++ b/store/postgres/src/deployment_store.rs @@ -51,11 +51,11 @@ use crate::block_range::{BLOCK_COLUMN, BLOCK_RANGE_COLUMN}; use crate::deployment::{self, OnSync}; use crate::detail::ErrorDetail; use crate::dynds::DataSourcesTable; -use crate::primary::DeploymentId; +use crate::primary::{DeploymentId, Primary}; use crate::relational::index::{CreateIndex, IndexList, Method}; use crate::relational::{Layout, LayoutCache, SqlName, Table}; use crate::relational_queries::FromEntityData; -use crate::{advisory_lock, catalog, copy, retry}; +use crate::{advisory_lock, catalog, retry}; use crate::{connection_pool::ConnectionPool, detail}; use crate::{dynds, primary::Site}; @@ -93,6 +93,8 @@ type PruneHandle = JoinHandle>; pub struct StoreInner { logger: Logger, + primary: Primary, + pool: ConnectionPool, read_only_pools: Vec, @@ -130,6 +132,7 @@ impl Deref for DeploymentStore { impl DeploymentStore { pub fn new( logger: &Logger, + primary: Primary, pool: ConnectionPool, read_only_pools: Vec, mut pool_weights: Vec, @@ -160,6 +163,7 @@ impl DeploymentStore { // Create the store let store = StoreInner { logger: logger.clone(), + primary, pool, read_only_pools, replica_order, @@ -1235,7 +1239,7 @@ impl DeploymentStore { req: PruneRequest, ) -> Result<(), StoreError> { { - if copy::is_source(&logger, &store.pool, &site)? { + if store.is_source(&site)? { debug!( logger, "Skipping pruning since this deployment is being copied" @@ -1520,6 +1524,7 @@ impl DeploymentStore { // with the corresponding tables in `self` let copy_conn = crate::copy::Connection::new( logger, + self.primary.cheap_clone(), self.pool.clone(), src.clone(), dst.clone(), @@ -1848,6 +1853,10 @@ impl DeploymentStore { }) .await } + + fn is_source(&self, site: &Site) -> Result { + self.primary.is_source(site) + } } /// Tries to fetch a [`Table`] either by its Entity name or its SQL name. diff --git a/store/postgres/src/primary.rs b/store/postgres/src/primary.rs index f329ae4bba2..39df898ba32 100644 --- a/store/postgres/src/primary.rs +++ b/store/postgres/src/primary.rs @@ -36,6 +36,7 @@ use graph::{ store::scalar::ToPrimitive, subgraph::{status, DeploymentFeatures}, }, + derive::CheapClone, prelude::{ anyhow, chrono::{DateTime, Utc}, @@ -53,9 +54,9 @@ use maybe_owned::MaybeOwnedMut; use std::{ borrow::Borrow, collections::HashMap, - convert::TryFrom, - convert::TryInto, + convert::{TryFrom, TryInto}, fmt, + sync::Arc, time::{SystemTime, UNIX_EPOCH}, }; @@ -1826,6 +1827,52 @@ impl<'a> Connection<'a> { } } +/// A limited interface to query the primary database. +#[derive(Clone, CheapClone)] +pub struct Primary { + pool: Arc, +} + +impl Primary { + pub fn new(pool: Arc) -> Self { + // This really indicates a programming error + if pool.shard != *PRIMARY_SHARD { + panic!("Primary pool must be the primary shard"); + } + + Primary { pool } + } + + /// Return `true` if the site is the source of a copy operation. The copy + /// operation might be just queued or in progress already. This method will + /// block until a fdw connection becomes available. + pub fn is_source(&self, site: &Site) -> Result { + use active_copies as ac; + + let mut conn = self.pool.get()?; + + select(diesel::dsl::exists( + ac::table + .filter(ac::src.eq(site.id)) + .filter(ac::cancelled_at.is_null()), + )) + .get_result::(&mut conn) + .map_err(StoreError::from) + } + + pub fn is_copy_cancelled(&self, dst: &Site) -> Result { + use active_copies as ac; + + let mut conn = self.pool.get()?; + + ac::table + .filter(ac::dst.eq(dst.id)) + .select(ac::cancelled_at.is_not_null()) + .get_result::(&mut conn) + .map_err(StoreError::from) + } +} + /// Return `true` if we deem this installation to be empty, defined as /// having no deployments and no subgraph names in the database pub fn is_empty(conn: &mut PgConnection) -> Result { diff --git a/store/postgres/src/subgraph_store.rs b/store/postgres/src/subgraph_store.rs index 0beeadf345d..339c66cee3f 100644 --- a/store/postgres/src/subgraph_store.rs +++ b/store/postgres/src/subgraph_store.rs @@ -39,7 +39,7 @@ use graph::{ use crate::{ connection_pool::ConnectionPool, deployment::{OnSync, SubgraphHealth}, - primary::{self, DeploymentId, Mirror as PrimaryMirror, Site}, + primary::{self, DeploymentId, Mirror as PrimaryMirror, Primary, Site}, relational::{ index::{IndexList, Method}, Layout, @@ -360,6 +360,12 @@ impl SubgraphStoreInner { sender: Arc, registry: Arc, ) -> Self { + let primary = stores + .iter() + .find(|(name, _, _, _)| name == &*PRIMARY_SHARD) + .map(|(_, pool, _, _)| Primary::new(Arc::new(pool.clone()))) + .expect("primary shard must be present"); + let mirror = { let pools = HashMap::from_iter( stores @@ -376,6 +382,7 @@ impl SubgraphStoreInner { name, Arc::new(DeploymentStore::new( &logger, + primary.cheap_clone(), main_pool, read_only_pools, weights, From ee88833b53f548200eda2e16840511e7cbb05c29 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Wed, 9 Apr 2025 18:33:27 -0700 Subject: [PATCH 143/150] store: Log more when copy workers have an error --- store/postgres/src/copy.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/store/postgres/src/copy.rs b/store/postgres/src/copy.rs index a20c9b2a29d..fd736276cbd 100644 --- a/store/postgres/src/copy.rs +++ b/store/postgres/src/copy.rs @@ -1091,6 +1091,7 @@ impl Connection { W::Err(e) => { // This is a panic in the background task. We need to // cancel all other tasks and return the error + error!(self.logger, "copy worker panicked: {}", e); self.cancel_workers(progress, workers).await; return Err(e); } @@ -1115,6 +1116,7 @@ impl Connection { return Ok(Status::Cancelled); } (Err(e), _) => { + error!(self.logger, "copy worker had an error: {}", e); self.cancel_workers(progress, workers).await; return Err(e); } From 63b2a5c2d3a67e21ffbaac2d74a843cd049d24aa Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Wed, 9 Apr 2025 23:06:46 -0700 Subject: [PATCH 144/150] store: Make sure we use the right connection to unlock the copy lock Otherwise, the lock will linger and can block an attempt to restart a copy that failed for transient reasons --- store/postgres/src/copy.rs | 107 ++++++++++++++++++++++++++++++------- 1 file changed, 88 insertions(+), 19 deletions(-) diff --git a/store/postgres/src/copy.rs b/store/postgres/src/copy.rs index fd736276cbd..22ddee394f6 100644 --- a/store/postgres/src/copy.rs +++ b/store/postgres/src/copy.rs @@ -64,8 +64,6 @@ const ACCEPTABLE_REPLICATION_LAG: Duration = Duration::from_secs(30); /// the lag again const REPLICATION_SLEEP: Duration = Duration::from_secs(10); -type PooledPgConnection = PooledConnection>; - lazy_static! { static ref STATEMENT_TIMEOUT: Option = ENV_VARS .store @@ -667,17 +665,77 @@ impl From> for WorkerResult { } } +/// We pass connections back and forth between the control loop and various +/// workers. We need to make sure that we end up with the connection that +/// was used to acquire the copy lock in the right place so we can release +/// the copy lock which is only possible with the connection that acquired +/// it. +/// +/// This struct helps us with that. It wraps a connection and tracks whether +/// the connection was used to acquire the copy lock +struct LockTrackingConnection { + inner: PooledConnection>, + has_lock: bool, +} + +impl LockTrackingConnection { + fn new(inner: PooledConnection>) -> Self { + Self { + inner, + has_lock: false, + } + } + + fn transaction(&mut self, f: F) -> Result + where + F: FnOnce(&mut PgConnection) -> Result, + { + let conn = &mut self.inner; + conn.transaction(|conn| f(conn)) + } + + /// Put `self` into `other` if `self` has the lock. + fn extract(self, other: &mut Option) { + if self.has_lock { + *other = Some(self); + } + } + + fn lock(&mut self, logger: &Logger, dst: &Site) -> Result<(), StoreError> { + if self.has_lock { + warn!(logger, "already acquired copy lock for {}", dst); + return Ok(()); + } + advisory_lock::lock_copying(&mut self.inner, dst)?; + self.has_lock = true; + Ok(()) + } + + fn unlock(&mut self, logger: &Logger, dst: &Site) -> Result<(), StoreError> { + if !self.has_lock { + error!( + logger, + "tried to release copy lock for {} even though we are not the owner", dst + ); + return Ok(()); + } + advisory_lock::unlock_copying(&mut self.inner, dst)?; + self.has_lock = false; + Ok(()) + } +} + /// A helper to run copying of one table. We need to thread `conn` and /// `table` from the control loop to the background worker and back again to /// the control loop. This worker facilitates that struct CopyTableWorker { - conn: PooledPgConnection, + conn: LockTrackingConnection, table: TableState, result: Result, } impl CopyTableWorker { - fn new(conn: PooledPgConnection, table: TableState) -> Self { + fn new(conn: LockTrackingConnection, table: TableState) -> Self { Self { conn, table, @@ -699,7 +757,7 @@ impl CopyTableWorker { fn run_inner(&mut self, logger: Logger, progress: &CopyProgress) -> Result { use Status::*; - let conn = &mut self.conn; + let conn = &mut self.conn.inner; progress.start_table(&self.table); while !self.table.finished() { // It is important that this check happens outside the write @@ -855,7 +913,7 @@ pub struct Connection { /// individual table. Except for that case, this will always be /// `Some(..)`. Most code shouldn't access `self.conn` directly, but use /// `self.transaction` - conn: Option, + conn: Option, pool: ConnectionPool, primary: Primary, workers: usize, @@ -901,9 +959,9 @@ impl Connection { } false })?; - let conn = Some(conn); let src_manifest_idx_and_name = Arc::new(src_manifest_idx_and_name); let dst_manifest_idx_and_name = Arc::new(dst_manifest_idx_and_name); + let conn = Some(LockTrackingConnection::new(conn)); Ok(Self { logger, conn, @@ -990,6 +1048,7 @@ impl Connection { let Some(table) = state.unfinished.pop() else { return None; }; + let conn = LockTrackingConnection::new(conn); let worker = CopyTableWorker::new(conn, table); Some(Box::pin( @@ -1031,7 +1090,7 @@ impl Connection { let result = workers.select().await; match result { Ok(worker) => { - self.conn = Some(worker.conn); + worker.conn.extract(&mut self.conn); } Err(e) => { /* Ignore; we had an error previously */ @@ -1098,7 +1157,7 @@ impl Connection { W::Ok(worker) => { // Put the connection back into self.conn so that we can use it // in the next iteration. - self.conn = Some(worker.conn); + worker.conn.extract(&mut self.conn); match (worker.result, progress.is_cancelled()) { (Ok(Status::Finished), false) => { @@ -1207,20 +1266,30 @@ impl Connection { ); let dst_site = self.dst.site.cheap_clone(); - self.transaction(|conn| advisory_lock::lock_copying(conn, &dst_site))?; + let Some(conn) = self.conn.as_mut() else { + return Err(constraint_violation!( + "copy connection went missing (copy_data)" + )); + }; + conn.lock(&self.logger, &dst_site)?; let res = self.copy_data_internal(index_list).await; - if self.conn.is_none() { - // A background worker panicked and left us without our - // dedicated connection, but we still need to release the copy - // lock; get a normal connection, not from the fdw pool for that - // as that will be much less contended. We won't be holding on - // to the connection for long as `res` will be an error and we - // will abort starting this subgraph - self.conn = Some(self.pool.get()?); + match self.conn.as_mut() { + None => { + // A background worker panicked and left us without our + // dedicated connection; we would need to get that + // connection to unlock the advisory lock. We can't do that, + // so we just log an error + warn!( + self.logger, + "can't unlock copy lock since the default worker panicked; lock will linger until session ends" + ); + } + Some(conn) => { + conn.unlock(&self.logger, &dst_site)?; + } } - self.transaction(|conn| advisory_lock::unlock_copying(conn, &dst_site))?; if matches!(res, Ok(Status::Cancelled)) { warn!(&self.logger, "Copying was cancelled and is incomplete"); From 17360f56c7657e49d2b5da2fafa5d8d633d556f7 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Thu, 10 Apr 2025 09:42:21 -0700 Subject: [PATCH 145/150] graph: Allow remove followed by remove in write batches If subgraphs delete entities without checking if they exist, we get two removes in a row. In the database, the second remove would just lead to a query that changes nothing. We'll do the same when putting a write batch together. Fixes https://github.com/graphprotocol/graph-node/issues/5449 --- graph/src/components/store/write.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/graph/src/components/store/write.rs b/graph/src/components/store/write.rs index aa56fdcc910..6f899633bd8 100644 --- a/graph/src/components/store/write.rs +++ b/graph/src/components/store/write.rs @@ -439,7 +439,7 @@ impl RowGroup { // clamping an old version match (&*prev_row, &row) { (Insert { end: None, .. } | Overwrite { end: None, .. }, Insert { .. }) - | (Remove { .. }, Overwrite { .. } | Remove { .. }) + | (Remove { .. }, Overwrite { .. }) | ( Insert { end: Some(_), .. } | Overwrite { end: Some(_), .. }, Overwrite { .. } | Remove { .. }, @@ -450,6 +450,11 @@ impl RowGroup { row )) } + (Remove { .. }, Remove { .. }) => { + // Ignore the new row, since prev_row is already a + // delete. This can happen when subgraphs delete + // entities without checking if they even exist + } ( Insert { end: Some(_), .. } | Overwrite { end: Some(_), .. } | Remove { .. }, Insert { .. }, From c0516729d432b7c8aa39ec88cfb0804ca6a1a0f5 Mon Sep 17 00:00:00 2001 From: Zoran Cvetkov <36600146+zorancv@users.noreply.github.com> Date: Mon, 14 Apr 2025 12:16:53 +0300 Subject: [PATCH 146/150] test: fix reorg threshold --- chain/ethereum/src/chain.rs | 2 +- graph/src/data/subgraph/mod.rs | 4 +-- graph/src/env/mod.rs | 46 +++++++++++++++++--------- node/src/chain.rs | 2 +- node/src/manager/commands/prune.rs | 6 ++-- node/src/manager/commands/rewind.rs | 4 +-- store/postgres/src/block_store.rs | 2 +- store/postgres/src/deployment.rs | 6 +++- store/postgres/src/deployment_store.rs | 2 +- tests/src/config.rs | 3 +- tests/src/fixture/ethereum.rs | 2 +- tests/tests/runner_tests.rs | 6 +++- 12 files changed, 55 insertions(+), 30 deletions(-) diff --git a/chain/ethereum/src/chain.rs b/chain/ethereum/src/chain.rs index 117e3033b18..911d4d3ebfe 100644 --- a/chain/ethereum/src/chain.rs +++ b/chain/ethereum/src/chain.rs @@ -614,7 +614,7 @@ impl Blockchain for Chain { // present in the DB. Box::new(PollingBlockIngestor::new( logger, - graph::env::ENV_VARS.reorg_threshold, + graph::env::ENV_VARS.reorg_threshold(), self.chain_client(), self.chain_store().cheap_clone(), self.polling_ingestor_interval, diff --git a/graph/src/data/subgraph/mod.rs b/graph/src/data/subgraph/mod.rs index 3e7bc7061ab..77c8ba67d36 100644 --- a/graph/src/data/subgraph/mod.rs +++ b/graph/src/data/subgraph/mod.rs @@ -504,9 +504,9 @@ impl Graft { // The graft point must be at least `reorg_threshold` blocks // behind the subgraph head so that a reorg can not affect the // data that we copy for grafting - (Some(ptr), true) if self.block + ENV_VARS.reorg_threshold > ptr.number => Err(GraftBaseInvalid(format!( + (Some(ptr), true) if self.block + ENV_VARS.reorg_threshold() > ptr.number => Err(GraftBaseInvalid(format!( "failed to graft onto `{}` at block {} since it's only at block {} which is within the reorg threshold of {} blocks", - self.base, self.block, ptr.number, ENV_VARS.reorg_threshold + self.base, self.block, ptr.number, ENV_VARS.reorg_threshold() ))), // If the base deployment is failed *and* the `graft.block` is not // less than the `base.block`, the graft shouldn't be permitted. diff --git a/graph/src/env/mod.rs b/graph/src/env/mod.rs index 48fa0ba4688..eff0ebea16e 100644 --- a/graph/src/env/mod.rs +++ b/graph/src/env/mod.rs @@ -15,9 +15,16 @@ use crate::{ runtime::gas::CONST_MAX_GAS_PER_HANDLER, }; +#[cfg(debug_assertions)] +use std::sync::Mutex; + lazy_static! { pub static ref ENV_VARS: EnvVars = EnvVars::from_env().unwrap(); } +#[cfg(debug_assertions)] +lazy_static! { + pub static ref TEST_WITH_NO_REORG: Mutex = Mutex::new(false); +} /// Panics if: /// - The value is not UTF8. @@ -181,7 +188,7 @@ pub struct EnvVars { pub static_filters_threshold: usize, /// Set by the environment variable `ETHEREUM_REORG_THRESHOLD`. The default /// value is 250 blocks. - pub reorg_threshold: BlockNumber, + reorg_threshold: BlockNumber, /// The time to wait between polls when using polling block ingestor. /// The value is set by `ETHERUM_POLLING_INTERVAL` in millis and the /// default is 1000. @@ -259,16 +266,6 @@ impl EnvVars { let mapping_handlers = InnerMappingHandlers::init_from_env()?.into(); let store = InnerStore::init_from_env()?.try_into()?; - // The default reorganization (reorg) threshold is set to 250. - // For testing purposes, we need to set this threshold to 0 because: - // 1. Many tests involve reverting blocks. - // 2. Blocks cannot be reverted below the reorg threshold. - // Therefore, during tests, we want to set the reorg threshold to 0. - let reorg_threshold = - inner - .reorg_threshold - .unwrap_or_else(|| if cfg!(debug_assertions) { 0 } else { 250 }); - Ok(Self { graphql, mappings: mapping_handlers, @@ -322,13 +319,15 @@ impl EnvVars { external_http_base_url: inner.external_http_base_url, external_ws_base_url: inner.external_ws_base_url, static_filters_threshold: inner.static_filters_threshold, - reorg_threshold, + reorg_threshold: inner.reorg_threshold, ingestor_polling_interval: Duration::from_millis(inner.ingestor_polling_interval), subgraph_settings: inner.subgraph_settings, prefer_substreams_block_streams: inner.prefer_substreams_block_streams, enable_dips_metrics: inner.enable_dips_metrics.0, history_blocks_override: inner.history_blocks_override, - min_history_blocks: inner.min_history_blocks.unwrap_or(2 * reorg_threshold), + min_history_blocks: inner + .min_history_blocks + .unwrap_or(2 * inner.reorg_threshold), dips_metrics_object_store_url: inner.dips_metrics_object_store_url, section_map: inner.section_map, firehose_grpc_max_decode_size_mb: inner.firehose_grpc_max_decode_size_mb, @@ -375,6 +374,23 @@ impl EnvVars { .filter(|x| !x.is_empty()) .collect() } + #[cfg(debug_assertions)] + pub fn reorg_threshold(&self) -> i32 { + // The default reorganization (reorg) threshold is set to 250. + // For testing purposes, we need to set this threshold to 0 because: + // 1. Many tests involve reverting blocks. + // 2. Blocks cannot be reverted below the reorg threshold. + // Therefore, during tests, we want to set the reorg threshold to 0. + if *TEST_WITH_NO_REORG.lock().unwrap() { + 0 + } else { + self.reorg_threshold + } + } + #[cfg(not(debug_assertions))] + pub fn reorg_threshold(&self) -> i32 { + self.reorg_threshold + } } impl Default for EnvVars { @@ -473,8 +489,8 @@ struct Inner { #[envconfig(from = "GRAPH_STATIC_FILTERS_THRESHOLD", default = "10000")] static_filters_threshold: usize, // JSON-RPC specific. - #[envconfig(from = "ETHEREUM_REORG_THRESHOLD")] - reorg_threshold: Option, + #[envconfig(from = "ETHEREUM_REORG_THRESHOLD", default = "250")] + reorg_threshold: BlockNumber, #[envconfig(from = "ETHEREUM_POLLING_INTERVAL", default = "1000")] ingestor_polling_interval: u64, #[envconfig(from = "GRAPH_EXPERIMENTAL_SUBGRAPH_SETTINGS")] diff --git a/node/src/chain.rs b/node/src/chain.rs index 239db116e55..4ff45b8211a 100644 --- a/node/src/chain.rs +++ b/node/src/chain.rs @@ -460,7 +460,7 @@ pub async fn networks_as_chains( Arc::new(adapter_selector), Arc::new(EthereumRuntimeAdapterBuilder {}), eth_adapters, - ENV_VARS.reorg_threshold, + ENV_VARS.reorg_threshold(), polling_interval, true, ); diff --git a/node/src/manager/commands/prune.rs b/node/src/manager/commands/prune.rs index c169577ee65..2c3c2ae2386 100644 --- a/node/src/manager/commands/prune.rs +++ b/node/src/manager/commands/prune.rs @@ -188,13 +188,13 @@ pub async fn run( println!("prune {deployment}"); println!(" latest: {latest}"); - println!(" final: {}", latest - ENV_VARS.reorg_threshold); + println!(" final: {}", latest - ENV_VARS.reorg_threshold()); println!(" earliest: {}\n", latest - history); let mut req = PruneRequest::new( &deployment, history, - ENV_VARS.reorg_threshold, + ENV_VARS.reorg_threshold(), status.earliest_block_number, latest, )?; @@ -217,7 +217,7 @@ pub async fn run( store.subgraph_store().set_history_blocks( &deployment, history, - ENV_VARS.reorg_threshold, + ENV_VARS.reorg_threshold(), )?; } diff --git a/node/src/manager/commands/rewind.rs b/node/src/manager/commands/rewind.rs index 339f2ec979a..629c4b6e70f 100644 --- a/node/src/manager/commands/rewind.rs +++ b/node/src/manager/commands/rewind.rs @@ -133,13 +133,13 @@ pub async fn run( let deployment_details = deployment_store.deployment_details_for_id(locator)?; let block_number_to = block_ptr_to.as_ref().map(|b| b.number).unwrap_or(0); - if block_number_to < deployment_details.earliest_block_number + ENV_VARS.reorg_threshold { + if block_number_to < deployment_details.earliest_block_number + ENV_VARS.reorg_threshold() { bail!( "The block number {} is not safe to rewind to for deployment {}. The earliest block number of this deployment is {}. You can only safely rewind to block number {}", block_ptr_to.as_ref().map(|b| b.number).unwrap_or(0), locator, deployment_details.earliest_block_number, - deployment_details.earliest_block_number + ENV_VARS.reorg_threshold + deployment_details.earliest_block_number + ENV_VARS.reorg_threshold() ); } } diff --git a/store/postgres/src/block_store.rs b/store/postgres/src/block_store.rs index f69267fff17..762a2642524 100644 --- a/store/postgres/src/block_store.rs +++ b/store/postgres/src/block_store.rs @@ -503,7 +503,7 @@ impl BlockStore { }; if let Some(head_block) = store.remove_cursor(&&store.chain)? { - let lower_bound = head_block.saturating_sub(ENV_VARS.reorg_threshold * 2); + let lower_bound = head_block.saturating_sub(ENV_VARS.reorg_threshold() * 2); info!(&self.logger, "Removed cursor for non-firehose chain, now cleaning shallow blocks"; "network" => &store.chain, "lower_bound" => lower_bound); store.cleanup_shallow_blocks(lower_bound)?; } diff --git a/store/postgres/src/deployment.rs b/store/postgres/src/deployment.rs index 92181ac5a6c..5d83a563181 100644 --- a/store/postgres/src/deployment.rs +++ b/store/postgres/src/deployment.rs @@ -546,10 +546,14 @@ pub fn revert_block_ptr( // Work around a Diesel issue with serializing BigDecimals to numeric let number = format!("{}::numeric", ptr.number); + // Intention is to revert to a block lower than the reorg threshold, on the other + // hand the earliest we can possibly go is genesys block, so go to genesys even + // if it's within the reorg threshold. + let earliest_block = i32::max(ptr.number - ENV_VARS.reorg_threshold(), 0); let affected_rows = update( d::table .filter(d::deployment.eq(id.as_str())) - .filter(d::earliest_block_number.le(ptr.number - ENV_VARS.reorg_threshold)), + .filter(d::earliest_block_number.le(earliest_block)), ) .set(( d::latest_ethereum_block_number.eq(sql(&number)), diff --git a/store/postgres/src/deployment_store.rs b/store/postgres/src/deployment_store.rs index e497430c2bf..e07b4659436 100644 --- a/store/postgres/src/deployment_store.rs +++ b/store/postgres/src/deployment_store.rs @@ -1261,7 +1261,7 @@ impl DeploymentStore { let req = PruneRequest::new( &site.as_ref().into(), history_blocks, - ENV_VARS.reorg_threshold, + ENV_VARS.reorg_threshold(), earliest_block, latest_block, )?; diff --git a/tests/src/config.rs b/tests/src/config.rs index 54b07b0a5a8..6cdd97a216f 100644 --- a/tests/src/config.rs +++ b/tests/src/config.rs @@ -175,7 +175,8 @@ impl Config { .stdout(stdout) .stderr(stderr) .args(args) - .env("GRAPH_STORE_WRITE_BATCH_DURATION", "5"); + .env("GRAPH_STORE_WRITE_BATCH_DURATION", "5") + .env("ETHEREUM_REORG_THRESHOLD", "0"); status!( "graph-node", diff --git a/tests/src/fixture/ethereum.rs b/tests/src/fixture/ethereum.rs index fc651a512db..d93ac25c235 100644 --- a/tests/src/fixture/ethereum.rs +++ b/tests/src/fixture/ethereum.rs @@ -64,7 +64,7 @@ pub async fn chain( triggers_adapter, Arc::new(NoopRuntimeAdapterBuilder {}), eth_adapters, - ENV_VARS.reorg_threshold, + ENV_VARS.reorg_threshold(), ENV_VARS.ingestor_polling_interval, // We assume the tested chain is always ingestible for now true, diff --git a/tests/tests/runner_tests.rs b/tests/tests/runner_tests.rs index ac645884b5d..261c886dfea 100644 --- a/tests/tests/runner_tests.rs +++ b/tests/tests/runner_tests.rs @@ -12,7 +12,7 @@ use graph::data::store::scalar::Bytes; use graph::data::subgraph::schema::{SubgraphError, SubgraphHealth}; use graph::data::value::Word; use graph::data_source::CausalityRegion; -use graph::env::EnvVars; +use graph::env::{EnvVars, TEST_WITH_NO_REORG}; use graph::ipfs; use graph::ipfs::test_utils::add_files_to_local_ipfs_node_for_testing; use graph::object; @@ -109,6 +109,8 @@ fn assert_eq_ignore_backtrace(err: &SubgraphError, expected: &SubgraphError) { #[tokio::test] async fn data_source_revert() -> anyhow::Result<()> { + *TEST_WITH_NO_REORG.lock().unwrap() = true; + let RunnerTestRecipe { stores, test_info } = RunnerTestRecipe::new("data_source_revert", "data-source-revert").await; @@ -179,6 +181,8 @@ async fn data_source_revert() -> anyhow::Result<()> { // since it uses the same deployment id. data_source_long_revert().await.unwrap(); + *TEST_WITH_NO_REORG.lock().unwrap() = false; + Ok(()) } From bafc3dd52439b458ef0e0e972558fb29f1fda42f Mon Sep 17 00:00:00 2001 From: Zoran Cvetkov <36600146+zorancv@users.noreply.github.com> Date: Tue, 15 Apr 2025 10:51:16 +0300 Subject: [PATCH 147/150] test: check grafting from pre 0.0.6 to 0.0.6 and later --- .github/workflows/ci.yml | 2 +- tests/docker-compose.yml | 2 +- .../integration-tests/base/abis/Contract.abi | 33 +++++ tests/integration-tests/base/package.json | 25 ++++ tests/integration-tests/base/schema.graphql | 5 + tests/integration-tests/base/src/mapping.ts | 9 ++ tests/integration-tests/base/subgraph.yaml | 25 ++++ .../grafted/abis/Contract.abi | 33 +++++ tests/integration-tests/grafted/package.json | 25 ++++ .../integration-tests/grafted/schema.graphql | 5 + .../integration-tests/grafted/src/mapping.ts | 9 ++ tests/integration-tests/grafted/subgraph.yaml | 30 ++++ tests/src/contract.rs | 11 +- tests/src/subgraph.rs | 2 +- tests/tests/integration_tests.rs | 140 ++++++++++++++++-- 15 files changed, 337 insertions(+), 19 deletions(-) create mode 100644 tests/integration-tests/base/abis/Contract.abi create mode 100644 tests/integration-tests/base/package.json create mode 100644 tests/integration-tests/base/schema.graphql create mode 100644 tests/integration-tests/base/src/mapping.ts create mode 100644 tests/integration-tests/base/subgraph.yaml create mode 100644 tests/integration-tests/grafted/abis/Contract.abi create mode 100644 tests/integration-tests/grafted/package.json create mode 100644 tests/integration-tests/grafted/schema.graphql create mode 100644 tests/integration-tests/grafted/src/mapping.ts create mode 100644 tests/integration-tests/grafted/subgraph.yaml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0fa6d58bbb7..24993639945 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -153,7 +153,7 @@ jobs: - name: Install Foundry uses: foundry-rs/foundry-toolchain@v1 - name: Start anvil - run: anvil --gas-limit 100000000000 --base-fee 1 --block-time 2 --port 3021 & + run: anvil --gas-limit 100000000000 --base-fee 1 --block-time 2 --timestamp 1743944919 --port 3021 & - name: Install graph CLI run: curl -sSL http://cli.thegraph.com/install.sh | sudo bash diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml index f45360fd367..7385b4b08a2 100644 --- a/tests/docker-compose.yml +++ b/tests/docker-compose.yml @@ -23,7 +23,7 @@ services: image: ghcr.io/foundry-rs/foundry:stable ports: - '3021:8545' - command: "'anvil --host 0.0.0.0 --gas-limit 100000000000 --base-fee 1 --block-time 5 --mnemonic \"test test test test test test test test test test test junk\"'" + command: "'anvil --host 0.0.0.0 --gas-limit 100000000000 --base-fee 1 --block-time 2 --timestamp 1743944919 --mnemonic \"test test test test test test test test test test test junk\"'" # graph-node ports: # json-rpc: 8020 diff --git a/tests/integration-tests/base/abis/Contract.abi b/tests/integration-tests/base/abis/Contract.abi new file mode 100644 index 00000000000..02da1a9e7f3 --- /dev/null +++ b/tests/integration-tests/base/abis/Contract.abi @@ -0,0 +1,33 @@ +[ + { + "inputs": [], + "stateMutability": "nonpayable", + "type": "constructor" + }, + { + "anonymous": false, + "inputs": [ + { + "indexed": false, + "internalType": "uint16", + "name": "x", + "type": "uint16" + } + ], + "name": "Trigger", + "type": "event" + }, + { + "inputs": [ + { + "internalType": "uint16", + "name": "x", + "type": "uint16" + } + ], + "name": "emitTrigger", + "outputs": [], + "stateMutability": "nonpayable", + "type": "function" + } +] diff --git a/tests/integration-tests/base/package.json b/tests/integration-tests/base/package.json new file mode 100644 index 00000000000..2cfb6b94def --- /dev/null +++ b/tests/integration-tests/base/package.json @@ -0,0 +1,25 @@ +{ + "name": "base-subgraph", + "version": "0.1.0", + "scripts": { + "build-contracts": "../../common/build-contracts.sh", + "codegen": "graph codegen --skip-migrations", + "test": "yarn build-contracts && truffle test --compile-none --network test", + "create:test": "graph create test/base-subgraph --node $GRAPH_NODE_ADMIN_URI", + "deploy:test": "graph deploy test/base-subgraph --version-label v0.0.1 --ipfs $IPFS_URI --node $GRAPH_NODE_ADMIN_URI" + }, + "devDependencies": { + "@graphprotocol/graph-cli": "0.69.0", + "@graphprotocol/graph-ts": "0.34.0", + "solc": "^0.8.2" + }, + "dependencies": { + "@truffle/contract": "^4.3", + "@truffle/hdwallet-provider": "^1.2", + "apollo-fetch": "^0.7.0", + "babel-polyfill": "^6.26.0", + "babel-register": "^6.26.0", + "gluegun": "^4.6.1", + "truffle": "^5.2" + } +} \ No newline at end of file diff --git a/tests/integration-tests/base/schema.graphql b/tests/integration-tests/base/schema.graphql new file mode 100644 index 00000000000..f7034353d73 --- /dev/null +++ b/tests/integration-tests/base/schema.graphql @@ -0,0 +1,5 @@ +type BaseData @entity(immutable: true) { + id: ID! + data: String! + blockNumber: BigInt! +} \ No newline at end of file diff --git a/tests/integration-tests/base/src/mapping.ts b/tests/integration-tests/base/src/mapping.ts new file mode 100644 index 00000000000..11767070a5b --- /dev/null +++ b/tests/integration-tests/base/src/mapping.ts @@ -0,0 +1,9 @@ +import { ethereum } from '@graphprotocol/graph-ts' +import { BaseData } from '../generated/schema' + +export function handleBlock(block: ethereum.Block): void { + let entity = new BaseData(block.number.toString()) + entity.data = 'from base' + entity.blockNumber = block.number + entity.save() +} \ No newline at end of file diff --git a/tests/integration-tests/base/subgraph.yaml b/tests/integration-tests/base/subgraph.yaml new file mode 100644 index 00000000000..808b446c622 --- /dev/null +++ b/tests/integration-tests/base/subgraph.yaml @@ -0,0 +1,25 @@ +specVersion: 0.0.5 +description: Base Subgraph +repository: https://github.com/graphprotocol/graph-node +schema: + file: ./schema.graphql +dataSources: + - kind: ethereum/contract + name: SimpleContract + network: test + source: + address: "0x5FbDB2315678afecb367f032d93F642f64180aa3" + abi: SimpleContract + startBlock: 0 + mapping: + kind: ethereum/events + apiVersion: 0.0.6 + language: wasm/assemblyscript + entities: + - BaseData + abis: + - name: SimpleContract + file: ./abis/Contract.abi + blockHandlers: + - handler: handleBlock + file: ./src/mapping.ts \ No newline at end of file diff --git a/tests/integration-tests/grafted/abis/Contract.abi b/tests/integration-tests/grafted/abis/Contract.abi new file mode 100644 index 00000000000..02da1a9e7f3 --- /dev/null +++ b/tests/integration-tests/grafted/abis/Contract.abi @@ -0,0 +1,33 @@ +[ + { + "inputs": [], + "stateMutability": "nonpayable", + "type": "constructor" + }, + { + "anonymous": false, + "inputs": [ + { + "indexed": false, + "internalType": "uint16", + "name": "x", + "type": "uint16" + } + ], + "name": "Trigger", + "type": "event" + }, + { + "inputs": [ + { + "internalType": "uint16", + "name": "x", + "type": "uint16" + } + ], + "name": "emitTrigger", + "outputs": [], + "stateMutability": "nonpayable", + "type": "function" + } +] diff --git a/tests/integration-tests/grafted/package.json b/tests/integration-tests/grafted/package.json new file mode 100644 index 00000000000..d45b6fc6727 --- /dev/null +++ b/tests/integration-tests/grafted/package.json @@ -0,0 +1,25 @@ +{ + "name": "grafted-subgraph", + "version": "0.1.0", + "scripts": { + "build-contracts": "../../common/build-contracts.sh", + "codegen": "graph codegen --skip-migrations", + "test": "yarn build-contracts && truffle test --compile-none --network test", + "create:test": "graph create test/grafted-subgraph --node $GRAPH_NODE_ADMIN_URI", + "deploy:test": "graph deploy test/grafted-subgraph --version-label v0.0.1 --ipfs $IPFS_URI --node $GRAPH_NODE_ADMIN_URI" + }, + "devDependencies": { + "@graphprotocol/graph-cli": "0.69.0", + "@graphprotocol/graph-ts": "0.34.0", + "solc": "^0.8.2" + }, + "dependencies": { + "@truffle/contract": "^4.3", + "@truffle/hdwallet-provider": "^1.2", + "apollo-fetch": "^0.7.0", + "babel-polyfill": "^6.26.0", + "babel-register": "^6.26.0", + "gluegun": "^4.6.1", + "truffle": "^5.2" + } +} \ No newline at end of file diff --git a/tests/integration-tests/grafted/schema.graphql b/tests/integration-tests/grafted/schema.graphql new file mode 100644 index 00000000000..b83083fd466 --- /dev/null +++ b/tests/integration-tests/grafted/schema.graphql @@ -0,0 +1,5 @@ +type GraftedData @entity(immutable: true) { + id: ID! + data: String! + blockNumber: BigInt! +} \ No newline at end of file diff --git a/tests/integration-tests/grafted/src/mapping.ts b/tests/integration-tests/grafted/src/mapping.ts new file mode 100644 index 00000000000..742d5d67c54 --- /dev/null +++ b/tests/integration-tests/grafted/src/mapping.ts @@ -0,0 +1,9 @@ +import { ethereum } from '@graphprotocol/graph-ts' +import { GraftedData } from '../generated/schema' + +export function handleBlock(block: ethereum.Block): void { + let entity = new GraftedData(block.number.toString()) + entity.data = 'to grafted' + entity.blockNumber = block.number + entity.save() +} \ No newline at end of file diff --git a/tests/integration-tests/grafted/subgraph.yaml b/tests/integration-tests/grafted/subgraph.yaml new file mode 100644 index 00000000000..f946f201941 --- /dev/null +++ b/tests/integration-tests/grafted/subgraph.yaml @@ -0,0 +1,30 @@ +specVersion: 0.0.6 +description: Grafted Subgraph +repository: https://github.com/graphprotocol/graph-node +schema: + file: ./schema.graphql +dataSources: + - kind: ethereum/contract + name: SimpleContract + network: test + source: + address: "0x5FbDB2315678afecb367f032d93F642f64180aa3" + abi: SimpleContract + startBlock: 0 + mapping: + kind: ethereum/events + apiVersion: 0.0.6 + language: wasm/assemblyscript + entities: + - GraftedData + abis: + - name: SimpleContract + file: ./abis/Contract.abi + blockHandlers: + - handler: handleBlock + file: ./src/mapping.ts +features: + - grafting +graft: + base: QmQpiC9bJGFssQfeZippfQ7rcTv7QA67X7jUejc8nV125F + block: 2 \ No newline at end of file diff --git a/tests/src/contract.rs b/tests/src/contract.rs index 4fdf767b041..05fda947839 100644 --- a/tests/src/contract.rs +++ b/tests/src/contract.rs @@ -7,7 +7,7 @@ use graph::prelude::{ api::{Eth, Namespace}, contract::{tokens::Tokenize, Contract as Web3Contract, Options}, transports::Http, - types::{Address, Bytes, TransactionReceipt}, + types::{Address, Block, BlockId, BlockNumber, Bytes, TransactionReceipt, H256}, }, }; // web3 version 0.18 does not expose this; once the graph crate updates to @@ -165,4 +165,13 @@ impl Contract { } Ok(contracts) } + + pub async fn latest_block() -> Option> { + let eth = Self::eth(); + let block = eth + .block(BlockId::Number(BlockNumber::Latest)) + .await + .unwrap_or_default(); + block + } } diff --git a/tests/src/subgraph.rs b/tests/src/subgraph.rs index 810b87cbb78..92e42836b68 100644 --- a/tests/src/subgraph.rs +++ b/tests/src/subgraph.rs @@ -164,7 +164,7 @@ impl Subgraph { } /// Make a GraphQL query to the index node API - pub async fn index_with_vars(&self, text: &str, vars: Value) -> anyhow::Result { + pub async fn query_with_vars(text: &str, vars: Value) -> anyhow::Result { let endpoint = CONFIG.graph_node.index_node_uri(); graphql_query_with_vars(&endpoint, text, vars).await } diff --git a/tests/tests/integration_tests.rs b/tests/tests/integration_tests.rs index 5c6ab96968d..9df36f7145a 100644 --- a/tests/tests/integration_tests.rs +++ b/tests/tests/integration_tests.rs @@ -11,7 +11,7 @@ use std::future::Future; use std::pin::Pin; -use std::time::{Duration, Instant}; +use std::time::{self, Duration, Instant}; use anyhow::{anyhow, bail, Context, Result}; use graph::futures03::StreamExt; @@ -25,6 +25,8 @@ use tokio::process::{Child, Command}; use tokio::task::JoinError; use tokio::time::sleep; +const SUBGRAPH_LAST_GRAFTING_BLOCK: i32 = 3; + type TestFn = Box< dyn FnOnce(TestContext) -> Pin> + Send>> + Sync @@ -110,6 +112,15 @@ impl TestCase { } } + fn new_with_grafting(name: &str, test: fn(TestContext) -> T, base_subgraph: &str) -> Self + where + T: Future> + Send + 'static, + { + let mut test_case = Self::new(name, test); + test_case.source_subgraph = Some(base_subgraph.to_string()); + test_case + } + fn new_with_source_subgraph( name: &str, test: fn(TestContext) -> T, @@ -246,7 +257,7 @@ impl TestCase { let subgraph = self.deploy_and_wait(source, contracts).await?; status!( source, - "source subgraph deployed with hash {}", + "Source subgraph deployed with hash {}", subgraph.deployment ); } @@ -456,9 +467,8 @@ async fn test_block_handlers(ctx: TestContext) -> anyhow::Result<()> { .await?; // test subgraphFeatures endpoint returns handlers correctly - let subgraph_features = subgraph - .index_with_vars( - "query GetSubgraphFeatures($deployment: String!) { + let subgraph_features = Subgraph::query_with_vars( + "query GetSubgraphFeatures($deployment: String!) { subgraphFeatures(subgraphId: $deployment) { specVersion apiVersion @@ -468,9 +478,9 @@ async fn test_block_handlers(ctx: TestContext) -> anyhow::Result<()> { handlers } }", - json!({ "deployment": subgraph.deployment }), - ) - .await?; + json!({ "deployment": subgraph.deployment }), + ) + .await?; let handlers = &subgraph_features["data"]["subgraphFeatures"]["handlers"]; assert!( handlers.is_array(), @@ -697,9 +707,8 @@ async fn test_non_fatal_errors(ctx: TestContext) -> anyhow::Result<()> { } }"; - let resp = subgraph - .index_with_vars(query, json!({ "deployment" : subgraph.deployment })) - .await?; + let resp = + Subgraph::query_with_vars(query, json!({ "deployment" : subgraph.deployment })).await?; let subgraph_features = &resp["data"]["subgraphFeatures"]; let exp = json!({ "specVersion": "0.0.4", @@ -796,6 +805,82 @@ async fn test_remove_then_update(ctx: TestContext) -> anyhow::Result<()> { Ok(()) } +async fn test_subgraph_grafting(ctx: TestContext) -> anyhow::Result<()> { + async fn get_block_hash(block_number: i32) -> Option { + const FETCH_BLOCK_HASH: &str = r#" + query blockHashFromNumber($network: String!, $blockNumber: Int!) { + hash: blockHashFromNumber( + network: $network, + blockNumber: $blockNumber, + ) } "#; + let vars = json!({ + "network": "test", + "blockNumber": block_number + }); + + let resp = Subgraph::query_with_vars(FETCH_BLOCK_HASH, vars) + .await + .unwrap(); + assert_eq!(None, resp.get("errors")); + resp["data"]["hash"].as_str().map(|s| s.to_owned()) + } + + let subgraph = ctx.subgraph; + + assert!(subgraph.healthy); + + let block_hashes: Vec<&str> = vec![ + "384c705d4d1933ae8ba89026f016f09854057a267e1143e47bb7511d772a35d4", + "b90423eead33404dae0684169d35edd494b36802b721fb8de0bb8bc036c10480", + "2a6c4b65d659e0485371a93bc1ac0f0d7bc0f25a454b5f23a842335fea0638d5", + ]; + + let pois: Vec<&str> = vec![ + "0xde9e5650e22e61def6990d3fc4bd5915a4e8e0dd54af0b6830bf064aab16cc03", + "0x5d790dca3e37bd9976345d32d437b84ba5ea720a0b6ea26231a866e9f078bd52", + "0x719c04b78e01804c86f2bd809d20f481e146327af07227960e2242da365754ef", + ]; + + for i in 1..4 { + let block_hash = get_block_hash(i).await.unwrap(); + // We need to make sure that the preconditions for POI are fulfiled + // namely that the blockchain produced the proper block hashes for the + // blocks of which we will check the POI. + assert_eq!(block_hash, block_hashes[(i - 1) as usize]); + + const FETCH_POI: &str = r#" + query proofOfIndexing($subgraph: String!, $blockNumber: Int!, $blockHash: String!, $indexer: String!) { + proofOfIndexing( + subgraph: $subgraph, + blockNumber: $blockNumber, + blockHash: $blockHash, + indexer: $indexer + ) } "#; + + let zero_addr = "0000000000000000000000000000000000000000"; + let vars = json!({ + "subgraph": subgraph.deployment, + "blockNumber": i, + "blockHash": block_hash, + "indexer": zero_addr, + }); + let resp = Subgraph::query_with_vars(FETCH_POI, vars).await?; + assert_eq!(None, resp.get("errors")); + assert!(resp["data"]["proofOfIndexing"].is_string()); + let poi = resp["data"]["proofOfIndexing"].as_str().unwrap(); + // Check the expected value of the POI. The transition from the old legacy + // hashing to the new one is done in the block #2 anything before that + // should not change as the legacy code will not be updated. Any change + // after that might indicate a change in the way new POI is now calculated. + // Change on the block #2 would mean a change in the transitioning + // from the old to the new algorithm hence would be reflected only + // subgraphs that are grafting from pre 0.0.5 to 0.0.6 or newer. + assert_eq!(poi, pois[(i - 1) as usize]); + } + + Ok(()) +} + async fn test_poi_for_failed_subgraph(ctx: TestContext) -> anyhow::Result<()> { let subgraph = ctx.subgraph; const INDEXING_STATUS: &str = r#" @@ -829,9 +914,9 @@ async fn test_poi_for_failed_subgraph(ctx: TestContext) -> anyhow::Result<()> { } async fn fetch_status(subgraph: &Subgraph) -> anyhow::Result { - let resp = subgraph - .index_with_vars(INDEXING_STATUS, json!({ "subgraphName": subgraph.name })) - .await?; + let resp = + Subgraph::query_with_vars(INDEXING_STATUS, json!({ "subgraphName": subgraph.name })) + .await?; assert_eq!(None, resp.get("errors")); let statuses = &resp["data"]["statuses"]; assert_eq!(1, statuses.as_array().unwrap().len()); @@ -877,7 +962,7 @@ async fn test_poi_for_failed_subgraph(ctx: TestContext) -> anyhow::Result<()> { "blockNumber": block_number, "blockHash": status.latest_block["hash"], }); - let resp = subgraph.index_with_vars(FETCH_POI, vars).await?; + let resp = Subgraph::query_with_vars(FETCH_POI, vars).await?; assert_eq!(None, resp.get("errors")); assert!(resp["data"]["proofOfIndexing"].is_string()); Ok(()) @@ -915,6 +1000,25 @@ async fn test_multiple_subgraph_datasources(ctx: TestContext) -> anyhow::Result< Ok(()) } +async fn wait_for_blockchain_block(block_number: i32) -> bool { + // Wait up to 5 minutes for the expected block to appear + const STATUS_WAIT: Duration = Duration::from_secs(300); + const REQUEST_REPEATING: Duration = time::Duration::from_secs(1); + let start = Instant::now(); + while start.elapsed() < STATUS_WAIT { + let latest_block = Contract::latest_block().await; + if let Some(latest_block) = latest_block { + if let Some(number) = latest_block.number { + if number >= block_number.into() { + return true; + } + } + } + tokio::time::sleep(REQUEST_REPEATING).await; + } + false +} + /// The main test entrypoint. #[tokio::test] async fn integration_tests() -> anyhow::Result<()> { @@ -936,6 +1040,7 @@ async fn integration_tests() -> anyhow::Result<()> { TestCase::new("timestamp", test_timestamp), TestCase::new("ethereum-api-tests", test_eth_api), TestCase::new("topic-filter", test_topic_filters), + TestCase::new_with_grafting("grafted", test_subgraph_grafting, "base"), TestCase::new_with_source_subgraph( "subgraph-data-sources", subgraph_data_sources, @@ -958,6 +1063,11 @@ async fn integration_tests() -> anyhow::Result<()> { cases }; + // Here we wait for a block in the blockchain in order not to influence + // block hashes for all the blocks until the end of the grafting tests. + // Currently the last used block for grafting test is the block 3. + assert!(wait_for_blockchain_block(SUBGRAPH_LAST_GRAFTING_BLOCK).await); + let contracts = Contract::deploy_all().await?; status!("setup", "Resetting database"); From baf33244fb1f10071f3b8387b17dbe6869c611b0 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Tue, 8 Apr 2025 16:29:03 -0700 Subject: [PATCH 148/150] graph, store: Remove GRAPH_STORE_LAST_ROLLUP_FROM_POI This flag was only meant as a safety switch in case the fixed behavior caused trouble. Since it's not been needed in several months, it's safe to remove it. --- graph/src/env/store.rs | 11 ---------- store/postgres/src/deployment_store.rs | 12 ++--------- store/postgres/src/relational.rs | 29 +------------------------- store/postgres/src/writable.rs | 8 +++---- 4 files changed, 6 insertions(+), 54 deletions(-) diff --git a/graph/src/env/store.rs b/graph/src/env/store.rs index 3ecf92e0388..8197d07b6bc 100644 --- a/graph/src/env/store.rs +++ b/graph/src/env/store.rs @@ -129,14 +129,6 @@ pub struct EnvVarsStore { pub use_brin_for_all_query_types: bool, /// Temporary env var to disable certain lookups in the chain store pub disable_block_cache_for_lookup: bool, - /// Temporary env var to fall back to the old broken way of determining - /// the time of the last rollup from the POI table instead of the new - /// way that fixes - /// https://github.com/graphprotocol/graph-node/issues/5530 Remove this - /// and all code that is dead as a consequence once this has been vetted - /// sufficiently, probably after 2024-12-01 - /// Defaults to `false`, i.e. using the new fixed behavior - pub last_rollup_from_poi: bool, /// Safety switch to increase the number of columns used when /// calculating the chunk size in `InsertQuery::chunk_size`. This can be /// used to work around Postgres errors complaining 'number of @@ -197,7 +189,6 @@ impl TryFrom for EnvVarsStore { create_gin_indexes: x.create_gin_indexes, use_brin_for_all_query_types: x.use_brin_for_all_query_types, disable_block_cache_for_lookup: x.disable_block_cache_for_lookup, - last_rollup_from_poi: x.last_rollup_from_poi, insert_extra_cols: x.insert_extra_cols, fdw_fetch_size: x.fdw_fetch_size, }; @@ -276,8 +267,6 @@ pub struct InnerStore { use_brin_for_all_query_types: bool, #[envconfig(from = "GRAPH_STORE_DISABLE_BLOCK_CACHE_FOR_LOOKUP", default = "false")] disable_block_cache_for_lookup: bool, - #[envconfig(from = "GRAPH_STORE_LAST_ROLLUP_FROM_POI", default = "false")] - last_rollup_from_poi: bool, #[envconfig(from = "GRAPH_STORE_INSERT_EXTRA_COLS", default = "0")] insert_extra_cols: usize, #[envconfig(from = "GRAPH_STORE_FDW_FETCH_SIZE", default = "1000")] diff --git a/store/postgres/src/deployment_store.rs b/store/postgres/src/deployment_store.rs index e07b4659436..92de85f316e 100644 --- a/store/postgres/src/deployment_store.rs +++ b/store/postgres/src/deployment_store.rs @@ -904,20 +904,12 @@ impl DeploymentStore { .await } - pub(crate) fn block_time( - &self, - site: Arc, - block: BlockNumber, - ) -> Result, StoreError> { + pub(crate) fn block_time(&self, site: Arc) -> Result, StoreError> { let store = self.cheap_clone(); let mut conn = self.get_conn()?; let layout = store.layout(&mut conn, site.cheap_clone())?; - if ENV_VARS.store.last_rollup_from_poi { - layout.block_time(&mut conn, block) - } else { - layout.last_rollup(&mut conn) - } + layout.last_rollup(&mut conn) } pub(crate) async fn get_proof_of_indexing( diff --git a/store/postgres/src/relational.rs b/store/postgres/src/relational.rs index d148060efc2..fb181b7e74d 100644 --- a/store/postgres/src/relational.rs +++ b/store/postgres/src/relational.rs @@ -32,7 +32,6 @@ use graph::blockchain::block_stream::{EntityOperationKind, EntitySourceOperation use graph::blockchain::BlockTime; use graph::cheap_clone::CheapClone; use graph::components::store::write::{RowGroup, WriteChunk}; -use graph::components::subgraph::PoICausalityRegion; use graph::constraint_violation; use graph::data::graphql::TypeExt as _; use graph::data::query::Trace; @@ -69,7 +68,7 @@ use crate::{ }, }; use graph::components::store::{AttributeNames, DerivedEntityQuery}; -use graph::data::store::{Id, IdList, IdType, BYTES_SCALAR}; +use graph::data::store::{IdList, IdType, BYTES_SCALAR}; use graph::data::subgraph::schema::POI_TABLE; use graph::prelude::{ anyhow, info, BlockNumber, DeploymentHash, Entity, EntityOperation, Logger, @@ -1113,32 +1112,6 @@ impl Layout { Ok(Arc::new(layout)) } - pub(crate) fn block_time( - &self, - conn: &mut PgConnection, - block: BlockNumber, - ) -> Result, StoreError> { - let block_time_name = self.input_schema.poi_block_time(); - let poi_type = self.input_schema.poi_type(); - let id = Id::String(Word::from(PoICausalityRegion::from_network( - &self.site.network, - ))); - let key = poi_type.key(id); - - let block_time = self - .find(conn, &key, block)? - .and_then(|entity| { - entity.get(&block_time_name).map(|value| { - value - .as_int8() - .ok_or_else(|| constraint_violation!("block_time must have type Int8")) - }) - }) - .transpose()? - .map(|value| BlockTime::since_epoch(value, 0)); - Ok(block_time) - } - /// Find the time of the last rollup for the subgraph. We do this by /// looking for the maximum timestamp in any aggregation table and /// adding a little bit more than the corresponding interval to it. This diff --git a/store/postgres/src/writable.rs b/store/postgres/src/writable.rs index 26e559bcbc9..3d85042d07c 100644 --- a/store/postgres/src/writable.rs +++ b/store/postgres/src/writable.rs @@ -95,8 +95,8 @@ impl LastRollup { let kind = match (has_aggregations, block) { (false, _) => LastRollup::NotNeeded, (true, None) => LastRollup::Unknown, - (true, Some(block)) => { - let block_time = store.block_time(site, block)?; + (true, Some(_)) => { + let block_time = store.block_time(site)?; block_time .map(|b| LastRollup::Some(b)) .unwrap_or(LastRollup::Unknown) @@ -240,9 +240,7 @@ impl SyncStore { firehose_cursor, )?; - let block_time = self - .writable - .block_time(self.site.cheap_clone(), block_ptr_to.number)?; + let block_time = self.writable.block_time(self.site.cheap_clone())?; self.last_rollup.set(block_time) }) } From e6f55b174da714791d920c5f53ffdf942d808566 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Fri, 11 Apr 2025 14:45:47 -0700 Subject: [PATCH 149/150] store: Do not drop the default copy connection when running out of work When the default worker finished while other workers were still copying, and there was no more work to do, we inadvertently dropped the default connection. --- store/postgres/src/copy.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/store/postgres/src/copy.rs b/store/postgres/src/copy.rs index 22ddee394f6..2e8807b2aa8 100644 --- a/store/postgres/src/copy.rs +++ b/store/postgres/src/copy.rs @@ -1020,6 +1020,7 @@ impl Connection { return None; }; let Some(table) = state.unfinished.pop() else { + self.conn = Some(conn); return None; }; From 23427029ea4b614c0e02a76e73f11e0ebe1e3a92 Mon Sep 17 00:00:00 2001 From: David Lutterkort Date: Sat, 12 Apr 2025 12:29:03 -0700 Subject: [PATCH 150/150] node: Make number of blocking threads configurable --- node/src/main.rs | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/node/src/main.rs b/node/src/main.rs index 9b0e94250dc..b2003dff28f 100644 --- a/node/src/main.rs +++ b/node/src/main.rs @@ -78,8 +78,21 @@ fn read_expensive_queries( Ok(queries) } -#[tokio::main] -async fn main() { +fn main() { + let max_blocking: usize = std::env::var("GRAPH_MAX_BLOCKING_THREADS") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(512); + + tokio::runtime::Builder::new_multi_thread() + .enable_all() + .max_blocking_threads(max_blocking) + .build() + .unwrap() + .block_on(async { main_inner().await }) +} + +async fn main_inner() { env_logger::init(); let env_vars = Arc::new(EnvVars::from_env().unwrap());