From 6fded3f402e4a6d02734d07e227d224edd6dcd15 Mon Sep 17 00:00:00 2001 From: Ahmed Farghal Date: Thu, 20 Jun 2024 13:29:30 +0100 Subject: [PATCH] [reorg] Stage 1 - Remove schema/schema-api (now in `restate_types::schema`) - Remove cluster-controller and merge with restate-admin - Use `prost-dto` for cluster-state - Move networking types to `restate_types::net` - Protobuf internal types reorg --- Cargo.lock | 174 ++--- Cargo.toml | 6 +- cli/src/clients/admin_interface.rs | 1 + cli/src/clients/datafusion_helpers.rs | 20 +- cli/src/commands/deployments/describe.rs | 10 +- cli/src/commands/deployments/list.rs | 2 +- cli/src/commands/deployments/register.rs | 5 +- cli/src/commands/deployments/remove.rs | 2 +- cli/src/commands/services/list.rs | 2 +- .../commands/services/status/agg_status.rs | 2 +- .../services/status/detailed_status.rs | 2 +- cli/src/commands/services/status/mod.rs | 19 +- cli/src/commands/state/util.rs | 3 +- cli/src/ui/deployments.rs | 5 +- cli/src/ui/service_handlers.rs | 5 +- crates/admin-rest-model/Cargo.toml | 3 +- crates/admin-rest-model/src/deployments.rs | 12 +- crates/admin-rest-model/src/handlers.rs | 5 +- crates/admin-rest-model/src/services.rs | 6 +- crates/admin-rest-model/src/subscriptions.rs | 6 +- crates/admin/Cargo.toml | 23 +- crates/{node-services => admin}/build.rs | 23 +- crates/admin/protobuf/cluster_ctrl_svc.proto | 33 + .../src/cluster_controller}/cluster_state.rs | 58 +- .../src/cluster_controller/mod.rs} | 6 +- .../admin/src/cluster_controller/protobuf.rs | 14 + .../src/cluster_controller}/service.rs | 46 +- crates/admin/src/lib.rs | 1 + crates/admin/src/rest_api/deployments.rs | 2 +- crates/admin/src/rest_api/handlers.rs | 1 + crates/admin/src/rest_api/mod.rs | 2 +- crates/admin/src/rest_api/services.rs | 1 + crates/admin/src/rest_api/subscriptions.rs | 2 +- crates/admin/src/schema_registry/error.rs | 6 +- crates/admin/src/schema_registry/mod.rs | 16 +- crates/admin/src/schema_registry/updater.rs | 26 +- crates/admin/src/service.rs | 4 +- crates/admin/src/state.rs | 2 +- crates/admin/src/storage_query/query.rs | 2 +- crates/cluster-controller/Cargo.toml | 37 - crates/core/Cargo.toml | 2 - crates/core/src/metadata/manager.rs | 5 +- crates/core/src/metadata/mod.rs | 5 +- crates/core/src/network/error.rs | 2 +- crates/core/src/network/message_router.rs | 12 +- crates/core/src/network/network_sender.rs | 2 +- crates/core/src/test_env.rs | 12 +- crates/ingress-dispatcher/Cargo.toml | 2 - crates/ingress-dispatcher/src/dispatcher.rs | 7 +- crates/ingress-dispatcher/src/lib.rs | 12 +- crates/ingress-http/Cargo.toml | 4 +- crates/ingress-http/src/handler/error.rs | 2 +- crates/ingress-http/src/handler/health.rs | 9 +- crates/ingress-http/src/handler/invocation.rs | 15 +- crates/ingress-http/src/handler/mod.rs | 4 +- .../ingress-http/src/handler/path_parsing.rs | 7 +- crates/ingress-http/src/handler/responses.rs | 10 +- .../src/handler/service_handler.rs | 28 +- crates/ingress-http/src/handler/tests.rs | 32 +- crates/ingress-http/src/handler/workflow.rs | 15 +- crates/ingress-http/src/lib.rs | 22 +- crates/ingress-http/src/server.rs | 4 +- crates/ingress-kafka/Cargo.toml | 3 - crates/ingress-kafka/src/consumer_task.rs | 12 +- .../src/subscription_controller.rs | 2 +- crates/invoker-impl/Cargo.toml | 2 - .../invoker-impl/src/invocation_task/mod.rs | 2 +- .../service_protocol_runner.rs | 6 +- crates/invoker-impl/src/lib.rs | 4 +- crates/metadata-store/Cargo.toml | 1 - crates/network/Cargo.toml | 9 +- crates/network/build.rs | 30 + .../proto => network/protobuf}/node_svc.proto | 22 +- crates/network/src/connection.rs | 12 +- crates/network/src/connection_manager.rs | 22 +- crates/network/src/error.rs | 2 +- crates/network/src/handshake.rs | 4 +- crates/network/src/lib.rs | 1 + crates/network/src/networking.rs | 2 +- crates/network/src/protobuf.rs | 16 + crates/network/src/rpc_router.rs | 12 +- crates/node-protocol/Cargo.toml | 33 - crates/node-protocol/build.rs | 41 - crates/node-protocol/src/common.rs | 170 ---- crates/node-protocol/src/lib.rs | 181 ----- crates/node-protocol/src/node.rs | 108 --- crates/node-services/Cargo.toml | 27 - .../proto/cluster_ctrl_svc.proto | 79 -- crates/node-services/src/lib.rs | 34 - crates/node/Cargo.toml | 8 +- crates/node/src/lib.rs | 3 +- .../network_server/handler/cluster_ctrl.rs | 103 +-- .../node/src/network_server/handler/node.rs | 16 +- crates/node/src/network_server/service.rs | 18 +- crates/node/src/roles/admin.rs | 8 +- crates/node/src/roles/worker.rs | 7 +- crates/schema-api/Cargo.toml | 37 - crates/schema-api/src/lib.rs | 728 ------------------ crates/schema/Cargo.toml | 28 - crates/schema/src/deployment.rs | 107 --- crates/schema/src/invocation_target.rs | 41 - crates/schema/src/service.rs | 97 --- crates/schema/src/subscriptions.rs | 47 -- crates/service-protocol/Cargo.toml | 3 +- crates/service-protocol/src/discovery.rs | 2 +- crates/storage-query-datafusion/Cargo.toml | 2 - .../storage-query-datafusion/src/context.rs | 4 +- .../src/deployment/row.rs | 2 +- .../src/deployment/table.rs | 2 +- crates/storage-query-datafusion/src/mocks.rs | 21 +- .../src/service/row.rs | 2 +- .../src/service/table.rs | 2 +- crates/types/Cargo.toml | 5 +- crates/types/build.rs | 37 +- crates/types/protobuf/restate/cluster.proto | 68 ++ .../protobuf/restate}/common.proto | 11 +- .../protobuf/restate}/node.proto | 18 +- crates/types/src/cluster/cluster_state.rs | 128 +++ crates/types/src/cluster/mod.rs | 11 + crates/types/src/identifiers.rs | 13 + crates/types/src/lib.rs | 4 +- crates/types/src/logs/mod.rs | 13 + crates/types/src/net.rs | 112 --- .../src/net}/cluster_controller.rs | 9 +- .../src => types/src/net}/codec.rs | 12 +- .../src => types/src/net}/error.rs | 2 +- .../src => types/src/net}/ingress.rs | 5 +- .../src => types/src/net}/metadata.rs | 15 +- crates/types/src/net/mod.rs | 328 ++++++++ .../src/net}/partition_processor_manager.rs | 9 +- .../src => types/src/net}/status.rs | 0 crates/types/src/node_id.rs | 34 + crates/types/src/partition_table.rs | 17 +- crates/types/src/processors/mod.rs | 66 -- crates/types/src/protobuf.rs | 159 ++++ crates/types/src/schema/deployment.rs | 376 +++++++++ .../src/schema}/invocation_target.rs | 63 +- .../src/lib.rs => types/src/schema/mod.rs} | 39 +- crates/types/src/schema/service.rs | 291 +++++++ crates/types/src/schema/subscriptions.rs | 262 +++++++ crates/types/src/version.rs | 14 + crates/worker/Cargo.toml | 4 - crates/worker/src/invoker_integration.rs | 11 +- crates/worker/src/lib.rs | 8 +- crates/worker/src/partition/leadership/mod.rs | 2 +- crates/worker/src/partition/mod.rs | 11 +- .../worker/src/partition_processor_manager.rs | 22 +- crates/worker/src/subscription_controller.rs | 2 +- crates/worker/src/subscription_integration.rs | 10 +- tools/restatectl/Cargo.toml | 3 +- .../src/commands/dump/cluster_state.rs | 18 +- tools/xtask/Cargo.toml | 3 +- tools/xtask/src/main.rs | 19 +- 153 files changed, 2430 insertions(+), 2737 deletions(-) rename crates/{node-services => admin}/build.rs (50%) create mode 100644 crates/admin/protobuf/cluster_ctrl_svc.proto rename crates/{cluster-controller/src => admin/src/cluster_controller}/cluster_state.rs (81%) rename crates/{cluster-controller/src/lib.rs => admin/src/cluster_controller/mod.rs} (86%) create mode 100644 crates/admin/src/cluster_controller/protobuf.rs rename crates/{cluster-controller/src => admin/src/cluster_controller}/service.rs (95%) delete mode 100644 crates/cluster-controller/Cargo.toml create mode 100644 crates/network/build.rs rename crates/{node-services/proto => network/protobuf}/node_svc.proto (67%) create mode 100644 crates/network/src/protobuf.rs delete mode 100644 crates/node-protocol/Cargo.toml delete mode 100644 crates/node-protocol/build.rs delete mode 100644 crates/node-protocol/src/common.rs delete mode 100644 crates/node-protocol/src/lib.rs delete mode 100644 crates/node-protocol/src/node.rs delete mode 100644 crates/node-services/Cargo.toml delete mode 100644 crates/node-services/proto/cluster_ctrl_svc.proto delete mode 100644 crates/node-services/src/lib.rs delete mode 100644 crates/schema-api/Cargo.toml delete mode 100644 crates/schema-api/src/lib.rs delete mode 100644 crates/schema/Cargo.toml delete mode 100644 crates/schema/src/deployment.rs delete mode 100644 crates/schema/src/invocation_target.rs delete mode 100644 crates/schema/src/service.rs delete mode 100644 crates/schema/src/subscriptions.rs create mode 100644 crates/types/protobuf/restate/cluster.proto rename crates/{node-protocol/proto => types/protobuf/restate}/common.proto (84%) rename crates/{node-protocol/proto => types/protobuf/restate}/node.proto (74%) create mode 100644 crates/types/src/cluster/cluster_state.rs create mode 100644 crates/types/src/cluster/mod.rs delete mode 100644 crates/types/src/net.rs rename crates/{node-protocol/src => types/src/net}/cluster_controller.rs (85%) rename crates/{node-protocol/src => types/src/net}/codec.rs (94%) rename crates/{node-protocol/src => types/src/net}/error.rs (93%) rename crates/{node-protocol/src => types/src/net}/ingress.rs (84%) rename crates/{node-protocol/src => types/src/net}/metadata.rs (86%) create mode 100644 crates/types/src/net/mod.rs rename crates/{node-protocol/src => types/src/net}/partition_processor_manager.rs (86%) rename crates/{node-protocol/src => types/src/net}/status.rs (100%) delete mode 100644 crates/types/src/processors/mod.rs create mode 100644 crates/types/src/protobuf.rs create mode 100644 crates/types/src/schema/deployment.rs rename crates/{schema-api/src => types/src/schema}/invocation_target.rs (93%) rename crates/{schema/src/lib.rs => types/src/schema/mod.rs} (86%) create mode 100644 crates/types/src/schema/service.rs create mode 100644 crates/types/src/schema/subscriptions.rs diff --git a/Cargo.lock b/Cargo.lock index 144b217388..d6b92deaf6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4971,6 +4971,40 @@ dependencies = [ "syn 2.0.65", ] +[[package]] +name = "prost-dto" +version = "0.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5145d3fa63cf018899b3fdb7595a84597ac4315a2296663d81b15d87994e8a" +dependencies = [ + "prost-dto-core", + "prost-dto-derive", +] + +[[package]] +name = "prost-dto-core" +version = "0.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfc357b4d026d9c61112e75827e7f01e6b7fbe38664836194b19c063ca124ba" +dependencies = [ + "chrono", + "darling 0.20.8", + "proc-macro2", + "quote", + "syn 2.0.65", +] + +[[package]] +name = "prost-dto-derive" +version = "0.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4feef623adf0ea256c19bdc3fbcad12060be6b3fb5f29bd471b1eeb02b3d33f6" +dependencies = [ + "proc-macro2", + "prost-dto-core", + "syn 2.0.65", +] + [[package]] name = "prost-types" version = "0.12.3" @@ -5251,6 +5285,7 @@ name = "restate-admin" version = "1.1.0" dependencies = [ "anyhow", + "arc-swap", "arrow-flight", "axum", "bytes", @@ -5260,18 +5295,20 @@ dependencies = [ "derive_builder", "derive_more", "futures", + "googletest", "http 0.2.12", "hyper 0.14.28", "okapi-operation", + "prost", + "prost-dto", + "prost-types", "restate-admin-rest-model", "restate-bifrost", "restate-core", "restate-errors", "restate-fs-util", "restate-futures-util", - "restate-node-services", - "restate-schema", - "restate-schema-api", + "restate-network", "restate-serde-util", "restate-service-client", "restate-service-protocol", @@ -5287,6 +5324,7 @@ dependencies = [ "thiserror", "tokio", "tonic 0.10.2", + "tonic-build", "tower", "tracing", "tracing-subscriber", @@ -5299,7 +5337,6 @@ dependencies = [ "bytes", "http 0.2.12", "humantime", - "restate-schema-api", "restate-serde-util", "restate-types", "schemars", @@ -5468,29 +5505,6 @@ dependencies = [ "unicode-width", ] -[[package]] -name = "restate-cluster-controller" -version = "1.1.0" -dependencies = [ - "anyhow", - "arc-swap", - "codederror", - "derive_builder", - "futures", - "googletest", - "restate-bifrost", - "restate-core", - "restate-network", - "restate-node-protocol", - "restate-types", - "schemars", - "serde", - "test-log", - "thiserror", - "tokio", - "tracing", -] - [[package]] name = "restate-core" version = "1.1.0" @@ -5510,8 +5524,6 @@ dependencies = [ "humantime", "metrics", "rand", - "restate-node-protocol", - "restate-schema", "restate-test-util", "restate-types", "schemars", @@ -5593,8 +5605,6 @@ dependencies = [ "googletest", "restate-bifrost", "restate-core", - "restate-node-protocol", - "restate-schema-api", "restate-storage-api", "restate-test-util", "restate-types", @@ -5631,7 +5641,6 @@ dependencies = [ "restate-core", "restate-errors", "restate-ingress-dispatcher", - "restate-schema-api", "restate-serde-util", "restate-service-protocol", "restate-test-util", @@ -5665,8 +5674,6 @@ dependencies = [ "restate-core", "restate-errors", "restate-ingress-dispatcher", - "restate-schema", - "restate-schema-api", "restate-test-util", "restate-timer-queue", "restate-types", @@ -5720,7 +5727,6 @@ dependencies = [ "restate-futures-util", "restate-invoker-api", "restate-queue", - "restate-schema-api", "restate-service-client", "restate-service-protocol", "restate-test-util", @@ -5758,7 +5764,6 @@ dependencies = [ "prost-types", "restate-core", "restate-grpc-util", - "restate-node-protocol", "restate-rocksdb", "restate-types", "rocksdb", @@ -5795,11 +5800,11 @@ dependencies = [ "metrics", "once_cell", "pin-project", + "prost", + "prost-types", "rand", "restate-core", "restate-grpc-util", - "restate-node-protocol", - "restate-node-services", "restate-test-util", "restate-types", "static_assertions", @@ -5807,6 +5812,7 @@ dependencies = [ "tokio", "tokio-stream", "tonic 0.10.2", + "tonic-build", "tower", "tracing", "tracing-subscriber", @@ -5841,17 +5847,12 @@ dependencies = [ "prost-types", "restate-admin", "restate-bifrost", - "restate-cluster-controller", "restate-core", "restate-errors", "restate-grpc-util", "restate-metadata-store", "restate-network", - "restate-node-protocol", - "restate-node-services", "restate-rocksdb", - "restate-schema", - "restate-schema-api", "restate-service-client", "restate-service-protocol", "restate-storage-query-datafusion", @@ -5879,42 +5880,6 @@ dependencies = [ "tracing-subscriber", ] -[[package]] -name = "restate-node-protocol" -version = "1.1.0" -dependencies = [ - "bytes", - "bytestring", - "derive_more", - "enum-map", - "flexbuffers", - "prost", - "prost-build", - "prost-types", - "restate-schema", - "restate-types", - "serde", - "serde_with", - "strum 0.26.2", - "strum_macros 0.26.2", - "thiserror", -] - -[[package]] -name = "restate-node-services" -version = "1.1.0" -dependencies = [ - "anyhow", - "bytes", - "prost", - "prost-types", - "restate-node-protocol", - "restate-types", - "thiserror", - "tonic 0.10.2", - "tonic-build", -] - [[package]] name = "restate-partition-store" version = "1.1.0" @@ -6000,44 +5965,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "restate-schema" -version = "1.1.0" -dependencies = [ - "arc-swap", - "bytes", - "derive_more", - "flexbuffers", - "http 0.2.12", - "restate-schema-api", - "restate-test-util", - "restate-types", - "serde", - "serde_with", - "strum_macros 0.26.2", -] - -[[package]] -name = "restate-schema-api" -version = "1.1.0" -dependencies = [ - "anyhow", - "base64 0.21.7", - "bytes", - "bytestring", - "http 0.2.12", - "humantime", - "itertools 0.11.0", - "restate-base64-util", - "restate-serde-util", - "restate-types", - "schemars", - "serde", - "serde_with", - "thiserror", - "tracing", -] - [[package]] name = "restate-serde-util" version = "1.1.0" @@ -6144,7 +6071,6 @@ dependencies = [ "regress 0.9.1", "restate-base64-util", "restate-errors", - "restate-schema-api", "restate-service-client", "restate-test-util", "restate-types", @@ -6201,7 +6127,6 @@ dependencies = [ "restate-invoker-api", "restate-partition-store", "restate-rocksdb", - "restate-schema-api", "restate-service-protocol", "restate-storage-api", "restate-types", @@ -6329,6 +6254,7 @@ dependencies = [ "hostname", "http 0.2.12", "humantime", + "itertools 0.11.0", "jsonptr", "num-traits", "once_cell", @@ -6336,6 +6262,7 @@ dependencies = [ "prettyplease", "prost", "prost-build", + "prost-dto", "prost-types", "rand", "regress 0.9.1", @@ -6427,11 +6354,8 @@ dependencies = [ "restate-invoker-impl", "restate-metadata-store", "restate-network", - "restate-node-protocol", "restate-partition-store", "restate-rocksdb", - "restate-schema", - "restate-schema-api", "restate-serde-util", "restate-service-client", "restate-service-protocol", @@ -6472,9 +6396,8 @@ dependencies = [ "crossterm 0.27.0", "ctrlc", "prost-types", + "restate-admin", "restate-cli-util", - "restate-node-protocol", - "restate-node-services", "restate-types", "thiserror", "tokio", @@ -8513,8 +8436,7 @@ dependencies = [ "restate-bifrost", "restate-core", "restate-metadata-store", - "restate-node-services", - "restate-schema-api", + "restate-network", "restate-service-client", "restate-service-protocol", "restate-storage-query-datafusion", diff --git a/Cargo.toml b/Cargo.toml index c01d0ef4bf..7ff803c2f6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,7 +37,6 @@ restate-base64-util = { path = "crates/base64-util" } restate-benchmarks = { path = "crates/benchmarks" } restate-bifrost = { path = "crates/bifrost" } restate-cli-util = { path = "crates/cli-util" } -restate-cluster-controller = { path = "crates/cluster-controller" } restate-core = { path = "crates/core" } restate-errors = { path = "crates/errors" } restate-fs-util = { path = "crates/fs-util" } @@ -52,13 +51,9 @@ restate-admin-rest-model = { path = "crates/admin-rest-model" } restate-metadata-store = { path = "crates/metadata-store" } restate-network = { path = "crates/network" } restate-node = { path = "crates/node" } -restate-node-protocol = { path = "crates/node-protocol" } -restate-node-services = { path = "crates/node-services" } restate-partition-store = { path = "crates/partition-store" } restate-queue = { path = "crates/queue" } restate-rocksdb = { path = "crates/rocksdb" } -restate-schema = { path = "crates/schema" } -restate-schema-api = { path = "crates/schema-api" } restate-serde-util = { path = "crates/serde-util" } restate-server = { path = "server" } restate-service-client = { path = "crates/service-client" } @@ -127,6 +122,7 @@ paste = "1.0" pin-project = "1.0" prost = "0.12.1" prost-build = "0.12.1" +prost-dto = { version = "0.0.2" } prost-types = "0.12.1" rand = "0.8.5" rayon = { version = "1.10" } diff --git a/cli/src/clients/admin_interface.rs b/cli/src/clients/admin_interface.rs index 40b2717b51..260c0c2177 100644 --- a/cli/src/clients/admin_interface.rs +++ b/cli/src/clients/admin_interface.rs @@ -14,6 +14,7 @@ use super::AdminClient; use restate_admin_rest_model::deployments::*; use restate_admin_rest_model::services::*; +use restate_types::schema::service::ServiceMetadata; pub trait AdminClientInterface { /// Check if the admin service is healthy by invoking /health diff --git a/cli/src/clients/datafusion_helpers.rs b/cli/src/clients/datafusion_helpers.rs index 591dfc2387..0354123ac6 100644 --- a/cli/src/clients/datafusion_helpers.rs +++ b/cli/src/clients/datafusion_helpers.rs @@ -14,21 +14,21 @@ use std::collections::HashMap; use std::fmt::Display; use std::str::FromStr; -use super::DataFusionHttpClient; - +use anyhow::Result; use arrow::array::{Array, ArrayAccessor, AsArray, StringArray}; use arrow::datatypes::{ArrowTemporalType, Date64Type}; use arrow::record_batch::RecordBatch; -use clap::ValueEnum; -use restate_admin_rest_model::deployments::DeploymentId; - -use anyhow::Result; use arrow_convert::{ArrowDeserialize, ArrowField}; use bytes::Bytes; use chrono::{DateTime, Duration, Local, TimeZone}; -use restate_admin_rest_model::services::ServiceType; +use clap::ValueEnum; + use restate_service_protocol::awakeable_id::AwakeableIdentifier; +use restate_types::identifiers::DeploymentId; use restate_types::identifiers::{InvocationId, ServiceId}; +use restate_types::invocation::ServiceType; + +use super::DataFusionHttpClient; static JOURNAL_QUERY_LIMIT: usize = 100; @@ -433,7 +433,7 @@ pub async fn count_deployment_active_inv_by_method( let mut output = vec![]; let query = format!( - "SELECT + "SELECT target_service_name, target_handler_name, COUNT(id) AS inv_count @@ -472,7 +472,7 @@ pub async fn get_service_status( // Inbox analysis (pending invocations).... { let query = format!( - "SELECT + "SELECT target_service_name, target_handler_name, COUNT(id), @@ -614,7 +614,7 @@ pub async fn get_locked_keys_status( // Inbox analysis (pending invocations).... { let query = format!( - "SELECT + "SELECT service_name, service_key, COUNT(id), diff --git a/cli/src/commands/deployments/describe.rs b/cli/src/commands/deployments/describe.rs index 11b0810e42..44abfa096e 100644 --- a/cli/src/commands/deployments/describe.rs +++ b/cli/src/commands/deployments/describe.rs @@ -13,9 +13,13 @@ use std::collections::HashMap; use anyhow::Result; use cling::prelude::*; use comfy_table::{Cell, Table}; + use restate_admin_rest_model::deployments::ServiceNameRevPair; -use restate_admin_rest_model::services::ServiceMetadata; use restate_cli_util::ui::console::{Styled, StyledTable}; +use restate_cli_util::ui::stylesheet::Style; +use restate_cli_util::ui::watcher::Watch; +use restate_cli_util::{c_eprintln, c_indent_table, c_indentln, c_println, c_title}; +use restate_types::schema::service::ServiceMetadata; use crate::cli_env::CliEnv; use crate::clients::datafusion_helpers::count_deployment_active_inv_by_method; @@ -26,10 +30,6 @@ use crate::ui::deployments::{ }; use crate::ui::service_handlers::icon_for_service_type; -use restate_cli_util::ui::stylesheet::Style; -use restate_cli_util::ui::watcher::Watch; -use restate_cli_util::{c_eprintln, c_indent_table, c_indentln, c_println, c_title}; - #[derive(Run, Parser, Collect, Clone)] #[cling(run = "run_describe")] #[clap(visible_alias = "get")] diff --git a/cli/src/commands/deployments/list.rs b/cli/src/commands/deployments/list.rs index 7f01f1d6da..63e68411f0 100644 --- a/cli/src/commands/deployments/list.rs +++ b/cli/src/commands/deployments/list.rs @@ -15,12 +15,12 @@ use cling::prelude::*; use comfy_table::{Cell, Table}; use restate_admin_rest_model::deployments::{Deployment, DeploymentResponse, ServiceNameRevPair}; -use restate_admin_rest_model::services::ServiceMetadata; use restate_cli_util::c_error; use restate_cli_util::ui::console::{Styled, StyledTable}; use restate_cli_util::ui::stylesheet::Style; use restate_cli_util::ui::watcher::Watch; use restate_types::identifiers::DeploymentId; +use restate_types::schema::service::ServiceMetadata; use crate::cli_env::CliEnv; use crate::clients::datafusion_helpers::count_deployment_active_inv; diff --git a/cli/src/commands/deployments/register.rs b/cli/src/commands/deployments/register.rs index a20ba4a5a2..d676a9709d 100644 --- a/cli/src/commands/deployments/register.rs +++ b/cli/src/commands/deployments/register.rs @@ -18,11 +18,12 @@ use comfy_table::Table; use http::{HeaderName, HeaderValue, StatusCode, Uri}; use indicatif::ProgressBar; -use restate_admin_rest_model::deployments::{Deployment, LambdaARN, RegisterDeploymentRequest}; -use restate_admin_rest_model::services::ServiceMetadata; +use restate_admin_rest_model::deployments::{Deployment, RegisterDeploymentRequest}; use restate_cli_util::ui::console::{confirm_or_exit, Styled, StyledTable}; use restate_cli_util::ui::stylesheet::Style; use restate_cli_util::{c_eprintln, c_error, c_indent_table, c_indentln, c_success, c_warn}; +use restate_types::identifiers::LambdaARN; +use restate_types::schema::service::ServiceMetadata; use crate::cli_env::CliEnv; use crate::clients::{AdminClient, AdminClientInterface, MetasClientError}; diff --git a/cli/src/commands/deployments/remove.rs b/cli/src/commands/deployments/remove.rs index 8ca8978d7a..d9682b38df 100644 --- a/cli/src/commands/deployments/remove.rs +++ b/cli/src/commands/deployments/remove.rs @@ -16,10 +16,10 @@ use comfy_table::Table; use indoc::indoc; use restate_admin_rest_model::deployments::ServiceNameRevPair; -use restate_admin_rest_model::services::ServiceMetadata; use restate_cli_util::ui::console::{confirm_or_exit, Styled, StyledTable}; use restate_cli_util::ui::stylesheet::Style; use restate_cli_util::{c_eprintln, c_error, c_indentln, c_success}; +use restate_types::schema::service::ServiceMetadata; use crate::cli_env::CliEnv; use crate::clients::datafusion_helpers::count_deployment_active_inv_by_method; diff --git a/cli/src/commands/services/list.rs b/cli/src/commands/services/list.rs index 0bc15b2c58..e67bb206d7 100644 --- a/cli/src/commands/services/list.rs +++ b/cli/src/commands/services/list.rs @@ -15,11 +15,11 @@ use cling::prelude::*; use comfy_table::Table; use restate_admin_rest_model::deployments::DeploymentResponse; -use restate_admin_rest_model::services::HandlerMetadata; use restate_cli_util::ui::console::StyledTable; use restate_cli_util::ui::watcher::Watch; use restate_cli_util::{c_error, c_println}; use restate_types::identifiers::DeploymentId; +use restate_types::schema::service::HandlerMetadata; use crate::cli_env::CliEnv; use crate::clients::AdminClientInterface; diff --git a/cli/src/commands/services/status/agg_status.rs b/cli/src/commands/services/status/agg_status.rs index 8ff5320312..e89e60b611 100644 --- a/cli/src/commands/services/status/agg_status.rs +++ b/cli/src/commands/services/status/agg_status.rs @@ -11,8 +11,8 @@ use anyhow::Result; use indicatif::ProgressBar; -use restate_admin_rest_model::services::ServiceType; use restate_cli_util::{c_error, c_title}; +use restate_types::invocation::ServiceType; use super::{render_locked_keys, render_services_status, Status}; use crate::clients::datafusion_helpers::{get_locked_keys_status, get_service_status}; diff --git a/cli/src/commands/services/status/detailed_status.rs b/cli/src/commands/services/status/detailed_status.rs index b2541cfefd..15304fe84d 100644 --- a/cli/src/commands/services/status/detailed_status.rs +++ b/cli/src/commands/services/status/detailed_status.rs @@ -11,8 +11,8 @@ use anyhow::Result; use indicatif::ProgressBar; -use restate_admin_rest_model::services::ServiceType; use restate_cli_util::c_title; +use restate_types::invocation::ServiceType; use super::{render_locked_keys, render_services_status, Status}; use crate::clients::datafusion_helpers::{ diff --git a/cli/src/commands/services/status/mod.rs b/cli/src/commands/services/status/mod.rs index ed7dc08250..18c3fc69c0 100644 --- a/cli/src/commands/services/status/mod.rs +++ b/cli/src/commands/services/status/mod.rs @@ -11,24 +11,25 @@ mod agg_status; mod detailed_status; -use crate::cli_env::CliEnv; -use crate::clients::datafusion_helpers::{ - InvocationState, ServiceHandlerLockedKeysMap, ServiceStatus, ServiceStatusMap, -}; -use crate::clients::AdminClient; -use crate::ui::invocations::invocation_status; -use crate::ui::service_handlers::icon_for_service_type; - use anyhow::Result; use chrono_humanize::Tense; use cling::prelude::*; use comfy_table::{Cell, Table}; -use restate_admin_rest_model::services::ServiceMetadata; + use restate_cli_util::c_println; use restate_cli_util::ui::console::{Styled, StyledTable}; use restate_cli_util::ui::stylesheet::Style; use restate_cli_util::ui::watcher::Watch; use restate_cli_util::ui::{duration_to_human_precise, duration_to_human_rough}; +use restate_types::schema::service::ServiceMetadata; + +use crate::cli_env::CliEnv; +use crate::clients::datafusion_helpers::{ + InvocationState, ServiceHandlerLockedKeysMap, ServiceStatus, ServiceStatusMap, +}; +use crate::clients::AdminClient; +use crate::ui::invocations::invocation_status; +use crate::ui::service_handlers::icon_for_service_type; #[derive(Run, Parser, Collect, Clone)] #[cling(run = "run_status")] diff --git a/cli/src/commands/state/util.rs b/cli/src/commands/state/util.rs index 5f9fea326b..bf3039d0f0 100644 --- a/cli/src/commands/state/util.rs +++ b/cli/src/commands/state/util.rs @@ -22,8 +22,9 @@ use comfy_table::{Cell, Table}; use itertools::Itertools; use serde_json::Value; -use restate_admin_rest_model::services::{ModifyServiceStateRequest, ServiceType}; +use restate_admin_rest_model::services::ModifyServiceStateRequest; use restate_cli_util::ui::console::StyledTable; +use restate_types::invocation::ServiceType; use restate_types::state_mut::StateMutationVersion; use crate::cli_env::CliEnv; diff --git a/cli/src/ui/deployments.rs b/cli/src/ui/deployments.rs index 56b8910a4b..4f4ddefbc5 100644 --- a/cli/src/ui/deployments.rs +++ b/cli/src/ui/deployments.rs @@ -12,10 +12,11 @@ use std::collections::HashMap; use comfy_table::{Cell, Color, Table}; -use restate_admin_rest_model::deployments::{Deployment, ProtocolType, ServiceNameRevPair}; -use restate_admin_rest_model::services::ServiceMetadata; +use restate_admin_rest_model::deployments::{Deployment, ServiceNameRevPair}; use restate_cli_util::ui::console::StyledTable; use restate_types::identifiers::DeploymentId; +use restate_types::schema::deployment::ProtocolType; +use restate_types::schema::service::ServiceMetadata; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum DeploymentStatus { diff --git a/cli/src/ui/service_handlers.rs b/cli/src/ui/service_handlers.rs index b727fd25bd..c5de1d2eab 100644 --- a/cli/src/ui/service_handlers.rs +++ b/cli/src/ui/service_handlers.rs @@ -9,9 +9,10 @@ // by the Apache License, Version 2.0. use comfy_table::{Cell, Color, Table}; -use restate_admin_rest_model::handlers::HandlerMetadata; -use restate_admin_rest_model::services::ServiceType; + use restate_cli_util::ui::console::{Icon, StyledTable}; +use restate_types::invocation::ServiceType; +use restate_types::schema::service::HandlerMetadata; pub fn create_service_handlers_table(handlers: &[HandlerMetadata]) -> Table { let mut table = Table::new_styled(); diff --git a/crates/admin-rest-model/Cargo.toml b/crates/admin-rest-model/Cargo.toml index 38329e17db..b3436c9133 100644 --- a/crates/admin-rest-model/Cargo.toml +++ b/crates/admin-rest-model/Cargo.toml @@ -9,11 +9,10 @@ publish = false [features] default = [] -schema = ["dep:schemars", "restate-schema-api/serde_schema", "restate-serde-util/schema", "restate-types/schemars"] +schema = ["dep:schemars", "restate-serde-util/schema", "restate-types/schemars"] [dependencies] restate-types = { workspace = true } -restate-schema-api = { workspace = true, features = [ "deployment", "serde", "subscription", "service" ] } restate-serde-util = { workspace = true } bytes = { workspace = true } diff --git a/crates/admin-rest-model/src/deployments.rs b/crates/admin-rest-model/src/deployments.rs index 9e6030bdaf..0d1d35764a 100644 --- a/crates/admin-rest-model/src/deployments.rs +++ b/crates/admin-rest-model/src/deployments.rs @@ -9,18 +9,16 @@ // by the Apache License, Version 2.0. use http::Uri; -use restate_schema_api::service::ServiceMetadata; -use restate_serde_util::SerdeableHeaderHashMap; use serde::{Deserialize, Serialize}; use serde_with::serde_as; use std::time::SystemTime; -// Export schema types to be used by other crates without exposing the fact -// that we are using proxying to restate-schema-api or restate-types -use restate_schema_api::deployment::DeploymentType; -pub use restate_schema_api::deployment::{DeploymentMetadata, ProtocolType}; +use restate_serde_util::SerdeableHeaderHashMap; use restate_types::identifiers::ServiceRevision; -pub use restate_types::identifiers::{DeploymentId, LambdaARN}; +use restate_types::identifiers::{DeploymentId, LambdaARN}; +use restate_types::schema::deployment::DeploymentType; +use restate_types::schema::deployment::{DeploymentMetadata, ProtocolType}; +use restate_types::schema::service::ServiceMetadata; #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] #[derive(Clone, Debug, Serialize, Deserialize)] diff --git a/crates/admin-rest-model/src/handlers.rs b/crates/admin-rest-model/src/handlers.rs index 3ce5ce83b9..7659d26d5d 100644 --- a/crates/admin-rest-model/src/handlers.rs +++ b/crates/admin-rest-model/src/handlers.rs @@ -9,9 +9,8 @@ // by the Apache License, Version 2.0. use serde::{Deserialize, Serialize}; -// Export schema types to be used by other crates without exposing the fact -// that we are using proxying to restate-schema-api or restate-types -pub use restate_schema_api::service::HandlerMetadata; + +use restate_types::schema::service::HandlerMetadata; #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] #[derive(Debug, Serialize, Deserialize)] diff --git a/crates/admin-rest-model/src/services.rs b/crates/admin-rest-model/src/services.rs index 4997e157b4..08c4ce385e 100644 --- a/crates/admin-rest-model/src/services.rs +++ b/crates/admin-rest-model/src/services.rs @@ -13,11 +13,7 @@ use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::time::Duration; -// Export schema types to be used by other crates without exposing the fact -// that we are using proxying to restate-schema-api or restate-types -pub use restate_schema_api::service::{HandlerMetadata, ServiceMetadata}; -pub use restate_types::identifiers::ServiceRevision; -pub use restate_types::invocation::ServiceType; +use restate_types::schema::service::ServiceMetadata; #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] #[derive(Debug, Serialize, Deserialize)] diff --git a/crates/admin-rest-model/src/subscriptions.rs b/crates/admin-rest-model/src/subscriptions.rs index 2b50dd8de3..0067dfed65 100644 --- a/crates/admin-rest-model/src/subscriptions.rs +++ b/crates/admin-rest-model/src/subscriptions.rs @@ -11,13 +11,11 @@ use std::collections::HashMap; use http::Uri; -use restate_types::identifiers::SubscriptionId; use serde::{Deserialize, Serialize}; use serde_with::serde_as; -// Export schema types to be used by other crates without exposing the fact -// that we are using proxying to restate-schema-api or restate-types -pub use restate_schema_api::subscription::{ListSubscriptionFilter, Subscription}; +use restate_types::identifiers::SubscriptionId; +use restate_types::schema::subscriptions::Subscription; #[serde_as] #[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] diff --git a/crates/admin/Cargo.toml b/crates/admin/Cargo.toml index fa3e97b4a4..b4cef2cec0 100644 --- a/crates/admin/Cargo.toml +++ b/crates/admin/Cargo.toml @@ -9,6 +9,8 @@ publish = false [features] default = [] +clients = [] +servers = [] options_schema = ["restate-service-client/options_schema"] [dependencies] @@ -18,15 +20,14 @@ restate-core = { workspace = true } restate-errors = { workspace = true } restate-fs-util = { workspace = true } restate-futures-util = { workspace = true } -restate-node-services = { workspace = true, features = ["servers", "clients"] } -restate-schema = { workspace = true } -restate-schema-api = { workspace = true, features = ["deployment", "serde", "serde_schema"] } +restate-network = { workspace = true } restate-service-client = { workspace = true } restate-service-protocol = { workspace = true, features = ["discovery"] } restate-types = { workspace = true, features = ["schemars"] } restate-wal-protocol = { workspace = true } anyhow = { workspace = true } +arc-swap = { workspace = true } arrow-flight = { workspace = true } axum = { workspace = true } bytes = { workspace = true } @@ -39,22 +40,30 @@ futures = { workspace = true } http = { workspace = true } hyper = { workspace = true, features = ["full"] } okapi-operation = { version = "0.2.2", features = ["axum-integration"] } +prost = { workspace = true } +prost-dto = { workspace = true } +prost-types = { workspace = true } restate-serde-util = { workspace = true, features = ["schema"] } schemars = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } serde_with = { workspace = true } thiserror = { workspace = true } -tokio = { workspace = true, features = ["full"] } -tonic = { workspace = true } +tokio = { workspace = true } +tonic = { workspace = true, features = ["transport", "codegen", "prost", "gzip"] } tower = { workspace = true, features = ["load-shed", "limit"] } tracing = { workspace = true } +[build-dependencies] +tonic-build = { workspace = true } + [dev-dependencies] -restate-schema = { workspace = true, features = ["test-util"] } -restate-schema-api = { workspace = true, features = ["test-util"] } +restate-bifrost = { workspace = true, features = ["test-util"] } +restate-core = { workspace = true, features = ["test-util"] } restate-test-util = { workspace = true } +restate-types = { workspace = true, features = ["test-util"] } +googletest = { workspace = true } tempfile = { workspace = true } test-log = { workspace = true } tracing = { workspace = true } diff --git a/crates/node-services/build.rs b/crates/admin/build.rs similarity index 50% rename from crates/node-services/build.rs rename to crates/admin/build.rs index 3cad0433f6..16702231ef 100644 --- a/crates/node-services/build.rs +++ b/crates/admin/build.rs @@ -1,4 +1,4 @@ -// Copyright (c) 2023 - Restate Software, Inc., Restate GmbH. +// Copyright (c) 2024 - Restate Software, Inc., Restate GmbH. // All rights reserved. // // Use of this software is governed by the Business Source License @@ -21,24 +21,11 @@ fn main() -> Result<(), Box> { .client_mod_attribute("cluster_ctrl", "#[cfg(feature = \"clients\")]") // allow older protobuf compiler to be used .protoc_arg("--experimental_allow_proto3_optional") - .extern_path(".dev.restate.common", "::restate_node_protocol::common") + .extern_path(".restate.common", "::restate_types::protobuf::common") + .extern_path(".restate.cluster", "::restate_types::protobuf::cluster") .compile( - &["./proto/cluster_ctrl_svc.proto"], - &["proto", "../node-protocol/proto"], - )?; - - tonic_build::configure() - .bytes(["."]) - .file_descriptor_set_path(out_dir.join("node_svc_descriptor.bin")) - .server_mod_attribute("node_svc", "#[cfg(feature = \"servers\")]") - .client_mod_attribute("node_svc", "#[cfg(feature = \"clients\")]") - // allow older protobuf compiler to be used - .protoc_arg("--experimental_allow_proto3_optional") - .extern_path(".dev.restate.node", "::restate_node_protocol::node") - .extern_path(".dev.restate.common", "::restate_node_protocol::common") - .compile( - &["./proto/node_svc.proto"], - &["proto", "../node-protocol/proto"], + &["./protobuf/cluster_ctrl_svc.proto"], + &["protobuf", "../types/protobuf"], )?; Ok(()) diff --git a/crates/admin/protobuf/cluster_ctrl_svc.proto b/crates/admin/protobuf/cluster_ctrl_svc.proto new file mode 100644 index 0000000000..b343c61f79 --- /dev/null +++ b/crates/admin/protobuf/cluster_ctrl_svc.proto @@ -0,0 +1,33 @@ +// Copyright (c) 2024 - Restate Software, Inc., Restate GmbH +// +// This file is part of the Restate service protocol, which is +// released under the MIT license. +// +// You can find a copy of the license in file LICENSE in the root +// directory of this repository or package, or at +// https://github.com/restatedev/proto/blob/main/LICENSE + +syntax = "proto3"; + +import "restate/common.proto"; +import "restate/cluster.proto"; +import "google/protobuf/empty.proto"; + +package restate.cluster_ctrl; + +service ClusterCtrlSvc { + rpc GetClusterState(ClusterStateRequest) returns (ClusterStateResponse); + + rpc TrimLog(TrimLogRequest) returns (google.protobuf.Empty); +} + +message ClusterStateRequest {} + +message ClusterStateResponse { + restate.cluster.ClusterState cluster_state = 1; +} + +message TrimLogRequest { + uint64 log_id = 1; + uint64 trim_point = 2; +} diff --git a/crates/cluster-controller/src/cluster_state.rs b/crates/admin/src/cluster_controller/cluster_state.rs similarity index 81% rename from crates/cluster-controller/src/cluster_state.rs rename to crates/admin/src/cluster_controller/cluster_state.rs index 3a1058d507..969f45adc6 100644 --- a/crates/cluster-controller/src/cluster_state.rs +++ b/crates/admin/src/cluster_controller/cluster_state.rs @@ -12,50 +12,18 @@ use std::collections::BTreeMap; use std::sync::Arc; use arc_swap::ArcSwap; +use restate_types::cluster::cluster_state::{AliveNode, ClusterState, DeadNode, NodeState}; +use std::time::Instant; +use tokio::task::JoinHandle; + use restate_core::network::{MessageRouterBuilder, NetworkSender}; use restate_network::rpc_router::RpcRouter; -use restate_node_protocol::partition_processor_manager::GetProcessorsState; -use restate_types::identifiers::PartitionId; +use restate_types::net::partition_processor_manager::GetProcessorsState; use restate_types::nodes_config::Role; -use restate_types::processors::PartitionProcessorStatus; use restate_types::time::MillisSinceEpoch; -use tokio::task::JoinHandle; -use tokio::time::Instant; use restate_core::{Metadata, ShutdownError, TaskCenter}; -use restate_types::{GenerationalNodeId, PlainNodeId, Version}; - -/// A container for health information about every node and partition in the -/// cluster. -#[derive(Debug, Clone)] -pub struct ClusterState { - pub last_refreshed: Option, - pub nodes_config_version: Version, - pub partition_table_version: Version, - pub nodes: BTreeMap, -} - -impl ClusterState { - pub fn is_reliable(&self) -> bool { - // todo: make this configurable - // If the cluster state is older than 10 seconds, then it is not reliable. - self.last_refreshed - .map(|last_refreshed| last_refreshed.elapsed().as_secs() < 10) - .unwrap_or(false) - } -} - -#[derive(Debug, Clone)] -pub enum NodeState { - Alive { - last_heartbeat_at: MillisSinceEpoch, - generation: GenerationalNodeId, - partitions: BTreeMap, - }, - Dead { - last_seen_alive: Option, - }, -} +use restate_types::Version; pub struct ClusterStateRefresher { task_center: TaskCenter, @@ -184,13 +152,13 @@ where .nodes .get(&node_id) .and_then(|state| match state { - NodeState::Alive { + NodeState::Alive(AliveNode { last_heartbeat_at, .. - } => Some(*last_heartbeat_at), - NodeState::Dead { last_seen_alive } => *last_seen_alive, + }) => Some(*last_heartbeat_at), + NodeState::Dead(DeadNode { last_seen_alive }) => *last_seen_alive, }); - nodes.insert(node_id, NodeState::Dead { last_seen_alive }); + nodes.insert(node_id, NodeState::Dead(DeadNode { last_seen_alive })); continue; }; @@ -198,11 +166,11 @@ where nodes.insert( node_id, - NodeState::Alive { + NodeState::Alive(AliveNode { last_heartbeat_at: MillisSinceEpoch::now(), - generation: from, + generational_node_id: from, partitions: msg.state, - }, + }), ); } diff --git a/crates/cluster-controller/src/lib.rs b/crates/admin/src/cluster_controller/mod.rs similarity index 86% rename from crates/cluster-controller/src/lib.rs rename to crates/admin/src/cluster_controller/mod.rs index 57c4805805..f0af35f406 100644 --- a/crates/cluster-controller/src/lib.rs +++ b/crates/admin/src/cluster_controller/mod.rs @@ -8,8 +8,8 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -mod cluster_state; -mod service; +pub mod cluster_state; +pub mod protobuf; +pub mod service; -pub use cluster_state::NodeState; pub use service::{ClusterControllerHandle, Error, Service}; diff --git a/crates/admin/src/cluster_controller/protobuf.rs b/crates/admin/src/cluster_controller/protobuf.rs new file mode 100644 index 0000000000..8d5274e782 --- /dev/null +++ b/crates/admin/src/cluster_controller/protobuf.rs @@ -0,0 +1,14 @@ +// Copyright (c) 2024 - Restate Software, Inc., Restate GmbH. +// All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +tonic::include_proto!("restate.cluster_ctrl"); + +pub const FILE_DESCRIPTOR_SET: &[u8] = + tonic::include_file_descriptor_set!("cluster_ctrl_svc_descriptor"); diff --git a/crates/cluster-controller/src/service.rs b/crates/admin/src/cluster_controller/service.rs similarity index 95% rename from crates/cluster-controller/src/service.rs rename to crates/admin/src/cluster_controller/service.rs index 6d2df8f2ca..f8ac916639 100644 --- a/crates/cluster-controller/src/service.rs +++ b/crates/admin/src/cluster_controller/service.rs @@ -15,30 +15,29 @@ use std::sync::Arc; use codederror::CodedError; use futures::future::OptionFuture; use futures::{Stream, StreamExt}; +use tokio::sync::{mpsc, oneshot}; +use tokio::time; +use tokio::time::MissedTickBehavior; use tokio::time::{Instant, Interval}; +use tracing::{debug, warn}; -use restate_node_protocol::cluster_controller::{ - Action, AttachRequest, AttachResponse, RunPartition, -}; -use restate_node_protocol::common::{KeyRange, RequestId}; use restate_types::arc_util::Updateable; use restate_types::config::{AdminOptions, Configuration}; -use restate_types::partition_table::FixedPartitionTable; +use restate_types::net::cluster_controller::{Action, AttachRequest, AttachResponse, RunPartition}; +use restate_types::net::RequestId; +use restate_types::partition_table::{FixedPartitionTable, KeyRange}; use restate_bifrost::Bifrost; use restate_core::network::{MessageRouterBuilder, NetworkSender}; use restate_core::{cancellation_watcher, Metadata, ShutdownError, TaskCenter}; -use restate_node_protocol::MessageEnvelope; +use restate_types::cluster::cluster_state::RunMode; +use restate_types::cluster::cluster_state::{AliveNode, ClusterState, NodeState}; use restate_types::identifiers::PartitionId; use restate_types::logs::{LogId, Lsn, SequenceNumber}; -use restate_types::processors::RunMode; +use restate_types::net::MessageEnvelope; use restate_types::{GenerationalNodeId, Version}; -use tokio::sync::{mpsc, oneshot}; -use tokio::time; -use tokio::time::MissedTickBehavior; -use tracing::{debug, warn}; -use crate::cluster_state::{ClusterState, ClusterStateRefresher, NodeState}; +use super::cluster_state::ClusterStateRefresher; #[derive(Debug, thiserror::Error, CodedError)] pub enum Error { @@ -236,11 +235,11 @@ where for node_state in cluster_state.nodes.values() { match node_state { - NodeState::Alive { - generation, + NodeState::Alive(AliveNode { + generational_node_id, partitions, .. - } => { + }) => { for (partition_id, partition_processor_status) in partitions.iter() { let lsn = partition_processor_status .last_persisted_log_lsn @@ -248,10 +247,10 @@ where persisted_lsns_per_partition .entry(*partition_id) .or_default() - .insert(*generation, lsn); + .insert(*generational_node_id, lsn); } } - NodeState::Dead { .. } => { + NodeState::Dead(_) => { // nothing to do } } @@ -340,23 +339,22 @@ where #[cfg(test)] mod tests { - use crate::Service; + use super::Service; use googletest::matchers::eq; use googletest::{assert_that, pat}; use restate_bifrost::{Bifrost, Record, TrimGap}; use restate_core::network::{MessageHandler, NetworkSender}; use restate_core::{MockNetworkSender, TaskKind, TestCoreEnvBuilder}; - use restate_node_protocol::partition_processor_manager::{ - GetProcessorsState, ProcessorsStateResponse, - }; - use restate_node_protocol::MessageEnvelope; use restate_types::arc_util::Constant; + use restate_types::cluster::cluster_state::{PartitionProcessorStatus, RunMode}; use restate_types::config::AdminOptions; use restate_types::identifiers::PartitionId; use restate_types::logs::{LogId, Lsn, Payload, SequenceNumber}; - use restate_types::net::AdvertisedAddress; + use restate_types::net::partition_processor_manager::{ + GetProcessorsState, ProcessorsStateResponse, + }; + use restate_types::net::{AdvertisedAddress, MessageEnvelope}; use restate_types::nodes_config::{NodeConfig, NodesConfiguration, Role}; - use restate_types::processors::{PartitionProcessorStatus, RunMode}; use restate_types::{GenerationalNodeId, Version}; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; diff --git a/crates/admin/src/lib.rs b/crates/admin/src/lib.rs index ef837cd514..1ba4442ed4 100644 --- a/crates/admin/src/lib.rs +++ b/crates/admin/src/lib.rs @@ -8,6 +8,7 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. +pub mod cluster_controller; mod error; mod rest_api; mod schema_registry; diff --git a/crates/admin/src/rest_api/deployments.rs b/crates/admin/src/rest_api/deployments.rs index 2e45a09033..28c64cb223 100644 --- a/crates/admin/src/rest_api/deployments.rs +++ b/crates/admin/src/rest_api/deployments.rs @@ -21,7 +21,7 @@ use okapi_operation::*; use restate_admin_rest_model::deployments::*; use restate_service_client::Endpoint; use restate_service_protocol::discovery::DiscoverEndpoint; -use restate_types::identifiers::InvalidLambdaARN; +use restate_types::identifiers::{DeploymentId, InvalidLambdaARN}; use serde::Deserialize; /// Create deployment and return discovered services. diff --git a/crates/admin/src/rest_api/handlers.rs b/crates/admin/src/rest_api/handlers.rs index 7d224093f0..ce91dfec4c 100644 --- a/crates/admin/src/rest_api/handlers.rs +++ b/crates/admin/src/rest_api/handlers.rs @@ -15,6 +15,7 @@ use axum::extract::{Path, State}; use axum::Json; use okapi_operation::*; use restate_admin_rest_model::handlers::*; +use restate_types::schema::service::HandlerMetadata; /// List discovered handlers for service #[openapi( diff --git a/crates/admin/src/rest_api/mod.rs b/crates/admin/src/rest_api/mod.rs index ac38a0036d..9159700850 100644 --- a/crates/admin/src/rest_api/mod.rs +++ b/crates/admin/src/rest_api/mod.rs @@ -23,8 +23,8 @@ use codederror::CodedError; use okapi_operation::axum_integration::{delete, get, patch, post}; use okapi_operation::*; use restate_errors::warn_it; -use restate_schema_api::subscription::SubscriptionValidator; use restate_types::identifiers::PartitionKey; +use restate_types::schema::subscriptions::SubscriptionValidator; use restate_wal_protocol::{Destination, Header, Source}; use crate::state::AdminServiceState; diff --git a/crates/admin/src/rest_api/services.rs b/crates/admin/src/rest_api/services.rs index 76a5651d79..d5f888f554 100644 --- a/crates/admin/src/rest_api/services.rs +++ b/crates/admin/src/rest_api/services.rs @@ -21,6 +21,7 @@ use okapi_operation::*; use restate_admin_rest_model::services::ListServicesResponse; use restate_admin_rest_model::services::*; use restate_types::identifiers::{ServiceId, WithPartitionKey}; +use restate_types::schema::service::ServiceMetadata; use restate_types::state_mut::ExternalStateMutation; use restate_wal_protocol::{append_envelope_to_bifrost, Command, Envelope}; use tracing::warn; diff --git a/crates/admin/src/rest_api/subscriptions.rs b/crates/admin/src/rest_api/subscriptions.rs index b1ac61c718..889bd0fc79 100644 --- a/crates/admin/src/rest_api/subscriptions.rs +++ b/crates/admin/src/rest_api/subscriptions.rs @@ -12,7 +12,7 @@ use super::error::*; use crate::state::AdminServiceState; use restate_admin_rest_model::subscriptions::*; -use restate_schema_api::subscription::SubscriptionValidator; +use restate_types::schema::subscriptions::{ListSubscriptionFilter, SubscriptionValidator}; use crate::rest_api::log_error; use axum::extract::Query; diff --git a/crates/admin/src/schema_registry/error.rs b/crates/admin/src/schema_registry/error.rs index 713b29a674..626d8dcd11 100644 --- a/crates/admin/src/schema_registry/error.rs +++ b/crates/admin/src/schema_registry/error.rs @@ -8,16 +8,18 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -use crate::schema_registry::ServiceName; use http::header::InvalidHeaderValue; use http::Uri; + use restate_core::metadata_store::ReadModifyWriteError; use restate_core::ShutdownError; -use restate_schema_api::invocation_target::BadInputContentType; use restate_types::endpoint_manifest; use restate_types::errors::GenericError; use restate_types::identifiers::DeploymentId; use restate_types::invocation::ServiceType; +use restate_types::schema::invocation_target::BadInputContentType; + +use crate::schema_registry::ServiceName; #[derive(Debug, thiserror::Error, codederror::CodedError)] pub enum SchemaRegistryError { diff --git a/crates/admin/src/schema_registry/mod.rs b/crates/admin/src/schema_registry/mod.rs index cdfe84f224..e5a8fb2297 100644 --- a/crates/admin/src/schema_registry/mod.rs +++ b/crates/admin/src/schema_registry/mod.rs @@ -16,18 +16,18 @@ use crate::schema_registry::updater::SchemaUpdater; use http::Uri; use restate_core::metadata_store::MetadataStoreClient; use restate_core::{metadata, MetadataWriter}; -use restate_schema::Schema; -use restate_schema_api::deployment::{ - DeliveryOptions, Deployment, DeploymentMetadata, DeploymentResolver, -}; -use restate_schema_api::service::{HandlerMetadata, ServiceMetadata, ServiceMetadataResolver}; -use restate_schema_api::subscription::{ - ListSubscriptionFilter, Subscription, SubscriptionResolver, SubscriptionValidator, -}; use restate_service_client::Endpoint; use restate_service_protocol::discovery::{DiscoverEndpoint, ServiceDiscovery}; use restate_types::identifiers::{DeploymentId, ServiceRevision, SubscriptionId}; use restate_types::metadata_store::keys::SCHEMA_INFORMATION_KEY; +use restate_types::schema::deployment::{ + DeliveryOptions, Deployment, DeploymentMetadata, DeploymentResolver, +}; +use restate_types::schema::service::{HandlerMetadata, ServiceMetadata, ServiceMetadataResolver}; +use restate_types::schema::subscriptions::{ + ListSubscriptionFilter, Subscription, SubscriptionResolver, SubscriptionValidator, +}; +use restate_types::schema::Schema; use std::borrow::Borrow; use std::collections::HashMap; use std::ops::Deref; diff --git a/crates/admin/src/schema_registry/updater.rs b/crates/admin/src/schema_registry/updater.rs index a73f9df5a0..a814fc659c 100644 --- a/crates/admin/src/schema_registry/updater.rs +++ b/crates/admin/src/schema_registry/updater.rs @@ -13,22 +13,22 @@ use crate::schema_registry::error::{ }; use crate::schema_registry::{ModifyServiceChange, ServiceName}; use http::{HeaderValue, Uri}; -use restate_schema::deployment::DeploymentSchemas; -use restate_schema::service::{HandlerSchemas, ServiceLocation, ServiceSchemas}; -use restate_schema::Schema; -use restate_schema_api::deployment::DeploymentMetadata; -use restate_schema_api::invocation_target::{ - InputRules, InputValidationRule, InvocationTargetMetadata, OutputContentTypeRule, OutputRules, - DEFAULT_IDEMPOTENCY_RETENTION, DEFAULT_WORKFLOW_COMPLETION_RETENTION, -}; -use restate_schema_api::subscription::{ - EventReceiverServiceType, Sink, Source, Subscription, SubscriptionValidator, -}; use restate_types::endpoint_manifest; use restate_types::identifiers::{DeploymentId, SubscriptionId}; use restate_types::invocation::{ InvocationTargetType, ServiceType, VirtualObjectHandlerType, WorkflowHandlerType, }; +use restate_types::schema::deployment::DeploymentMetadata; +use restate_types::schema::deployment::DeploymentSchemas; +use restate_types::schema::invocation_target::{ + InputRules, InputValidationRule, InvocationTargetMetadata, OutputContentTypeRule, OutputRules, + DEFAULT_IDEMPOTENCY_RETENTION, DEFAULT_WORKFLOW_COMPLETION_RETENTION, +}; +use restate_types::schema::service::{HandlerSchemas, ServiceLocation, ServiceSchemas}; +use restate_types::schema::subscriptions::{ + EventReceiverServiceType, Sink, Source, Subscription, SubscriptionValidator, +}; +use restate_types::schema::Schema; use serde::{Deserialize, Serialize}; use std::collections::hash_map::Entry; use std::collections::HashMap; @@ -580,9 +580,9 @@ impl DiscoveredHandlerMetadata { mod tests { use super::*; - use restate_schema_api::deployment::{Deployment, DeploymentResolver}; - use restate_schema_api::service::ServiceMetadataResolver; use restate_test_util::{assert, assert_eq, let_assert}; + use restate_types::schema::deployment::{Deployment, DeploymentResolver}; + use restate_types::schema::service::ServiceMetadataResolver; use restate_types::Versioned; use test_log::test; diff --git a/crates/admin/src/service.rs b/crates/admin/src/service.rs index e23175b8ee..ca34d3f6c9 100644 --- a/crates/admin/src/service.rs +++ b/crates/admin/src/service.rs @@ -21,9 +21,9 @@ use tracing::info; use restate_core::metadata_store::MetadataStoreClient; use restate_core::{cancellation_watcher, task_center, MetadataWriter}; -use restate_node_services::node_svc::node_svc_client::NodeSvcClient; -use restate_schema_api::subscription::SubscriptionValidator; +use restate_network::protobuf::node_svc::node_svc_client::NodeSvcClient; use restate_service_protocol::discovery::ServiceDiscovery; +use restate_types::schema::subscriptions::SubscriptionValidator; use crate::schema_registry::SchemaRegistry; use crate::Error; diff --git a/crates/admin/src/state.rs b/crates/admin/src/state.rs index 4007f97771..9bdb2fc16f 100644 --- a/crates/admin/src/state.rs +++ b/crates/admin/src/state.rs @@ -12,7 +12,7 @@ use crate::schema_registry::SchemaRegistry; use restate_bifrost::Bifrost; use restate_core::TaskCenter; -use restate_node_services::node_svc::node_svc_client::NodeSvcClient; +use restate_network::protobuf::node_svc::node_svc_client::NodeSvcClient; use tonic::transport::Channel; #[derive(Clone, derive_builder::Builder)] diff --git a/crates/admin/src/storage_query/query.rs b/crates/admin/src/storage_query/query.rs index 0bbaafda29..3a9e7a84fb 100644 --- a/crates/admin/src/storage_query/query.rs +++ b/crates/admin/src/storage_query/query.rs @@ -30,7 +30,7 @@ use datafusion::arrow::ipc::writer::StreamWriter; use datafusion::arrow::record_batch::RecordBatch; use futures::{ready, Stream, StreamExt, TryStreamExt}; use okapi_operation::*; -use restate_node_services::node_svc::StorageQueryRequest; +use restate_network::protobuf::node_svc::StorageQueryRequest; use schemars::JsonSchema; use serde::Deserialize; use serde_with::serde_as; diff --git a/crates/cluster-controller/Cargo.toml b/crates/cluster-controller/Cargo.toml deleted file mode 100644 index d1af3b5ef2..0000000000 --- a/crates/cluster-controller/Cargo.toml +++ /dev/null @@ -1,37 +0,0 @@ -[package] -name = "restate-cluster-controller" -version.workspace = true -authors.workspace = true -edition.workspace = true -rust-version.workspace = true -license.workspace = true -publish = false - -[features] -default = [] -options_schema = ["dep:schemars"] - -[dependencies] -restate-bifrost = { workspace = true } -restate-core = { workspace = true } -restate-network = { workspace = true } -restate-node-protocol = { workspace = true } -restate-types = { workspace = true } - -anyhow = { workspace = true } -arc-swap = { workspace = true } -codederror = { workspace = true } -derive_builder = { workspace = true } -futures = { workspace = true } -schemars = { workspace = true, optional = true} -serde = { workspace = true } -thiserror = { workspace = true } -tokio = { workspace = true } -tracing = { workspace = true } - -[dev-dependencies] -restate-bifrost = { workspace = true, features = ["test-util"] } -restate-core = { workspace = true, features = ["test-util"] } - -googletest = { workspace = true } -test-log = { workspace = true } diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index 9e1fac7a7c..c58ef21e69 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -13,8 +13,6 @@ test-util = ["dep:rand", "tokio/test-util"] options_schema = ["dep:schemars"] [dependencies] -restate-node-protocol = { workspace = true } -restate-schema = { workspace = true } restate-types = { workspace = true } anyhow = { workspace = true } diff --git a/crates/core/src/metadata/manager.rs b/crates/core/src/metadata/manager.rs index a029d98021..f371bb3d48 100644 --- a/crates/core/src/metadata/manager.rs +++ b/crates/core/src/metadata/manager.rs @@ -9,6 +9,7 @@ // by the Apache License, Version 2.0. use arc_swap::{ArcSwap, ArcSwapOption}; +use restate_types::schema::Schema; use std::ops::Deref; use std::sync::Arc; @@ -16,12 +17,12 @@ use tokio::sync::mpsc; use tokio::sync::oneshot; use tracing::{debug, info, trace, warn}; -use restate_node_protocol::metadata::{MetadataMessage, MetadataUpdate, Schema}; -use restate_node_protocol::MessageEnvelope; use restate_types::logs::metadata::Logs; use restate_types::metadata_store::keys::{ BIFROST_CONFIG_KEY, NODES_CONFIG_KEY, PARTITION_TABLE_KEY, SCHEMA_INFORMATION_KEY, }; +use restate_types::net::metadata::{MetadataMessage, MetadataUpdate}; +use restate_types::net::MessageEnvelope; use restate_types::nodes_config::NodesConfiguration; use restate_types::partition_table::FixedPartitionTable; use restate_types::GenerationalNodeId; diff --git a/crates/core/src/metadata/mod.rs b/crates/core/src/metadata/mod.rs index 70317f5aab..7603690afa 100644 --- a/crates/core/src/metadata/mod.rs +++ b/crates/core/src/metadata/mod.rs @@ -13,6 +13,7 @@ mod manager; pub use manager::MetadataManager; +use restate_types::schema::{Schema, UpdateableSchema}; use std::sync::{Arc, OnceLock}; @@ -20,9 +21,9 @@ use arc_swap::{ArcSwap, ArcSwapOption}; use enum_map::EnumMap; use tokio::sync::{oneshot, watch}; -pub use restate_node_protocol::metadata::MetadataKind; -use restate_node_protocol::metadata::{MetadataContainer, Schema, UpdateableSchema}; use restate_types::logs::metadata::Logs; +use restate_types::net::metadata::MetadataContainer; +pub use restate_types::net::metadata::MetadataKind; use restate_types::nodes_config::NodesConfiguration; use restate_types::partition_table::FixedPartitionTable; use restate_types::{GenerationalNodeId, Version, Versioned}; diff --git a/crates/core/src/network/error.rs b/crates/core/src/network/error.rs index ecb20660b4..4ac90a8516 100644 --- a/crates/core/src/network/error.rs +++ b/crates/core/src/network/error.rs @@ -8,7 +8,7 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -use restate_node_protocol::CodecError; +use restate_types::net::CodecError; use restate_types::nodes_config::NodesConfigError; use crate::ShutdownError; diff --git a/crates/core/src/network/message_router.rs b/crates/core/src/network/message_router.rs index a86ac34e8f..71e6a479d8 100644 --- a/crates/core/src/network/message_router.rs +++ b/crates/core/src/network/message_router.rs @@ -15,12 +15,12 @@ use std::sync::Arc; use async_trait::async_trait; use futures::Stream; -use restate_node_protocol::codec::{Targeted, WireDecode}; -use restate_node_protocol::common::ProtocolVersion; -use restate_node_protocol::common::TargetName; -use restate_node_protocol::node::message::BinaryMessage; -use restate_node_protocol::CodecError; -use restate_node_protocol::MessageEnvelope; +use restate_types::net::codec::{Targeted, WireDecode}; +use restate_types::net::CodecError; +use restate_types::net::MessageEnvelope; +use restate_types::net::ProtocolVersion; +use restate_types::net::TargetName; +use restate_types::protobuf::node::message::BinaryMessage; use restate_types::GenerationalNodeId; use tokio::sync::mpsc; use tokio_stream::wrappers::ReceiverStream; diff --git a/crates/core/src/network/network_sender.rs b/crates/core/src/network/network_sender.rs index ef436aff92..ca5be0094f 100644 --- a/crates/core/src/network/network_sender.rs +++ b/crates/core/src/network/network_sender.rs @@ -8,7 +8,7 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -use restate_node_protocol::codec::{Targeted, WireEncode}; +use restate_types::net::codec::{Targeted, WireEncode}; use restate_types::NodeId; use super::NetworkSendError; diff --git a/crates/core/src/test_env.rs b/crates/core/src/test_env.rs index b438b98c75..72c8fc6ef1 100644 --- a/crates/core/src/test_env.rs +++ b/crates/core/src/test_env.rs @@ -13,19 +13,19 @@ use std::sync::Arc; use tokio::sync::{mpsc, RwLock}; -use restate_node_protocol::codec::{ - serialize_message, try_unwrap_binary_message, Targeted, WireEncode, -}; -use restate_node_protocol::metadata::MetadataKind; -use restate_node_protocol::node::{Header, Message}; -use restate_node_protocol::CURRENT_PROTOCOL_VERSION; use restate_types::logs::metadata::{create_static_metadata, ProviderKind}; use restate_types::metadata_store::keys::{ BIFROST_CONFIG_KEY, NODES_CONFIG_KEY, PARTITION_TABLE_KEY, }; +use restate_types::net::codec::{ + serialize_message, try_unwrap_binary_message, Targeted, WireEncode, +}; +use restate_types::net::metadata::MetadataKind; use restate_types::net::AdvertisedAddress; +use restate_types::net::CURRENT_PROTOCOL_VERSION; use restate_types::nodes_config::{NodeConfig, NodesConfiguration, Role}; use restate_types::partition_table::FixedPartitionTable; +use restate_types::protobuf::node::{Header, Message}; use restate_types::{GenerationalNodeId, NodeId, Version}; use tracing::info; diff --git a/crates/ingress-dispatcher/Cargo.toml b/crates/ingress-dispatcher/Cargo.toml index f74f1804c3..7efeca9469 100644 --- a/crates/ingress-dispatcher/Cargo.toml +++ b/crates/ingress-dispatcher/Cargo.toml @@ -14,8 +14,6 @@ test-util = ["dep:restate-test-util"] [dependencies] restate-bifrost = { workspace = true } restate-core = { workspace = true } -restate-node-protocol = { workspace = true } -restate-schema-api = { workspace = true, features = ["subscription"] } # todo: only needed for DedupInformation :-( Probably fixed by merging restate-storage-api with restate-types restate-storage-api = { workspace = true } restate-test-util = { workspace = true, optional = true } diff --git a/crates/ingress-dispatcher/src/dispatcher.rs b/crates/ingress-dispatcher/src/dispatcher.rs index 975e181ce3..5e1879a2cf 100644 --- a/crates/ingress-dispatcher/src/dispatcher.rs +++ b/crates/ingress-dispatcher/src/dispatcher.rs @@ -18,11 +18,12 @@ use dashmap::DashMap; use restate_bifrost::Bifrost; use restate_core::metadata; use restate_core::network::MessageHandler; -use restate_node_protocol::codec::Targeted; -use restate_node_protocol::ingress::IngressMessage; use restate_storage_api::deduplication_table::DedupInformation; use restate_types::identifiers::{IngressRequestId, PartitionKey, WithPartitionKey}; use restate_types::message::MessageIndex; +use restate_types::net::codec::Targeted; +use restate_types::net::ingress::IngressMessage; +use restate_types::net::MessageEnvelope; use restate_types::GenerationalNodeId; use restate_wal_protocol::{ append_envelope_to_bifrost, Command, Destination, Envelope, Header, Source, @@ -146,7 +147,7 @@ impl DispatchIngressRequest for IngressDispatcher { impl MessageHandler for IngressDispatcher { type MessageType = IngressMessage; - async fn on_message(&self, msg: restate_node_protocol::MessageEnvelope) { + async fn on_message(&self, msg: MessageEnvelope) { let (peer, msg) = msg.split(); trace!("Processing message '{}' from '{}'", msg.kind(), peer); diff --git a/crates/ingress-dispatcher/src/lib.rs b/crates/ingress-dispatcher/src/lib.rs index c4da40fdf1..cfa261d439 100644 --- a/crates/ingress-dispatcher/src/lib.rs +++ b/crates/ingress-dispatcher/src/lib.rs @@ -8,9 +8,14 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. +use std::fmt::Display; +use std::future::Future; +use std::hash::Hash; + use bytes::Bytes; +use tokio::sync::oneshot; + use restate_core::metadata; -use restate_schema_api::subscription::{EventReceiverServiceType, Sink, Subscription}; use restate_types::identifiers::{ partitioner, IngressRequestId, InvocationId, PartitionKey, WithPartitionKey, }; @@ -21,10 +26,7 @@ use restate_types::invocation::{ SubmitNotificationSink, VirtualObjectHandlerType, WorkflowHandlerType, }; use restate_types::message::MessageIndex; -use std::fmt::Display; -use std::future::Future; -use std::hash::Hash; -use tokio::sync::oneshot; +use restate_types::schema::subscriptions::{EventReceiverServiceType, Sink, Subscription}; mod dispatcher; pub mod error; diff --git a/crates/ingress-http/Cargo.toml b/crates/ingress-http/Cargo.toml index d813ee76ed..787bcca2c3 100644 --- a/crates/ingress-http/Cargo.toml +++ b/crates/ingress-http/Cargo.toml @@ -17,7 +17,6 @@ restate-core = { workspace = true } restate-errors = { workspace = true } restate-ingress-dispatcher = { workspace = true } restate-serde-util = { workspace = true } -restate-schema-api = { workspace = true, features = ["service", "invocation_target"]} restate-types = { workspace = true } restate-service-protocol = { workspace = true, features = [ "awakeable-id" ] } @@ -68,7 +67,6 @@ restate-core = { workspace = true, features = ["test-util"] } restate-ingress-dispatcher = { workspace = true, features = ["test-util"] } restate-test-util = { workspace = true } restate-types = { workspace = true, features = ["test-util"] } -restate-schema-api = { workspace = true, features = ["test-util"]} hyper = { version = "1.0", features = ["full"] } hyper-util = { version = "0.1", features = ["full"] } @@ -76,4 +74,4 @@ hyper-util = { version = "0.1", features = ["full"] } tracing-test = { workspace = true } tracing-subscriber = { workspace = true } -googletest = { workspace = true } \ No newline at end of file +googletest = { workspace = true } diff --git a/crates/ingress-http/src/handler/error.rs b/crates/ingress-http/src/handler/error.rs index 3b8c1968f0..a0bbbb6ee0 100644 --- a/crates/ingress-http/src/handler/error.rs +++ b/crates/ingress-http/src/handler/error.rs @@ -12,8 +12,8 @@ use super::APPLICATION_JSON; use bytes::Bytes; use http::{header, Response, StatusCode}; -use restate_schema_api::invocation_target::InputValidationError; use restate_types::errors::{IdDecodeError, InvocationError}; +use restate_types::schema::invocation_target::InputValidationError; use serde::Serialize; use std::string; diff --git a/crates/ingress-http/src/handler/health.rs b/crates/ingress-http/src/handler/health.rs index 448bc9d1e0..b04411c51f 100644 --- a/crates/ingress-http/src/handler/health.rs +++ b/crates/ingress-http/src/handler/health.rs @@ -8,15 +8,16 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -use super::{Handler, APPLICATION_JSON}; - -use crate::handler::error::HandlerError; use bytes::Bytes; use http::{header, Method, Request, Response, StatusCode}; use http_body_util::Full; -use restate_schema_api::service::ServiceMetadataResolver; use serde::Serialize; +use restate_types::schema::service::ServiceMetadataResolver; + +use super::{Handler, APPLICATION_JSON}; +use crate::handler::error::HandlerError; + #[derive(Debug, Serialize)] #[cfg_attr(test, derive(serde::Deserialize))] #[serde(rename_all = "camelCase")] diff --git a/crates/ingress-http/src/handler/invocation.rs b/crates/ingress-http/src/handler/invocation.rs index 37bfea8b36..dddc826851 100644 --- a/crates/ingress-http/src/handler/invocation.rs +++ b/crates/ingress-http/src/handler/invocation.rs @@ -8,20 +8,21 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -use super::path_parsing::{InvocationRequestType, InvocationTargetType, TargetType}; -use super::Handler; -use super::HandlerError; - -use crate::{GetOutputResult, InvocationStorageReader}; use bytes::Bytes; use http::{Method, Request, Response}; use http_body_util::Full; +use tracing::warn; + use restate_ingress_dispatcher::DispatchIngressRequest; use restate_ingress_dispatcher::IngressDispatcherRequest; -use restate_schema_api::invocation_target::InvocationTargetResolver; use restate_types::identifiers::IdempotencyId; use restate_types::invocation::InvocationQuery; -use tracing::warn; +use restate_types::schema::invocation_target::InvocationTargetResolver; + +use super::path_parsing::{InvocationRequestType, InvocationTargetType, TargetType}; +use super::Handler; +use super::HandlerError; +use crate::{GetOutputResult, InvocationStorageReader}; impl Handler where diff --git a/crates/ingress-http/src/handler/mod.rs b/crates/ingress-http/src/handler/mod.rs index 3f469e4726..e8885fae9b 100644 --- a/crates/ingress-http/src/handler/mod.rs +++ b/crates/ingress-http/src/handler/mod.rs @@ -18,8 +18,8 @@ use hyper::http::HeaderValue; use hyper::{Request, Response}; use path_parsing::RequestType; use restate_ingress_dispatcher::DispatchIngressRequest; -use restate_schema_api::invocation_target::InvocationTargetResolver; -use restate_schema_api::service::ServiceMetadataResolver; +use restate_types::schema::invocation_target::InvocationTargetResolver; +use restate_types::schema::service::ServiceMetadataResolver; use std::convert::Infallible; use std::task::{Context, Poll}; diff --git a/crates/ingress-http/src/handler/path_parsing.rs b/crates/ingress-http/src/handler/path_parsing.rs index 15e3a4a915..c11a0ba51f 100644 --- a/crates/ingress-http/src/handler/path_parsing.rs +++ b/crates/ingress-http/src/handler/path_parsing.rs @@ -8,12 +8,13 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. +use http::Uri; + +use restate_types::schema::service::ServiceMetadataResolver; + use super::Handler; use super::HandlerError; - use crate::InvocationStorageReader; -use http::Uri; -use restate_schema_api::service::ServiceMetadataResolver; pub(crate) enum WorkflowRequestType { Attach(String, String), diff --git a/crates/ingress-http/src/handler/responses.rs b/crates/ingress-http/src/handler/responses.rs index 74d4097ce5..c2c1e01253 100644 --- a/crates/ingress-http/src/handler/responses.rs +++ b/crates/ingress-http/src/handler/responses.rs @@ -8,16 +8,18 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -use crate::handler::error::HandlerError; -use crate::handler::Handler; use bytes::Bytes; use http::{header, HeaderName, Response}; use http_body_util::Full; -use restate_schema_api::invocation_target::InvocationTargetMetadata; +use tracing::{info, trace}; + use restate_types::identifiers::InvocationId; use restate_types::ingress::IngressResponseResult; use restate_types::invocation::InvocationTarget; -use tracing::{info, trace}; +use restate_types::schema::invocation_target::InvocationTargetMetadata; + +use crate::handler::error::HandlerError; +use crate::handler::Handler; pub(crate) const IDEMPOTENCY_EXPIRES: HeaderName = HeaderName::from_static("idempotency-expires"); /// Contains the string representation of the invocation id diff --git a/crates/ingress-http/src/handler/service_handler.rs b/crates/ingress-http/src/handler/service_handler.rs index abdc517c9d..bd1afb32a0 100644 --- a/crates/ingress-http/src/handler/service_handler.rs +++ b/crates/ingress-http/src/handler/service_handler.rs @@ -8,30 +8,34 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -use super::path_parsing::{InvokeType, ServiceRequestType, TargetType}; -use super::tracing::prepare_tracing_span; -use super::HandlerError; -use super::{Handler, APPLICATION_JSON}; +use std::time::{Duration, Instant, SystemTime}; -use crate::handler::responses::{IDEMPOTENCY_EXPIRES, X_RESTATE_ID}; -use crate::metric_definitions::{INGRESS_REQUESTS, INGRESS_REQUEST_DURATION, REQUEST_COMPLETED}; use bytes::Bytes; use bytestring::ByteString; use http::{header, HeaderMap, HeaderName, Method, Request, Response, StatusCode}; use http_body_util::{BodyExt, Full}; use metrics::{counter, histogram}; +use serde::de::IntoDeserializer; +use serde::{Deserialize, Serialize}; +use serde_with::serde_as; +use tracing::{info, trace, warn, Instrument}; + use restate_ingress_dispatcher::{DispatchIngressRequest, IngressDispatcherRequest}; -use restate_schema_api::invocation_target::{InvocationTargetMetadata, InvocationTargetResolver}; use restate_types::identifiers::InvocationId; use restate_types::invocation::{ Header, InvocationTarget, InvocationTargetType, ServiceInvocation, Source, SpanRelation, WorkflowHandlerType, }; -use serde::de::IntoDeserializer; -use serde::{Deserialize, Serialize}; -use serde_with::serde_as; -use std::time::{Duration, Instant, SystemTime}; -use tracing::{info, trace, warn, Instrument}; +use restate_types::schema::invocation_target::{ + InvocationTargetMetadata, InvocationTargetResolver, +}; + +use super::path_parsing::{InvokeType, ServiceRequestType, TargetType}; +use super::tracing::prepare_tracing_span; +use super::HandlerError; +use super::{Handler, APPLICATION_JSON}; +use crate::handler::responses::{IDEMPOTENCY_EXPIRES, X_RESTATE_ID}; +use crate::metric_definitions::{INGRESS_REQUESTS, INGRESS_REQUEST_DURATION, REQUEST_COMPLETED}; pub(crate) const IDEMPOTENCY_KEY: HeaderName = HeaderName::from_static("idempotency-key"); const DELAY_QUERY_PARAM: &str = "delay"; diff --git a/crates/ingress-http/src/handler/tests.rs b/crates/ingress-http/src/handler/tests.rs index f72c71c90b..898abe1b66 100644 --- a/crates/ingress-http/src/handler/tests.rs +++ b/crates/ingress-http/src/handler/tests.rs @@ -8,28 +8,23 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -use super::health::HealthResponse; -use super::mocks::*; -use super::service_handler::*; -use super::ConnectInfo; -use super::Handler; -use restate_ingress_dispatcher::{IngressInvocationResponse, SubmittedInvocationNotification}; use std::collections::HashMap; +use std::time::Duration; -use crate::handler::responses::X_RESTATE_ID; use bytes::Bytes; use bytestring::ByteString; use googletest::prelude::*; use http::StatusCode; use http::{Method, Request, Response}; use http_body_util::{BodyExt, Empty, Full}; +use tokio::sync::mpsc; +use tower::ServiceExt; +use tracing_test::traced_test; + use restate_core::TestCoreEnv; use restate_ingress_dispatcher::test_util::MockDispatcher; use restate_ingress_dispatcher::IngressDispatcherRequest; -use restate_schema_api::invocation_target::{ - InputContentType, InputRules, InputValidationRule, InvocationTargetMetadata, - OutputContentTypeRule, OutputRules, -}; +use restate_ingress_dispatcher::{IngressInvocationResponse, SubmittedInvocationNotification}; use restate_test_util::{assert, assert_eq}; use restate_types::identifiers::{IdempotencyId, InvocationId, ServiceId}; use restate_types::ingress::{IngressResponseResult, InvocationResponse}; @@ -37,10 +32,17 @@ use restate_types::invocation::{ Header, InvocationQuery, InvocationTarget, InvocationTargetType, VirtualObjectHandlerType, WorkflowHandlerType, }; -use std::time::Duration; -use tokio::sync::mpsc; -use tower::ServiceExt; -use tracing_test::traced_test; +use restate_types::schema::invocation_target::{ + InputContentType, InputRules, InputValidationRule, InvocationTargetMetadata, + OutputContentTypeRule, OutputRules, +}; + +use super::health::HealthResponse; +use super::mocks::*; +use super::service_handler::*; +use super::ConnectInfo; +use super::Handler; +use crate::handler::responses::X_RESTATE_ID; #[tokio::test] #[traced_test] diff --git a/crates/ingress-http/src/handler/workflow.rs b/crates/ingress-http/src/handler/workflow.rs index 5bc90dfa7d..dae2f91b59 100644 --- a/crates/ingress-http/src/handler/workflow.rs +++ b/crates/ingress-http/src/handler/workflow.rs @@ -8,20 +8,21 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -use super::path_parsing::WorkflowRequestType; -use super::Handler; -use super::HandlerError; - -use crate::{GetOutputResult, InvocationStorageReader}; use bytes::Bytes; use http::{Method, Request, Response}; use http_body_util::Full; +use tracing::{info, warn}; + use restate_ingress_dispatcher::DispatchIngressRequest; use restate_ingress_dispatcher::IngressDispatcherRequest; -use restate_schema_api::invocation_target::InvocationTargetResolver; use restate_types::identifiers::ServiceId; use restate_types::invocation::InvocationQuery; -use tracing::{info, warn}; +use restate_types::schema::invocation_target::InvocationTargetResolver; + +use super::path_parsing::WorkflowRequestType; +use super::Handler; +use super::HandlerError; +use crate::{GetOutputResult, InvocationStorageReader}; impl Handler where diff --git a/crates/ingress-http/src/lib.rs b/crates/ingress-http/src/lib.rs index d14a326b3f..98415ba9fe 100644 --- a/crates/ingress-http/src/lib.rs +++ b/crates/ingress-http/src/lib.rs @@ -55,22 +55,26 @@ pub trait InvocationStorageReader { // Contains some mocks we use in unit tests in this crate #[cfg(test)] mod mocks { - use super::*; + use std::collections::HashMap; use anyhow::Error; - use restate_schema_api::invocation_target::test_util::MockInvocationTargetResolver; - use restate_schema_api::invocation_target::{ - InvocationTargetMetadata, InvocationTargetResolver, DEFAULT_IDEMPOTENCY_RETENTION, - }; - use restate_schema_api::service::test_util::MockServiceMetadataResolver; - use restate_schema_api::service::{HandlerMetadata, ServiceMetadata, ServiceMetadataResolver}; + use serde::{Deserialize, Serialize}; + use restate_types::identifiers::DeploymentId; use restate_types::ingress::InvocationResponse; use restate_types::invocation::{ InvocationQuery, InvocationTargetType, ServiceType, VirtualObjectHandlerType, }; - use serde::{Deserialize, Serialize}; - use std::collections::HashMap; + use restate_types::schema::invocation_target::test_util::MockInvocationTargetResolver; + use restate_types::schema::invocation_target::{ + InvocationTargetMetadata, InvocationTargetResolver, DEFAULT_IDEMPOTENCY_RETENTION, + }; + use restate_types::schema::service::test_util::MockServiceMetadataResolver; + use restate_types::schema::service::{ + HandlerMetadata, ServiceMetadata, ServiceMetadataResolver, + }; + + use super::*; #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub(super) struct GreetingRequest { diff --git a/crates/ingress-http/src/server.rs b/crates/ingress-http/src/server.rs index accbf3781b..87e3e5fb74 100644 --- a/crates/ingress-http/src/server.rs +++ b/crates/ingress-http/src/server.rs @@ -20,9 +20,9 @@ use hyper_util::rt::TokioIo; use hyper_util::server::conn::auto; use restate_core::{cancellation_watcher, task_center, TaskKind}; use restate_ingress_dispatcher::{DispatchIngressRequest, IngressDispatcher}; -use restate_schema_api::invocation_target::InvocationTargetResolver; -use restate_schema_api::service::ServiceMetadataResolver; use restate_types::config::IngressOptions; +use restate_types::schema::invocation_target::InvocationTargetResolver; +use restate_types::schema::service::ServiceMetadataResolver; use std::convert::Infallible; use std::future::Future; use std::net::SocketAddr; diff --git a/crates/ingress-kafka/Cargo.toml b/crates/ingress-kafka/Cargo.toml index d737f8bd3c..a2337a92a6 100644 --- a/crates/ingress-kafka/Cargo.toml +++ b/crates/ingress-kafka/Cargo.toml @@ -15,7 +15,6 @@ options_schema = ["dep:schemars"] restate-core = { workspace = true } restate-errors = { workspace = true } restate-ingress-dispatcher = { workspace = true } -restate-schema-api = { workspace = true, features = ["subscription"] } restate-timer-queue = { workspace = true } restate-types = { workspace = true } @@ -34,8 +33,6 @@ tracing-opentelemetry = { workspace = true } [dev-dependencies] restate-ingress-dispatcher = { workspace = true, features = ["test-util"] } -restate-schema = { workspace = true } -restate-schema-api = { workspace = true, features = ["test-util"] } restate-test-util = { workspace = true } restate-types = { workspace = true, features = ["test-util"] } diff --git a/crates/ingress-kafka/src/consumer_task.rs b/crates/ingress-kafka/src/consumer_task.rs index 5e6d8ccd4e..616922e4d7 100644 --- a/crates/ingress-kafka/src/consumer_task.rs +++ b/crates/ingress-kafka/src/consumer_task.rs @@ -8,6 +8,8 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. +use std::fmt; + use base64::Engine; use bytes::Bytes; use opentelemetry::trace::TraceContextExt; @@ -15,17 +17,17 @@ use rdkafka::consumer::{Consumer, DefaultConsumerContext, StreamConsumer}; use rdkafka::error::KafkaError; use rdkafka::message::BorrowedMessage; use rdkafka::{ClientConfig, Message}; +use tokio::sync::oneshot; +use tracing::{debug, info, info_span, Instrument}; +use tracing_opentelemetry::OpenTelemetrySpanExt; + use restate_ingress_dispatcher::{ DeduplicationId, DispatchIngressRequest, IngressDispatcher, IngressDispatcherRequest, }; -use restate_schema_api::subscription::{EventReceiverServiceType, Sink, Subscription}; use restate_types::identifiers::SubscriptionId; use restate_types::invocation::{Header, SpanRelation}; use restate_types::message::MessageIndex; -use std::fmt; -use tokio::sync::oneshot; -use tracing::{debug, info, info_span, Instrument}; -use tracing_opentelemetry::OpenTelemetrySpanExt; +use restate_types::schema::subscriptions::{EventReceiverServiceType, Sink, Subscription}; #[derive(Debug, thiserror::Error)] pub enum Error { diff --git a/crates/ingress-kafka/src/subscription_controller.rs b/crates/ingress-kafka/src/subscription_controller.rs index 57805973fc..b456229baa 100644 --- a/crates/ingress-kafka/src/subscription_controller.rs +++ b/crates/ingress-kafka/src/subscription_controller.rs @@ -16,11 +16,11 @@ use crate::subscription_controller::task_orchestrator::TaskOrchestrator; use rdkafka::error::KafkaError; use restate_core::cancellation_watcher; use restate_ingress_dispatcher::IngressDispatcher; -use restate_schema_api::subscription::{Source, Subscription}; use restate_types::arc_util::Updateable; use restate_types::config::IngressOptions; use restate_types::identifiers::SubscriptionId; use restate_types::retries::RetryPolicy; +use restate_types::schema::subscriptions::{Source, Subscription}; use std::time::Duration; use tokio::sync::mpsc; diff --git a/crates/invoker-impl/Cargo.toml b/crates/invoker-impl/Cargo.toml index 9029306b30..cc52e3945a 100644 --- a/crates/invoker-impl/Cargo.toml +++ b/crates/invoker-impl/Cargo.toml @@ -18,7 +18,6 @@ restate-fs-util = { workspace = true } restate-futures-util = { workspace = true } restate-invoker-api = { workspace = true } restate-queue = { workspace = true } -restate-schema-api = { workspace = true, features = ["deployment"] } restate-service-client = { workspace = true } restate-service-protocol = { workspace = true, features = ["message"] } restate-timer-queue = { workspace = true } @@ -48,7 +47,6 @@ tracing-opentelemetry = { workspace = true } [dev-dependencies] restate-core = { workspace = true, features = ["test-util"] } restate-invoker-api = { workspace = true, features = ["test-util"] } -restate-schema-api = { workspace = true, features = ["test-util"] } restate-service-protocol = { workspace = true, features = ["codec"] } restate-test-util = { workspace = true } restate-types = { workspace = true } diff --git a/crates/invoker-impl/src/invocation_task/mod.rs b/crates/invoker-impl/src/invocation_task/mod.rs index 3ddc4604a5..7474fb992f 100644 --- a/crates/invoker-impl/src/invocation_task/mod.rs +++ b/crates/invoker-impl/src/invocation_task/mod.rs @@ -24,7 +24,6 @@ use restate_invoker_api::{ EagerState, EntryEnricher, InvocationErrorReport, InvokeInputJournal, JournalReader, StateReader, }; -use restate_schema_api::deployment::DeploymentResolver; use restate_service_client::{Request, ServiceClient, ServiceClientError}; use restate_service_protocol::message::{EncodingError, MessageType}; use restate_types::deployment::PinnedDeployment; @@ -33,6 +32,7 @@ use restate_types::identifiers::{DeploymentId, EntryIndex, InvocationId, Partiti use restate_types::invocation::InvocationTarget; use restate_types::journal::enriched::EnrichedRawEntry; use restate_types::journal::EntryType; +use restate_types::schema::deployment::DeploymentResolver; use restate_types::service_protocol::ServiceProtocolVersion; use restate_types::service_protocol::{MAX_SERVICE_PROTOCOL_VERSION, MIN_SERVICE_PROTOCOL_VERSION}; use std::collections::HashSet; diff --git a/crates/invoker-impl/src/invocation_task/service_protocol_runner.rs b/crates/invoker-impl/src/invocation_task/service_protocol_runner.rs index 48ebdba783..7041fa40b9 100644 --- a/crates/invoker-impl/src/invocation_task/service_protocol_runner.rs +++ b/crates/invoker-impl/src/invocation_task/service_protocol_runner.rs @@ -25,9 +25,6 @@ use opentelemetry_http::HeaderInjector; use opentelemetry_sdk::propagation::TraceContextPropagator; use restate_errors::warn_it; use restate_invoker_api::{EagerState, EntryEnricher, JournalMetadata}; -use restate_schema_api::deployment::{ - Deployment, DeploymentMetadata, DeploymentType, ProtocolType, -}; use restate_service_client::{Endpoint, Method, Parts, Request, ServiceClientError}; use restate_service_protocol::message::{ Decoder, Encoder, MessageHeader, MessageType, ProtocolMessage, @@ -37,6 +34,9 @@ use restate_types::identifiers::EntryIndex; use restate_types::invocation::ServiceInvocationSpanContext; use restate_types::journal::raw::PlainRawEntry; use restate_types::journal::EntryType; +use restate_types::schema::deployment::{ + Deployment, DeploymentMetadata, DeploymentType, ProtocolType, +}; use restate_types::service_protocol::ServiceProtocolVersion; use std::collections::HashSet; use std::future::poll_fn; diff --git a/crates/invoker-impl/src/lib.rs b/crates/invoker-impl/src/lib.rs index 0b89398c8c..bb78d861e2 100644 --- a/crates/invoker-impl/src/lib.rs +++ b/crates/invoker-impl/src/lib.rs @@ -29,7 +29,6 @@ use restate_invoker_api::{ InvokeInputJournal, JournalReader, StateReader, }; use restate_queue::SegmentQueue; -use restate_schema_api::deployment::DeploymentResolver; use restate_timer_queue::TimerQueue; use restate_types::arc_util::Updateable; use restate_types::config::{InvokerOptions, ServiceClientOptions}; @@ -39,6 +38,7 @@ use restate_types::journal::enriched::EnrichedRawEntry; use restate_types::journal::raw::PlainRawEntry; use restate_types::journal::Completion; use restate_types::retries::RetryPolicy; +use restate_types::schema::deployment::DeploymentResolver; use status_store::InvocationStatusStore; use std::collections::{HashMap, HashSet}; use std::future::Future; @@ -1031,12 +1031,12 @@ mod tests { use tokio_util::sync::CancellationToken; use restate_invoker_api::{entry_enricher, ServiceHandle}; - use restate_schema_api::deployment::test_util::MockDeploymentMetadataRegistry; use restate_test_util::{check, let_assert}; use restate_types::identifiers::{LeaderEpoch, PartitionId}; use restate_types::journal::enriched::EnrichedEntryHeader; use restate_types::journal::raw::RawEntry; use restate_types::retries::RetryPolicy; + use restate_types::schema::deployment::test_util::MockDeploymentMetadataRegistry; use crate::invocation_task::InvocationTaskError; use crate::quota::InvokerConcurrencyQuota; diff --git a/crates/metadata-store/Cargo.toml b/crates/metadata-store/Cargo.toml index 63dd97a2d7..a2bdaaf087 100644 --- a/crates/metadata-store/Cargo.toml +++ b/crates/metadata-store/Cargo.toml @@ -15,7 +15,6 @@ test-util = [] codederror = { workspace = true } restate-core = { workspace = true } restate-grpc-util = { workspace = true } -restate-node-protocol = { workspace = true } restate-rocksdb = { workspace = true } restate-types = { workspace = true } diff --git a/crates/network/Cargo.toml b/crates/network/Cargo.toml index 1bee13ff9d..76494d2379 100644 --- a/crates/network/Cargo.toml +++ b/crates/network/Cargo.toml @@ -10,8 +10,6 @@ publish = false [dependencies] restate-core = { workspace = true } restate-grpc-util = { workspace = true } -restate-node-protocol = { workspace = true } -restate-node-services = { workspace = true, features = ["clients"] } restate-types = { workspace = true } anyhow = { workspace = true } @@ -25,15 +23,20 @@ http = { workspace = true } metrics = { workspace = true } once_cell = { workspace = true } pin-project = { workspace = true } +prost = { workspace = true } +prost-types = { workspace = true } rand = { workspace = true } static_assertions = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true } tokio-stream = { workspace = true } -tonic = { workspace = true } +tonic = { workspace = true, features = ["transport", "codegen", "prost", "gzip"] } tower = { workspace = true } tracing = { workspace = true } +[build-dependencies] +tonic-build = { workspace = true } + [dev-dependencies] restate-core = { workspace = true, features = ["test-util"] } restate-test-util = { workspace = true } diff --git a/crates/network/build.rs b/crates/network/build.rs new file mode 100644 index 0000000000..abf6175818 --- /dev/null +++ b/crates/network/build.rs @@ -0,0 +1,30 @@ +// Copyright (c) 2024 - Restate Software, Inc., Restate GmbH. +// All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use std::env; +use std::path::PathBuf; + +fn main() -> Result<(), Box> { + let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap()); + + tonic_build::configure() + .bytes(["."]) + .file_descriptor_set_path(out_dir.join("node_svc_descriptor.bin")) + // allow older protobuf compiler to be used + .protoc_arg("--experimental_allow_proto3_optional") + .extern_path(".restate.node", "::restate_types::protobuf::node") + .extern_path(".restate.common", "::restate_types::protobuf::common") + .compile( + &["./protobuf/node_svc.proto"], + &["protobuf", "../types/protobuf"], + )?; + + Ok(()) +} diff --git a/crates/node-services/proto/node_svc.proto b/crates/network/protobuf/node_svc.proto similarity index 67% rename from crates/node-services/proto/node_svc.proto rename to crates/network/protobuf/node_svc.proto index 0300aa2d21..e69ced037e 100644 --- a/crates/node-services/proto/node_svc.proto +++ b/crates/network/protobuf/node_svc.proto @@ -10,10 +10,10 @@ syntax = "proto3"; import "google/protobuf/empty.proto"; -import "common.proto"; -import "node.proto"; +import "restate/common.proto"; +import "restate/node.proto"; -package dev.restate.node_svc; +package restate.node_svc; service NodeSvc { // Get identity information from this node. @@ -24,21 +24,13 @@ service NodeSvc { rpc QueryStorage(StorageQueryRequest) returns (stream StorageQueryResponse); // Create a bidirectional node-to-node stream - rpc CreateConnection(stream dev.restate.node.Message) returns (stream dev.restate.node.Message); -} - -enum NodeStatus { - NodeStatus_UNKNOWN = 0; - ALIVE = 1; - // The node is not fully running yet. - STARTING_UP = 2; - // The node is performing a graceful shutdown. - SHUTTING_DOWN = 3; + rpc CreateConnection(stream restate.node.Message) + returns (stream restate.node.Message); } message IdentResponse { - NodeStatus status = 1; - dev.restate.common.NodeId node_id = 2; + restate.common.NodeStatus status = 1; + restate.common.NodeId node_id = 2; } message StorageQueryRequest { string query = 1; } diff --git a/crates/network/src/connection.rs b/crates/network/src/connection.rs index a7190ba18d..20c425c0f4 100644 --- a/crates/network/src/connection.rs +++ b/crates/network/src/connection.rs @@ -17,12 +17,12 @@ use tracing::instrument; use restate_core::metadata; use restate_core::network::NetworkSendError; -use restate_node_protocol::codec::Targeted; -use restate_node_protocol::codec::{serialize_message, WireEncode}; -use restate_node_protocol::common::ProtocolVersion; -use restate_node_protocol::node::message; -use restate_node_protocol::node::Header; -use restate_node_protocol::node::Message; +use restate_types::net::codec::Targeted; +use restate_types::net::codec::{serialize_message, WireEncode}; +use restate_types::net::ProtocolVersion; +use restate_types::protobuf::node::message; +use restate_types::protobuf::node::Header; +use restate_types::protobuf::node::Message; use restate_types::GenerationalNodeId; use crate::metric_definitions::CONNECTION_SEND_DURATION; diff --git a/crates/network/src/connection_manager.rs b/crates/network/src/connection_manager.rs index 962946b344..0d52a8d842 100644 --- a/crates/network/src/connection_manager.rs +++ b/crates/network/src/connection_manager.rs @@ -16,7 +16,7 @@ use futures::stream::BoxStream; use futures::{Stream, StreamExt}; use rand::seq::SliceRandom; use restate_core::network::{Handler, MessageRouter}; -use restate_node_protocol::codec::try_unwrap_binary_message; +use restate_types::net::codec::try_unwrap_binary_message; use tokio::sync::mpsc; use tokio_stream::wrappers::ReceiverStream; use tonic::transport::Channel; @@ -25,10 +25,9 @@ use tracing::{debug, info, trace, warn, Instrument, Span}; use restate_core::metadata; use restate_core::{cancellation_watcher, current_task_id, task_center, TaskId, TaskKind}; use restate_grpc_util::create_grpc_channel_from_advertised_address; -use restate_node_protocol::node::message::{self, ConnectionControl}; -use restate_node_protocol::node::{Header, Hello, Message, Welcome}; -use restate_node_services::node_svc::node_svc_client::NodeSvcClient; use restate_types::net::AdvertisedAddress; +use restate_types::protobuf::node::message::{self, ConnectionControl}; +use restate_types::protobuf::node::{Header, Hello, Message, Welcome}; use restate_types::{GenerationalNodeId, NodeId, PlainNodeId}; use super::connection::{Connection, ConnectionSender}; @@ -38,6 +37,7 @@ use crate::metric_definitions::{ self, CONNECTION_DROPPED, INCOMING_CONNECTION, MESSAGE_PROCESSING_DURATION, MESSAGE_RECEIVED, ONGOING_DRAIN, OUTGOING_CONNECTION, }; +use crate::protobuf::node_svc::node_svc_client::NodeSvcClient; // todo: make this configurable const SEND_QUEUE_SIZE: usize = 1; @@ -283,11 +283,7 @@ impl ConnectionManager { node_id: GenerationalNodeId, ) -> Result, NetworkError> { let (tx, rx) = mpsc::channel(SEND_QUEUE_SIZE); - let connection = Connection::new( - node_id, - restate_node_protocol::common::CURRENT_PROTOCOL_VERSION, - tx, - ); + let connection = Connection::new(node_id, restate_types::net::CURRENT_PROTOCOL_VERSION, tx); let transformed = ReceiverStream::new(rx).map(Ok); let incoming = Box::pin(transformed); @@ -606,12 +602,12 @@ mod tests { use googletest::prelude::*; use restate_core::TestCoreEnv; - use restate_node_protocol::node::message; - use restate_node_protocol::{ - common::ProtocolVersion, CURRENT_PROTOCOL_VERSION, MIN_SUPPORTED_PROTOCOL_VERSION, - }; use restate_test_util::assert_eq; + use restate_types::net::{ + ProtocolVersion, CURRENT_PROTOCOL_VERSION, MIN_SUPPORTED_PROTOCOL_VERSION, + }; use restate_types::nodes_config::NodesConfigError; + use restate_types::protobuf::node::message; // Test handshake with a client #[tokio::test] diff --git a/crates/network/src/error.rs b/crates/network/src/error.rs index 0134e566ba..6c6a7c8fd6 100644 --- a/crates/network/src/error.rs +++ b/crates/network/src/error.rs @@ -9,7 +9,7 @@ // by the Apache License, Version 2.0. use restate_core::ShutdownError; -use restate_node_protocol::common::MIN_SUPPORTED_PROTOCOL_VERSION; +use restate_types::net::MIN_SUPPORTED_PROTOCOL_VERSION; use restate_types::nodes_config::NodesConfigError; use restate_types::NodeId; diff --git a/crates/network/src/handshake.rs b/crates/network/src/handshake.rs index 73925d5a79..78893930d5 100644 --- a/crates/network/src/handshake.rs +++ b/crates/network/src/handshake.rs @@ -11,8 +11,8 @@ use std::time::Duration; use futures::Stream; -use restate_node_protocol::common::{ProtocolVersion, CURRENT_PROTOCOL_VERSION}; -use restate_node_protocol::node::{message, Header, Hello, Message, Welcome}; +use restate_types::net::{ProtocolVersion, CURRENT_PROTOCOL_VERSION}; +use restate_types::protobuf::node::{message, Header, Hello, Message, Welcome}; use tokio_stream::StreamExt; use crate::error::ProtocolError; diff --git a/crates/network/src/lib.rs b/crates/network/src/lib.rs index 5d40fe1739..db56592c0e 100644 --- a/crates/network/src/lib.rs +++ b/crates/network/src/lib.rs @@ -14,6 +14,7 @@ pub mod error; mod handshake; pub(crate) mod metric_definitions; mod networking; +pub mod protobuf; pub mod rpc_router; pub use connection::ConnectionSender; diff --git a/crates/network/src/networking.rs b/crates/network/src/networking.rs index f7acbecd93..d7f97d585d 100644 --- a/crates/network/src/networking.rs +++ b/crates/network/src/networking.rs @@ -15,7 +15,7 @@ use tracing::{info, instrument, trace}; use restate_core::metadata; use restate_core::network::{NetworkSendError, NetworkSender}; -use restate_node_protocol::codec::{Targeted, WireEncode}; +use restate_types::net::codec::{Targeted, WireEncode}; use restate_types::NodeId; use crate::error::NetworkError; diff --git a/crates/network/src/protobuf.rs b/crates/network/src/protobuf.rs new file mode 100644 index 0000000000..458a5558ea --- /dev/null +++ b/crates/network/src/protobuf.rs @@ -0,0 +1,16 @@ +// Copyright (c) 2024 - Restate Software, Inc., Restate GmbH. +// All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +pub mod node_svc { + tonic::include_proto!("restate.node_svc"); + + pub const FILE_DESCRIPTOR_SET: &[u8] = + tonic::include_file_descriptor_set!("node_svc_descriptor"); +} diff --git a/crates/network/src/rpc_router.rs b/crates/network/src/rpc_router.rs index 8f091524fb..8245efcfe7 100644 --- a/crates/network/src/rpc_router.rs +++ b/crates/network/src/rpc_router.rs @@ -15,14 +15,14 @@ use dashmap::DashMap; use futures::stream::BoxStream; use futures::StreamExt; use restate_core::{cancellation_watcher, ShutdownError}; -use restate_node_protocol::codec::{Targeted, WireDecode, WireEncode}; +use restate_types::net::codec::{Targeted, WireDecode, WireEncode}; use restate_types::NodeId; use tokio::sync::oneshot; use restate_core::network::{ MessageHandler, MessageRouterBuilder, NetworkSendError, NetworkSender, }; -use restate_node_protocol::{MessageEnvelope, RpcMessage, RpcRequest}; +use restate_types::net::{MessageEnvelope, RpcMessage, RpcRequest}; use tracing::warn; /// A router for sending and receiving RPC messages through Networking @@ -293,7 +293,7 @@ where fn on_message( &self, - msg: restate_node_protocol::MessageEnvelope, + msg: MessageEnvelope, ) -> impl std::future::Future + Send { self.handle_message(msg); std::future::ready(()) @@ -304,7 +304,7 @@ where mod test { use super::*; use futures::future::join_all; - use restate_node_protocol::common::TargetName; + use restate_types::net::{CodecError, TargetName}; use restate_types::GenerationalNodeId; use tokio::sync::Barrier; @@ -334,8 +334,8 @@ mod test { impl WireDecode for TestResponse { fn decode( _: &mut B, - _: restate_node_protocol::common::ProtocolVersion, - ) -> Result + _: restate_types::net::ProtocolVersion, + ) -> Result where Self: Sized, { diff --git a/crates/node-protocol/Cargo.toml b/crates/node-protocol/Cargo.toml deleted file mode 100644 index 93789eb651..0000000000 --- a/crates/node-protocol/Cargo.toml +++ /dev/null @@ -1,33 +0,0 @@ -[package] -name = "restate-node-protocol" -version.workspace = true -authors.workspace = true -edition.workspace = true -rust-version.workspace = true -license.workspace = true -publish = false - -[features] -default = [] - -[dependencies] -restate-types = { workspace = true } -# merge schema creates with restate-types? -restate-schema = { workspace = true } - -bytes = { workspace = true } -bytestring = { workspace = true } -derive_more = { workspace = true } -enum-map = { workspace = true } -flexbuffers = { workspace = true } -prost = { workspace = true } -prost-types = { workspace = true } -serde = { workspace = true} -serde_with = { workspace = true } -strum = { workspace = true } -strum_macros = { workspace = true } -thiserror = { workspace = true } - -[build-dependencies] -prost-build = { workspace = true } - diff --git a/crates/node-protocol/build.rs b/crates/node-protocol/build.rs deleted file mode 100644 index 608beede0c..0000000000 --- a/crates/node-protocol/build.rs +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright (c) 2024 - Restate Software, Inc., Restate GmbH. -// All rights reserved. -// -// Use of this software is governed by the Business Source License -// included in the LICENSE file. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0. - -use std::env; -use std::path::PathBuf; - -fn main() -> Result<(), Box> { - let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap()); - - prost_build::Config::new() - .bytes(["."]) - .file_descriptor_set_path(out_dir.join("common_descriptor.bin")) - // allow older protobuf compiler to be used - .protoc_arg("--experimental_allow_proto3_optional") - .compile_protos(&["./proto/common.proto"], &["proto"])?; - - prost_build::Config::new() - .enum_attribute( - "MessageKind", - "#[derive(::enum_map::Enum, ::strum_macros::EnumIs)]", - ) - .enum_attribute( - "TargetName", - "#[derive(::enum_map::Enum, ::strum_macros::EnumIs, ::strum_macros::Display)]", - ) - .enum_attribute("Message.body", "#[derive(::strum_macros::EnumIs)]") - .bytes(["."]) - .file_descriptor_set_path(out_dir.join("node_descriptor.bin")) - // allow older protobuf compiler to be used - .protoc_arg("--experimental_allow_proto3_optional") - .compile_protos(&["./proto/node.proto"], &["proto"])?; - - Ok(()) -} diff --git a/crates/node-protocol/src/common.rs b/crates/node-protocol/src/common.rs deleted file mode 100644 index d43427b3f4..0000000000 --- a/crates/node-protocol/src/common.rs +++ /dev/null @@ -1,170 +0,0 @@ -// Copyright (c) 2024 - Restate Software, Inc., Restate GmbH. -// All rights reserved. -// -// Use of this software is governed by the Business Source License -// included in the LICENSE file. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0. - -use std::ops::RangeInclusive; -use std::sync::atomic::AtomicUsize; - -use restate_types::identifiers::PartitionKey; - -include!(concat!(env!("OUT_DIR"), "/dev.restate.common.rs")); - -pub static MIN_SUPPORTED_PROTOCOL_VERSION: ProtocolVersion = ProtocolVersion::Flexbuffers; -pub static CURRENT_PROTOCOL_VERSION: ProtocolVersion = ProtocolVersion::Flexbuffers; - -/// Used to identify a request in a RPC-style call going through Networking. -#[derive( - Debug, - derive_more::Display, - PartialEq, - Eq, - Clone, - Copy, - Hash, - PartialOrd, - Ord, - serde::Serialize, - serde::Deserialize, -)] -pub struct RequestId(u64); -impl RequestId { - pub fn new() -> Self { - Default::default() - } -} - -impl Default for RequestId { - fn default() -> Self { - static NEXT_REQUEST_ID: AtomicUsize = AtomicUsize::new(1); - RequestId( - NEXT_REQUEST_ID - .fetch_add(1, std::sync::atomic::Ordering::Relaxed) - .try_into() - .unwrap(), - ) - } -} - -pub const FILE_DESCRIPTOR_SET: &[u8] = - include_bytes!(concat!(env!("OUT_DIR"), "/common_descriptor.bin")); - -impl ProtocolVersion { - pub fn is_supported(&self) -> bool { - *self >= MIN_SUPPORTED_PROTOCOL_VERSION && *self <= CURRENT_PROTOCOL_VERSION - } -} - -impl From for restate_types::Version { - fn from(version: Version) -> Self { - restate_types::Version::from(version.value) - } -} - -impl From for Version { - fn from(version: restate_types::Version) -> Self { - Version { - value: version.into(), - } - } -} - -impl From for restate_types::NodeId { - fn from(node_id: NodeId) -> Self { - restate_types::NodeId::new(node_id.id, node_id.generation) - } -} - -impl From for NodeId { - fn from(node_id: restate_types::NodeId) -> Self { - NodeId { - id: node_id.id().into(), - generation: node_id.as_generational().map(|g| g.generation()), - } - } -} - -impl From for NodeId { - fn from(node_id: restate_types::PlainNodeId) -> Self { - let id: u32 = node_id.into(); - NodeId { - id, - generation: None, - } - } -} - -impl From for NodeId { - fn from(node_id: restate_types::GenerationalNodeId) -> Self { - NodeId { - id: node_id.raw_id(), - generation: Some(node_id.generation()), - } - } -} - -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] -pub struct KeyRange { - pub from: PartitionKey, - pub to: PartitionKey, -} - -impl From for RangeInclusive { - fn from(val: KeyRange) -> Self { - RangeInclusive::new(val.from, val.to) - } -} - -impl From for Lsn { - fn from(lsn: restate_types::logs::Lsn) -> Self { - let value: u64 = lsn.into(); - Lsn { value } - } -} - -impl std::fmt::Display for NodeId { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - if let Some(generation) = self.generation { - write!(f, "N{}:{}", self.id, generation) - } else { - write!(f, "N{}", self.id) - } - } -} - -impl std::fmt::Display for Lsn { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", self.value) - } -} - -impl std::fmt::Display for LeaderEpoch { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "e{}", self.value) - } -} - -impl From for LeaderEpoch { - fn from(epoch: restate_types::identifiers::LeaderEpoch) -> Self { - let value: u64 = epoch.into(); - LeaderEpoch { value } - } -} - -// write tests for RequestId -#[cfg(test)] -mod tests { - use super::*; - #[test] - fn test_request_id() { - let request_id1 = RequestId::new(); - let request_id2 = RequestId::new(); - let request_id3 = RequestId::default(); - assert!(request_id1.0 < request_id2.0 && request_id2.0 < request_id3.0); - } -} diff --git a/crates/node-protocol/src/lib.rs b/crates/node-protocol/src/lib.rs deleted file mode 100644 index 6bb6787f65..0000000000 --- a/crates/node-protocol/src/lib.rs +++ /dev/null @@ -1,181 +0,0 @@ -// Copyright (c) 2024 - Restate Software, Inc., Restate GmbH. -// All rights reserved. -// -// Use of this software is governed by the Business Source License -// included in the LICENSE file. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0. - -pub mod cluster_controller; -pub mod codec; -pub mod common; -mod error; -pub mod ingress; -pub mod metadata; -pub mod node; -pub mod partition_processor_manager; - -// re-exports for convenience -pub use common::CURRENT_PROTOCOL_VERSION; -pub use common::MIN_SUPPORTED_PROTOCOL_VERSION; -pub use error::*; - -use restate_types::GenerationalNodeId; - -use self::codec::Targeted; -use self::codec::WireDecode; - -/// A wrapper for a message that includes the sender id -pub struct MessageEnvelope { - peer: GenerationalNodeId, - connection_id: u64, - body: M, -} - -impl MessageEnvelope { - pub fn new(peer: GenerationalNodeId, connection_id: u64, body: M) -> Self { - Self { - peer, - connection_id, - body, - } - } -} - -impl MessageEnvelope { - pub fn connection_id(&self) -> u64 { - self.connection_id - } - - pub fn split(self) -> (GenerationalNodeId, M) { - (self.peer, self.body) - } - - pub fn body(&self) -> &M { - &self.body - } -} - -impl MessageEnvelope { - /// A unique identifier used by RPC-style messages to correlated requests and responses - pub fn correlation_id(&self) -> M::CorrelationId { - self.body.correlation_id() - } -} - -pub trait RpcMessage { - type CorrelationId: Clone + Send + Eq + PartialEq + std::fmt::Debug + std::hash::Hash; - fn correlation_id(&self) -> Self::CorrelationId; -} - -pub trait RpcRequest: RpcMessage + Targeted { - type Response: RpcMessage + Targeted; -} - -// to define a message, we need -// - Message type -// - message target -// -// Example: -// ``` -// define_message! { -// @message = IngressMessage, -// @target = TargetName::Ingress, -// } -// ``` -macro_rules! define_message { - ( - @message = $message:ty, - @target = $target:expr, - ) => { - impl crate::codec::Targeted for $message { - const TARGET: TargetName = $target; - fn kind(&self) -> &'static str { - stringify!($message) - } - } - - impl crate::codec::WireEncode for $message { - fn encode( - &self, - buf: &mut B, - protocol_version: crate::common::ProtocolVersion, - ) -> Result<(), crate::CodecError> { - // serialize message into buf - crate::codec::encode_default(self, buf, protocol_version) - } - } - - impl crate::codec::WireDecode for $message { - fn decode( - buf: &mut B, - protocol_version: crate::common::ProtocolVersion, - ) -> Result - where - Self: Sized, - { - crate::codec::decode_default(buf, protocol_version) - } - } - }; -} - -// to define an RPC, we need -// - Request type -// - request target -// - Response type -// - response Target -// -// Example: -// ``` -// define_rpc! { -// @request = AttachRequest, -// @response = AttachResponse, -// @request_target = TargetName::ClusterController, -// @response_target = TargetName::AttachResponse, -// } -// ``` -#[allow(unused_macros)] -macro_rules! define_rpc { - ( - @request = $request:ty, - @response = $response:ty, - @request_target = $request_target:expr, - @response_target = $response_target:expr, - ) => { - impl crate::RpcRequest for $request { - type Response = $response; - } - - impl crate::RpcMessage for $request { - type CorrelationId = crate::common::RequestId; - - fn correlation_id(&self) -> Self::CorrelationId { - self.request_id - } - } - - impl crate::RpcMessage for $response { - type CorrelationId = crate::common::RequestId; - - fn correlation_id(&self) -> Self::CorrelationId { - self.request_id - } - } - - crate::define_message! { - @message = $request, - @target = $request_target, - } - - crate::define_message! { - @message = $response, - @target = $response_target, - } - }; -} - -#[allow(unused_imports)] -use {define_message, define_rpc}; diff --git a/crates/node-protocol/src/node.rs b/crates/node-protocol/src/node.rs deleted file mode 100644 index b4eaa2d79e..0000000000 --- a/crates/node-protocol/src/node.rs +++ /dev/null @@ -1,108 +0,0 @@ -// Copyright (c) 2024 - Restate Software, Inc., Restate GmbH. -// All rights reserved. -// -// Use of this software is governed by the Business Source License -// included in the LICENSE file. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0. - -use restate_types::GenerationalNodeId; - -use crate::common::{ProtocolVersion, CURRENT_PROTOCOL_VERSION, MIN_SUPPORTED_PROTOCOL_VERSION}; - -use self::message::{BinaryMessage, ConnectionControl, Signal}; - -include!(concat!(env!("OUT_DIR"), "/dev.restate.node.rs")); - -pub const FILE_DESCRIPTOR_SET: &[u8] = - include_bytes!(concat!(env!("OUT_DIR"), "/node_descriptor.bin")); - -impl Hello { - pub fn new(my_node_id: GenerationalNodeId, cluster_name: String) -> Self { - Self { - min_protocol_version: MIN_SUPPORTED_PROTOCOL_VERSION.into(), - max_protocol_version: CURRENT_PROTOCOL_VERSION.into(), - my_node_id: Some(my_node_id.into()), - cluster_name, - } - } -} - -impl Header { - pub fn new(nodes_config_version: restate_types::Version) -> Self { - Self { - my_nodes_config_version: Some(nodes_config_version.into()), - } - } -} - -impl Welcome { - pub fn new(my_node_id: GenerationalNodeId, protocol_version: ProtocolVersion) -> Self { - Self { - my_node_id: Some(my_node_id.into()), - protocol_version: protocol_version.into(), - } - } -} - -impl Message { - pub fn new(header: Header, body: impl Into) -> Self { - Self { - header: Some(header), - body: Some(body.into()), - } - } -} - -impl From for message::Body { - fn from(value: Hello) -> Self { - message::Body::Hello(value) - } -} - -impl From for message::Body { - fn from(value: Welcome) -> Self { - message::Body::Welcome(value) - } -} - -impl From for message::Body { - fn from(value: ConnectionControl) -> Self { - message::Body::ConnectionControl(value) - } -} - -impl From for message::Body { - fn from(value: BinaryMessage) -> Self { - message::Body::Encoded(value) - } -} - -impl ConnectionControl { - pub fn connection_reset() -> Self { - Self { - signal: message::Signal::DrainConnection.into(), - message: "Connection is draining and will be dropped".to_owned(), - } - } - pub fn shutdown() -> Self { - Self { - signal: message::Signal::Shutdown.into(), - message: "Node is shutting down".to_owned(), - } - } - pub fn codec_error(message: impl Into) -> Self { - Self { - signal: message::Signal::CodecError.into(), - message: message.into(), - } - } -} - -impl std::fmt::Display for Signal { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", self.as_str_name()) - } -} diff --git a/crates/node-services/Cargo.toml b/crates/node-services/Cargo.toml deleted file mode 100644 index bcdf885cb2..0000000000 --- a/crates/node-services/Cargo.toml +++ /dev/null @@ -1,27 +0,0 @@ -[package] -name = "restate-node-services" -version.workspace = true -authors.workspace = true -edition.workspace = true -rust-version.workspace = true -license.workspace = true -publish = false - -[features] -default = [] -clients = [] -servers = [] - -[dependencies] -restate-node-protocol = { workspace = true } -restate-types = { workspace = true } - -anyhow = { workspace = true, optional = true } -bytes = { workspace = true, optional = true } -prost = { workspace = true } -prost-types = { workspace = true } -thiserror = { workspace = true, optional = true } -tonic = { workspace = true, features = ["transport", "codegen", "prost", "gzip"] } - -[build-dependencies] -tonic-build = { workspace = true } diff --git a/crates/node-services/proto/cluster_ctrl_svc.proto b/crates/node-services/proto/cluster_ctrl_svc.proto deleted file mode 100644 index 5bc3e273e0..0000000000 --- a/crates/node-services/proto/cluster_ctrl_svc.proto +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright (c) 2024 - Restate Software, Inc., Restate GmbH -// -// This file is part of the Restate service protocol, which is -// released under the MIT license. -// -// You can find a copy of the license in file LICENSE in the root -// directory of this repository or package, or at -// https://github.com/restatedev/proto/blob/main/LICENSE - -syntax = "proto3"; - -import "common.proto"; -import "google/protobuf/empty.proto"; -import "google/protobuf/duration.proto"; -import "google/protobuf/timestamp.proto"; - -package dev.restate.cluster_ctrl; - -service ClusterCtrlSvc { - rpc GetClusterState(ClusterStateRequest) returns (ClusterStateResponse); - - rpc TrimLog(TrimLogRequest) returns (google.protobuf.Empty); -} - -message ClusterStateRequest {} - -message ClusterStateResponse { - google.protobuf.Duration last_refreshed = 1; - dev.restate.common.Version nodes_config_version = 2; - map nodes = 3; -} - -message NodeState { - oneof state { - AliveNode alive = 1; - DeadNode dead = 2; - } -} - -message AliveNode { - dev.restate.common.NodeId generational_node_id = 1; - google.protobuf.Timestamp last_heartbeat_at = 2; - map partitions = 3; -} - -message DeadNode { google.protobuf.Timestamp last_seen_alive = 1; } - -enum RunMode { - RunMode_UNKNOWN = 0; - LEADER = 1; - FOLLOWER = 2; -} - -enum ReplayStatus { - ReplayStatus_UNKNOWN = 0; - STARTING = 1; - ACTIVE = 2; - CATCHING_UP = 3; -} - -message PartitionProcessorStatus { - google.protobuf.Timestamp updated_at = 1; - RunMode planned_mode = 2; - optional RunMode effective_mode = 3; - optional dev.restate.common.LeaderEpoch last_observed_leader_epoch = 4; - optional dev.restate.common.NodeId last_observed_leader_node = 5; - optional dev.restate.common.Lsn last_applied_log_lsn = 6; - optional google.protobuf.Timestamp last_record_applied_at = 7; - uint64 num_skipped_records = 8; - ReplayStatus replay_status = 9; - optional dev.restate.common.Lsn last_persisted_log_lsn = 10; - // Set if replay_status is CATCHING_UP - optional dev.restate.common.Lsn target_tail_lsn = 11; -} - -message TrimLogRequest { - uint64 log_id = 1; - uint64 trim_point = 2; -} diff --git a/crates/node-services/src/lib.rs b/crates/node-services/src/lib.rs deleted file mode 100644 index 83ab21201b..0000000000 --- a/crates/node-services/src/lib.rs +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright (c) 2023 - Restate Software, Inc., Restate GmbH. -// All rights reserved. -// -// Use of this software is governed by the Business Source License -// included in the LICENSE file. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0. - -pub mod cluster_ctrl { - tonic::include_proto!("dev.restate.cluster_ctrl"); - - pub const FILE_DESCRIPTOR_SET: &[u8] = - tonic::include_file_descriptor_set!("cluster_ctrl_svc_descriptor"); - - impl std::fmt::Display for RunMode { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let o = match self { - RunMode::Unknown => "UNKNOWN", - RunMode::Leader => "Leader", - RunMode::Follower => "Follower", - }; - write!(f, "{}", o) - } - } -} - -pub mod node_svc { - tonic::include_proto!("dev.restate.node_svc"); - - pub const FILE_DESCRIPTOR_SET: &[u8] = - tonic::include_file_descriptor_set!("node_svc_descriptor"); -} diff --git a/crates/node/Cargo.toml b/crates/node/Cargo.toml index 316355968d..7e270a2ef3 100644 --- a/crates/node/Cargo.toml +++ b/crates/node/Cargo.toml @@ -13,23 +13,17 @@ options_schema = [ "dep:schemars", "restate-admin/options_schema", "restate-worker/options_schema", - "restate-cluster-controller/options_schema", "restate-metadata-store/options_schema"] [dependencies] -restate-admin = { workspace = true } +restate-admin = { workspace = true, features = ["servers"] } restate-bifrost = { workspace = true } -restate-cluster-controller = { workspace = true } restate-core = { workspace = true } restate-errors = { workspace = true } restate-grpc-util = { workspace = true } restate-metadata-store = { workspace = true } restate-network = { workspace = true } -restate-node-protocol = { workspace = true } -restate-node-services = { workspace = true, features = ["servers"] } restate-rocksdb = { workspace = true } -restate-schema = { workspace = true } -restate-schema-api = { workspace = true } restate-service-client = { workspace = true } restate-service-protocol = { workspace = true, features = ["discovery"] } restate-storage-query-datafusion = { workspace = true } diff --git a/crates/node/src/lib.rs b/crates/node/src/lib.rs index ace3a1c25a..0a7880e87a 100644 --- a/crates/node/src/lib.rs +++ b/crates/node/src/lib.rs @@ -25,7 +25,7 @@ use tokio::time::Instant; use tracing::{debug, error, info, trace}; use restate_core::metadata_store::{MetadataStoreClientError, ReadWriteError}; -use restate_core::{spawn_metadata_manager, MetadataManager}; +use restate_core::{spawn_metadata_manager, MetadataKind, MetadataManager}; use restate_core::{task_center, TaskKind}; use restate_metadata_store::local::LocalMetadataStoreService; use restate_metadata_store::MetadataStoreClient; @@ -41,7 +41,6 @@ use restate_types::Version; use crate::cluster_marker::ClusterValidationError; use crate::network_server::{AdminDependencies, NetworkServer, WorkerDependencies}; use crate::roles::{AdminRole, WorkerRole}; -use restate_node_protocol::metadata::MetadataKind; #[derive(Debug, thiserror::Error, CodedError)] pub enum Error { diff --git a/crates/node/src/network_server/handler/cluster_ctrl.rs b/crates/node/src/network_server/handler/cluster_ctrl.rs index 3751a7fbd3..70d0375385 100644 --- a/crates/node/src/network_server/handler/cluster_ctrl.rs +++ b/crates/node/src/network_server/handler/cluster_ctrl.rs @@ -8,26 +8,16 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -use std::collections::{BTreeMap, HashMap}; - use tonic::{async_trait, Request, Response, Status}; use tracing::info; -use restate_cluster_controller::ClusterControllerHandle; -use restate_cluster_controller::NodeState; -use restate_metadata_store::MetadataStoreClient; -use restate_node_services::cluster_ctrl::cluster_ctrl_svc_server::ClusterCtrlSvc; -use restate_node_services::cluster_ctrl::node_state; -use restate_node_services::cluster_ctrl::AliveNode; -use restate_node_services::cluster_ctrl::DeadNode; -use restate_node_services::cluster_ctrl::{ +use restate_admin::cluster_controller::protobuf::cluster_ctrl_svc_server::ClusterCtrlSvc; +use restate_admin::cluster_controller::protobuf::{ ClusterStateRequest, ClusterStateResponse, TrimLogRequest, }; -use restate_types::identifiers::PartitionId; +use restate_admin::cluster_controller::ClusterControllerHandle; +use restate_metadata_store::MetadataStoreClient; use restate_types::logs::{LogId, Lsn}; -use restate_types::processors::PartitionProcessorStatus; -use restate_types::processors::RunMode; -use restate_types::PlainNodeId; use crate::network_server::AdminDependencies; @@ -58,11 +48,7 @@ impl ClusterCtrlSvc for ClusterCtrlSvcHandler { .map_err(|_| tonic::Status::aborted("Node is shutting down"))?; let resp = ClusterStateResponse { - last_refreshed: cluster_state - .last_refreshed - .and_then(|r| r.elapsed().try_into().ok()), - nodes_config_version: Some(cluster_state.nodes_config_version.into()), - nodes: to_protobuf_nodes(&cluster_state.nodes), + cluster_state: Some((*cluster_state).clone().into()), }; Ok(Response::new(resp)) } @@ -84,82 +70,3 @@ impl ClusterCtrlSvc for ClusterCtrlSvcHandler { Ok(Response::new(())) } } - -fn to_protobuf_nodes( - nodes: &BTreeMap, -) -> HashMap { - fn to_proto(node: &NodeState) -> restate_node_services::cluster_ctrl::NodeState { - let state = Some(match node { - NodeState::Alive { - last_heartbeat_at, - generation, - partitions, - } => { - let alive_node = AliveNode { - last_heartbeat_at: Some((*last_heartbeat_at).into()), - generational_node_id: Some((*generation).into()), - partitions: to_protobuf_partitions(partitions), - }; - node_state::State::Alive(alive_node) - } - NodeState::Dead { last_seen_alive } => { - let dead_node = DeadNode { - last_seen_alive: last_seen_alive.map(Into::into), - }; - node_state::State::Dead(dead_node) - } - }); - restate_node_services::cluster_ctrl::NodeState { state } - } - - let mut out = HashMap::with_capacity(nodes.len()); - for (id, node) in nodes { - out.insert((*id).into(), to_proto(node)); - } - out -} - -fn to_protobuf_partitions( - pps: &BTreeMap, -) -> HashMap { - fn to_proto( - pp: &PartitionProcessorStatus, - ) -> restate_node_services::cluster_ctrl::PartitionProcessorStatus { - let mut out = restate_node_services::cluster_ctrl::PartitionProcessorStatus::default(); - out.updated_at = Some(pp.updated_at.into()); - out.planned_mode = match pp.planned_mode { - RunMode::Leader => restate_node_services::cluster_ctrl::RunMode::Leader as i32, - RunMode::Follower => restate_node_services::cluster_ctrl::RunMode::Follower as i32, - }; - out.effective_mode = pp.effective_mode.map(|m| match m { - RunMode::Leader => restate_node_services::cluster_ctrl::RunMode::Leader as i32, - RunMode::Follower => restate_node_services::cluster_ctrl::RunMode::Follower as i32, - }); - out.last_observed_leader_epoch = pp.last_observed_leader_epoch.map(|e| e.into()); - out.last_observed_leader_node = pp.last_observed_leader_node.map(|e| e.into()); - out.last_applied_log_lsn = pp.last_applied_log_lsn.map(|l| l.into()); - out.last_record_applied_at = pp.last_record_applied_at.map(Into::into); - out.num_skipped_records = pp.num_skipped_records; - out.replay_status = match pp.replay_status { - restate_types::processors::ReplayStatus::Starting => { - restate_node_services::cluster_ctrl::ReplayStatus::Starting as i32 - } - restate_types::processors::ReplayStatus::Active => { - restate_node_services::cluster_ctrl::ReplayStatus::Active as i32 - } - restate_types::processors::ReplayStatus::CatchingUp { target_tail_lsn } => { - out.target_tail_lsn = Some(target_tail_lsn.into()); - restate_node_services::cluster_ctrl::ReplayStatus::CatchingUp as i32 - } - }; - out.last_persisted_log_lsn = pp.last_persisted_log_lsn.map(|l| l.into()); - - out - } - - let mut out = HashMap::with_capacity(pps.len()); - for (id, node) in pps { - out.insert((*id).into(), to_proto(node)); - } - out -} diff --git a/crates/node/src/network_server/handler/node.rs b/crates/node/src/network_server/handler/node.rs index 3e7e2bd26a..224a14ca0a 100644 --- a/crates/node/src/network_server/handler/node.rs +++ b/crates/node/src/network_server/handler/node.rs @@ -12,17 +12,19 @@ use arrow_flight::encode::FlightDataEncoderBuilder; use arrow_flight::error::FlightError; use futures::stream::BoxStream; use futures::TryStreamExt; -use restate_core::{metadata, TaskCenter}; -use restate_network::error::ProtocolError; use tokio_stream::StreamExt; use tonic::{Request, Response, Status, Streaming}; -use crate::network_server::WorkerDependencies; +use restate_core::{metadata, TaskCenter}; +use restate_network::error::ProtocolError; +use restate_network::protobuf::node_svc::node_svc_server::NodeSvc; +use restate_network::protobuf::node_svc::IdentResponse; +use restate_network::protobuf::node_svc::{StorageQueryRequest, StorageQueryResponse}; use restate_network::ConnectionManager; -use restate_node_protocol::node::Message; -use restate_node_services::node_svc::node_svc_server::NodeSvc; -use restate_node_services::node_svc::{IdentResponse, NodeStatus}; -use restate_node_services::node_svc::{StorageQueryRequest, StorageQueryResponse}; +use restate_types::protobuf::common::NodeStatus; +use restate_types::protobuf::node::Message; + +use crate::network_server::WorkerDependencies; pub struct NodeSvcHandler { task_center: TaskCenter, diff --git a/crates/node/src/network_server/service.rs b/crates/node/src/network_server/service.rs index 3f81f33033..cd3b8800b5 100644 --- a/crates/node/src/network_server/service.rs +++ b/crates/node/src/network_server/service.rs @@ -12,15 +12,13 @@ use axum::routing::get; use tonic::codec::CompressionEncoding; use tower_http::trace::TraceLayer; -use restate_cluster_controller::ClusterControllerHandle; +use restate_admin::cluster_controller::protobuf::cluster_ctrl_svc_server::ClusterCtrlSvcServer; +use restate_admin::cluster_controller::ClusterControllerHandle; use restate_core::{cancellation_watcher, task_center}; use restate_grpc_util::run_hyper_server; use restate_metadata_store::MetadataStoreClient; +use restate_network::protobuf::node_svc::node_svc_server::NodeSvcServer; use restate_network::ConnectionManager; -use restate_node_protocol::{common, node}; -use restate_node_services::cluster_ctrl; -use restate_node_services::cluster_ctrl::cluster_ctrl_svc_server::ClusterCtrlSvcServer; -use restate_node_services::node_svc::node_svc_server::NodeSvcServer; use restate_storage_query_datafusion::context::QueryContext; use restate_types::config::CommonOptions; use restate_worker::SubscriptionControllerHandle; @@ -77,12 +75,16 @@ impl NetworkServer { // -- GRPC Service Setup let mut reflection_service_builder = tonic_reflection::server::Builder::configure() - .register_encoded_file_descriptor_set(node::FILE_DESCRIPTOR_SET) - .register_encoded_file_descriptor_set(common::FILE_DESCRIPTOR_SET); + .register_encoded_file_descriptor_set( + restate_network::protobuf::node_svc::FILE_DESCRIPTOR_SET, + ) + .register_encoded_file_descriptor_set(restate_types::protobuf::FILE_DESCRIPTOR_SET); if self.admin_deps.is_some() { reflection_service_builder = reflection_service_builder - .register_encoded_file_descriptor_set(cluster_ctrl::FILE_DESCRIPTOR_SET); + .register_encoded_file_descriptor_set( + restate_admin::cluster_controller::protobuf::FILE_DESCRIPTOR_SET, + ); } let cluster_controller_service = self.admin_deps.map(|admin_deps| { diff --git a/crates/node/src/roles/admin.rs b/crates/node/src/roles/admin.rs index 5779ccb4fe..33958c2295 100644 --- a/crates/node/src/roles/admin.rs +++ b/crates/node/src/roles/admin.rs @@ -15,12 +15,12 @@ use restate_network::Networking; use std::time::Duration; use tonic::transport::Channel; +use restate_admin::cluster_controller::ClusterControllerHandle; use restate_admin::service::AdminService; use restate_bifrost::Bifrost; -use restate_cluster_controller::ClusterControllerHandle; use restate_core::metadata_store::MetadataStoreClient; use restate_core::{task_center, Metadata, MetadataWriter, TaskCenter, TaskKind}; -use restate_node_services::node_svc::node_svc_client::NodeSvcClient; +use restate_network::protobuf::node_svc::node_svc_client::NodeSvcClient; use restate_service_client::{AssumeRoleCacheMode, ServiceClient}; use restate_service_protocol::discovery::ServiceDiscovery; use restate_types::arc_util::ArcSwapExt; @@ -42,7 +42,7 @@ pub enum AdminRoleBuildError { pub struct AdminRole { updateable_config: UpdateableConfiguration, - controller: restate_cluster_controller::Service, + controller: restate_admin::cluster_controller::Service, admin: AdminService, } @@ -71,7 +71,7 @@ impl AdminRole { service_discovery, ); - let controller = restate_cluster_controller::Service::new( + let controller = restate_admin::cluster_controller::Service::new( updateable_config .clone() .map_as_updateable_owned(|c| &c.admin), diff --git a/crates/node/src/roles/worker.rs b/crates/node/src/roles/worker.rs index d124ab32f4..91af8fceaf 100644 --- a/crates/node/src/roles/worker.rs +++ b/crates/node/src/roles/worker.rs @@ -12,15 +12,14 @@ use codederror::CodedError; use restate_bifrost::Bifrost; use restate_core::network::MessageRouterBuilder; -use restate_core::{cancellation_watcher, metadata, task_center, Metadata}; +use restate_core::{cancellation_watcher, metadata, task_center, Metadata, MetadataKind}; use restate_core::{ShutdownError, TaskKind}; use restate_metadata_store::MetadataStoreClient; use restate_network::Networking; -use restate_node_protocol::metadata::MetadataKind; -use restate_schema::UpdateableSchema; -use restate_schema_api::subscription::SubscriptionResolver; use restate_storage_query_datafusion::context::QueryContext; use restate_types::config::UpdateableConfiguration; +use restate_types::schema::subscriptions::SubscriptionResolver; +use restate_types::schema::UpdateableSchema; use restate_types::Version; use restate_worker::SubscriptionController; use restate_worker::{SubscriptionControllerHandle, Worker}; diff --git a/crates/schema-api/Cargo.toml b/crates/schema-api/Cargo.toml deleted file mode 100644 index 6fbb4fd74b..0000000000 --- a/crates/schema-api/Cargo.toml +++ /dev/null @@ -1,37 +0,0 @@ -[package] -name = "restate-schema-api" -version.workspace = true -authors.workspace = true -edition.workspace = true -rust-version.workspace = true -license.workspace = true -publish = false - -[features] -default = [] - -deployment = ["dep:restate-types", "dep:http", "dep:base64", "dep:restate-base64-util", "dep:bytestring", "service"] -test-util = [] -serde = ["dep:serde", "dep:serde_with", "dep:restate-serde-util"] -serde_schema = ["serde", "dep:schemars", "restate-types?/schemars", "restate-serde-util?/schema"] -service = ["dep:bytes", "dep:restate-types", "dep:humantime"] -invocation_target = ["service", "dep:bytes", "dep:restate-types", "dep:thiserror", "dep:http", "dep:restate-serde-util", "dep:bytestring", "dep:itertools"] -subscription = ["dep:anyhow", "dep:restate-types", "dep:tracing", "dep:thiserror"] - -[dependencies] -restate-base64-util = { workspace = true, optional = true } -restate-serde-util = { workspace = true, optional = true } -restate-types = { workspace = true, optional = true } - -anyhow = { workspace = true, optional = true } -base64 = { workspace = true, optional = true } -bytes = { workspace = true, optional = true } -bytestring = { workspace = true, optional = true } -http = { workspace = true, optional = true } -humantime = { workspace = true, optional = true } -itertools = { workspace = true, optional = true } -schemars = { workspace = true, optional = true } -serde = { workspace = true, optional = true } -serde_with = { workspace = true, optional = true } -thiserror = { workspace = true, optional = true } -tracing = { workspace = true, optional = true } diff --git a/crates/schema-api/src/lib.rs b/crates/schema-api/src/lib.rs deleted file mode 100644 index c73abe7e53..0000000000 --- a/crates/schema-api/src/lib.rs +++ /dev/null @@ -1,728 +0,0 @@ -// Copyright (c) 2023 - Restate Software, Inc., Restate GmbH. -// All rights reserved. -// -// Use of this software is governed by the Business Source License -// included in the LICENSE file. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0. - -//! This crate contains all the different APIs for accessing schemas. - -#[cfg(feature = "invocation_target")] -pub mod invocation_target; - -#[cfg(feature = "deployment")] -pub mod deployment { - use crate::service::ServiceMetadata; - use bytestring::ByteString; - use http::header::{HeaderName, HeaderValue}; - use http::Uri; - use restate_types::identifiers::{DeploymentId, LambdaARN, ServiceRevision}; - use restate_types::time::MillisSinceEpoch; - use std::collections::HashMap; - use std::fmt; - use std::fmt::{Display, Formatter}; - use std::ops::RangeInclusive; - - #[derive(Debug, Copy, Clone, Eq, PartialEq)] - #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] - #[cfg_attr(feature = "serde_schema", derive(schemars::JsonSchema))] - pub enum ProtocolType { - RequestResponse, - BidiStream, - } - - #[derive(Debug, Clone, Default)] - #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] - #[cfg_attr(feature = "serde_schema", derive(schemars::JsonSchema))] - pub struct DeliveryOptions { - #[cfg_attr( - feature = "serde", - serde( - with = "serde_with::As::>" - ) - )] - #[cfg_attr(feature = "serde_schema", schemars(with = "HashMap"))] - pub additional_headers: HashMap, - } - - impl DeliveryOptions { - pub fn new(additional_headers: HashMap) -> Self { - Self { additional_headers } - } - } - - #[derive(Debug, Clone)] - #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] - #[cfg_attr(feature = "serde_schema", derive(schemars::JsonSchema))] - pub struct Deployment { - pub id: DeploymentId, - pub metadata: DeploymentMetadata, - } - - #[derive(Debug, Clone)] - #[cfg_attr(feature = "serde", serde_with::serde_as)] - #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] - #[cfg_attr(feature = "serde_schema", derive(schemars::JsonSchema))] - pub struct DeploymentMetadata { - pub ty: DeploymentType, - pub delivery_options: DeliveryOptions, - pub supported_protocol_versions: RangeInclusive, - pub created_at: MillisSinceEpoch, - } - - #[derive(Debug, Clone)] - #[cfg_attr(feature = "serde", serde_with::serde_as)] - #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] - #[cfg_attr(feature = "serde_schema", derive(schemars::JsonSchema))] - pub enum DeploymentType { - Http { - #[cfg_attr( - feature = "serde", - serde(with = "serde_with::As::") - )] - #[cfg_attr(feature = "serde_schema", schemars(with = "String"))] - address: Uri, - protocol_type: ProtocolType, - }, - Lambda { - arn: LambdaARN, - #[cfg_attr(feature = "serde_schema", schemars(with = "Option"))] - assume_role_arn: Option, - }, - } - - impl DeploymentType { - pub fn protocol_type(&self) -> ProtocolType { - match self { - DeploymentType::Http { protocol_type, .. } => *protocol_type, - DeploymentType::Lambda { .. } => ProtocolType::RequestResponse, - } - } - - pub fn normalized_address(&self) -> String { - match self { - DeploymentType::Http { address, .. } => { - // We use only authority and path, as those uniquely identify the deployment. - format!( - "{}{}", - address.authority().expect("Must have authority"), - address.path() - ) - } - DeploymentType::Lambda { arn, .. } => arn.to_string(), - } - } - } - - impl DeploymentMetadata { - pub fn new_http( - address: Uri, - protocol_type: ProtocolType, - delivery_options: DeliveryOptions, - supported_protocol_versions: RangeInclusive, - ) -> Self { - Self { - ty: DeploymentType::Http { - address, - protocol_type, - }, - delivery_options, - created_at: MillisSinceEpoch::now(), - supported_protocol_versions, - } - } - - pub fn new_lambda( - arn: LambdaARN, - assume_role_arn: Option, - delivery_options: DeliveryOptions, - supported_protocol_versions: RangeInclusive, - ) -> Self { - Self { - ty: DeploymentType::Lambda { - arn, - assume_role_arn, - }, - delivery_options, - created_at: MillisSinceEpoch::now(), - supported_protocol_versions, - } - } - - // address_display returns a Displayable identifier for the endpoint; for http endpoints this is a URI, - // and for Lambda deployments its the ARN - pub fn address_display(&self) -> impl Display + '_ { - struct Wrapper<'a>(&'a DeploymentType); - impl<'a> Display for Wrapper<'a> { - fn fmt(&self, f: &mut Formatter) -> fmt::Result { - match self { - Wrapper(DeploymentType::Http { address, .. }) => address.fmt(f), - Wrapper(DeploymentType::Lambda { arn, .. }) => arn.fmt(f), - } - } - } - Wrapper(&self.ty) - } - - pub fn created_at(&self) -> MillisSinceEpoch { - self.created_at - } - } - - pub trait DeploymentResolver { - fn resolve_latest_deployment_for_service( - &self, - service_name: impl AsRef, - ) -> Option; - - fn get_deployment(&self, deployment_id: &DeploymentId) -> Option; - - fn get_deployment_and_services( - &self, - deployment_id: &DeploymentId, - ) -> Option<(Deployment, Vec)>; - - fn get_deployments(&self) -> Vec<(Deployment, Vec<(String, ServiceRevision)>)>; - } - - #[cfg(feature = "test-util")] - pub mod test_util { - use super::*; - - use restate_types::service_protocol::MAX_SERVICE_PROTOCOL_VERSION_VALUE; - use std::collections::HashMap; - - impl Deployment { - pub fn mock() -> Deployment { - let id = "dp_15VqmTOnXH3Vv2pl5HOG7UB" - .parse() - .expect("valid stable deployment id"); - let metadata = DeploymentMetadata::new_http( - "http://localhost:9080".parse().unwrap(), - ProtocolType::BidiStream, - Default::default(), - 1..=MAX_SERVICE_PROTOCOL_VERSION_VALUE, - ); - - Deployment { id, metadata } - } - - pub fn mock_with_uri(uri: &str) -> Deployment { - let id = DeploymentId::new(); - let metadata = DeploymentMetadata::new_http( - uri.parse().unwrap(), - ProtocolType::BidiStream, - Default::default(), - 1..=MAX_SERVICE_PROTOCOL_VERSION_VALUE, - ); - Deployment { id, metadata } - } - } - - #[derive(Default, Clone, Debug)] - pub struct MockDeploymentMetadataRegistry { - pub deployments: HashMap, - pub latest_deployment: HashMap, - } - - impl MockDeploymentMetadataRegistry { - pub fn mock_service(&mut self, service: &str) { - self.mock_service_with_metadata(service, Deployment::mock()); - } - - pub fn mock_service_with_metadata(&mut self, service: &str, deployment: Deployment) { - self.latest_deployment - .insert(service.to_string(), deployment.id); - self.deployments.insert(deployment.id, deployment.metadata); - } - } - - impl DeploymentResolver for MockDeploymentMetadataRegistry { - fn resolve_latest_deployment_for_service( - &self, - service_name: impl AsRef, - ) -> Option { - self.latest_deployment - .get(service_name.as_ref()) - .and_then(|deployment_id| self.get_deployment(deployment_id)) - } - - fn get_deployment(&self, deployment_id: &DeploymentId) -> Option { - self.deployments - .get(deployment_id) - .cloned() - .map(|metadata| Deployment { - id: *deployment_id, - metadata, - }) - } - - fn get_deployment_and_services( - &self, - deployment_id: &DeploymentId, - ) -> Option<(Deployment, Vec)> { - self.deployments - .get(deployment_id) - .cloned() - .map(|metadata| { - ( - Deployment { - id: *deployment_id, - metadata, - }, - vec![], - ) - }) - } - - fn get_deployments(&self) -> Vec<(Deployment, Vec<(String, ServiceRevision)>)> { - self.deployments - .iter() - .map(|(id, metadata)| { - ( - Deployment { - id: *id, - metadata: metadata.clone(), - }, - vec![], - ) - }) - .collect() - } - } - } -} - -#[cfg(feature = "service")] -pub mod service { - use restate_types::identifiers::{DeploymentId, ServiceRevision}; - use restate_types::invocation::{ - InvocationTargetType, ServiceType, VirtualObjectHandlerType, WorkflowHandlerType, - }; - - #[derive(Debug, Clone)] - #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] - #[cfg_attr(feature = "serde_schema", derive(schemars::JsonSchema))] - pub struct ServiceMetadata { - /// # Name - /// - /// Fully qualified name of the service - pub name: String, - - pub handlers: Vec, - - pub ty: ServiceType, - - /// # Deployment Id - /// - /// Deployment exposing the latest revision of the service. - #[cfg_attr(feature = "serde_schema", schemars(with = "String"))] - pub deployment_id: DeploymentId, - - /// # Revision - /// - /// Latest revision of the service. - pub revision: ServiceRevision, - - /// # Public - /// - /// If true, the service can be invoked through the ingress. - /// If false, the service can be invoked only from another Restate service. - pub public: bool, - - /// # Idempotency retention - /// - /// The retention duration of idempotent requests for this service. - #[cfg_attr( - feature = "serde", - serde(with = "serde_with::As::") - )] - #[cfg_attr(feature = "serde_schema", schemars(with = "String"))] - pub idempotency_retention: humantime::Duration, - - /// # Workflow completion retention - /// - /// The retention duration of workflows. Only available on workflow services. - #[cfg_attr( - feature = "serde", - serde( - with = "serde_with::As::>", - skip_serializing_if = "Option::is_none", - default - ) - )] - #[cfg_attr(feature = "serde_schema", schemars(with = "Option"))] - pub workflow_completion_retention: Option, - } - - // This type is used only for exposing the handler metadata, and not internally. See [ServiceAndHandlerType]. - #[derive(Debug, Clone)] - #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] - #[cfg_attr(feature = "serde_schema", derive(schemars::JsonSchema))] - pub enum HandlerMetadataType { - Exclusive, - Shared, - Workflow, - } - - impl From for HandlerMetadataType { - fn from(value: InvocationTargetType) -> Self { - match value { - InvocationTargetType::Service => HandlerMetadataType::Shared, - InvocationTargetType::VirtualObject(h_ty) => match h_ty { - VirtualObjectHandlerType::Exclusive => HandlerMetadataType::Exclusive, - VirtualObjectHandlerType::Shared => HandlerMetadataType::Shared, - }, - InvocationTargetType::Workflow(h_ty) => match h_ty { - WorkflowHandlerType::Workflow => HandlerMetadataType::Workflow, - WorkflowHandlerType::Shared => HandlerMetadataType::Shared, - }, - } - } - } - - #[derive(Debug, Clone)] - #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] - #[cfg_attr(feature = "serde_schema", derive(schemars::JsonSchema))] - pub struct HandlerMetadata { - pub name: String, - - pub ty: HandlerMetadataType, - - // # Human readable input description - // - // If empty, no schema was provided by the user at discovery time. - pub input_description: String, - - // # Human readable output description - // - // If empty, no schema was provided by the user at discovery time. - pub output_description: String, - } - - /// This API will return services registered by the user. - pub trait ServiceMetadataResolver { - fn resolve_latest_service(&self, service_name: impl AsRef) -> Option; - - fn resolve_latest_service_type(&self, service_name: impl AsRef) - -> Option; - - fn list_services(&self) -> Vec; - } - - #[cfg(feature = "test-util")] - #[allow(dead_code)] - pub mod test_util { - use super::*; - - use std::collections::HashMap; - - #[derive(Debug, Default, Clone)] - pub struct MockServiceMetadataResolver(HashMap); - - impl MockServiceMetadataResolver { - pub fn add(&mut self, service_metadata: ServiceMetadata) { - self.0 - .insert(service_metadata.name.clone(), service_metadata); - } - } - - impl ServiceMetadataResolver for MockServiceMetadataResolver { - fn resolve_latest_service( - &self, - service_name: impl AsRef, - ) -> Option { - self.0.get(service_name.as_ref()).cloned() - } - - fn resolve_latest_service_type( - &self, - service_name: impl AsRef, - ) -> Option { - self.0.get(service_name.as_ref()).map(|c| c.ty) - } - - fn list_services(&self) -> Vec { - self.0.values().cloned().collect() - } - } - - impl ServiceMetadata { - pub fn mock_service( - name: impl AsRef, - handlers: impl IntoIterator>, - ) -> Self { - Self { - name: name.as_ref().to_string(), - handlers: handlers - .into_iter() - .map(|s| HandlerMetadata { - name: s.as_ref().to_string(), - ty: HandlerMetadataType::Shared, - input_description: "any".to_string(), - output_description: "any".to_string(), - }) - .collect(), - ty: ServiceType::Service, - deployment_id: Default::default(), - revision: 0, - public: true, - idempotency_retention: std::time::Duration::from_secs(60).into(), - workflow_completion_retention: None, - } - } - - pub fn mock_virtual_object( - name: impl AsRef, - handlers: impl IntoIterator>, - ) -> Self { - Self { - name: name.as_ref().to_string(), - handlers: handlers - .into_iter() - .map(|s| HandlerMetadata { - name: s.as_ref().to_string(), - ty: HandlerMetadataType::Exclusive, - input_description: "any".to_string(), - output_description: "any".to_string(), - }) - .collect(), - ty: ServiceType::VirtualObject, - deployment_id: Default::default(), - revision: 0, - public: true, - idempotency_retention: std::time::Duration::from_secs(60).into(), - workflow_completion_retention: None, - } - } - } - } -} - -#[cfg(feature = "subscription")] -pub mod subscription { - use restate_types::errors::GenericError; - use std::collections::HashMap; - use std::fmt; - - use restate_types::config::IngressOptions; - use restate_types::identifiers::SubscriptionId; - use tracing::warn; - - #[derive(Debug, Clone, Eq, PartialEq)] - #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] - #[cfg_attr(feature = "serde_schema", derive(schemars::JsonSchema))] - pub enum Source { - Kafka { cluster: String, topic: String }, - } - - impl fmt::Display for Source { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Source::Kafka { cluster, topic, .. } => { - write!(f, "kafka://{}/{}", cluster, topic) - } - } - } - } - - impl PartialEq<&str> for Source { - fn eq(&self, other: &&str) -> bool { - self.to_string().as_str() == *other - } - } - - /// Specialized version of [super::service::ServiceType] - #[derive(Debug, Clone, Eq, PartialEq)] - #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] - #[cfg_attr(feature = "serde_schema", derive(schemars::JsonSchema))] - pub enum EventReceiverServiceType { - VirtualObject, - Workflow, - Service, - } - - #[derive(Debug, Clone, Eq, PartialEq)] - #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] - #[cfg_attr(feature = "serde_schema", derive(schemars::JsonSchema))] - pub enum Sink { - Service { - name: String, - handler: String, - ty: EventReceiverServiceType, - }, - } - - impl fmt::Display for Sink { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Sink::Service { name, handler, .. } => { - write!(f, "service://{}/{}", name, handler) - } - } - } - } - - impl PartialEq<&str> for Sink { - fn eq(&self, other: &&str) -> bool { - self.to_string().as_str() == *other - } - } - - #[derive(Debug, Clone, Eq, PartialEq)] - #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] - #[cfg_attr(feature = "serde_schema", derive(schemars::JsonSchema))] - pub struct Subscription { - id: SubscriptionId, - source: Source, - sink: Sink, - metadata: HashMap, - } - - impl Subscription { - pub fn new( - id: SubscriptionId, - source: Source, - sink: Sink, - metadata: HashMap, - ) -> Self { - Self { - id, - source, - sink, - metadata, - } - } - - pub fn id(&self) -> SubscriptionId { - self.id - } - - pub fn source(&self) -> &Source { - &self.source - } - - pub fn sink(&self) -> &Sink { - &self.sink - } - - pub fn metadata(&self) -> &HashMap { - &self.metadata - } - - pub fn metadata_mut(&mut self) -> &mut HashMap { - &mut self.metadata - } - } - - pub enum ListSubscriptionFilter { - ExactMatchSink(String), - ExactMatchSource(String), - } - - impl ListSubscriptionFilter { - pub fn matches(&self, sub: &Subscription) -> bool { - match self { - ListSubscriptionFilter::ExactMatchSink(sink) => sub.sink == sink.as_str(), - ListSubscriptionFilter::ExactMatchSource(source) => sub.source == source.as_str(), - } - } - } - - pub trait SubscriptionResolver { - fn get_subscription(&self, id: SubscriptionId) -> Option; - - fn list_subscriptions(&self, filters: &[ListSubscriptionFilter]) -> Vec; - } - - pub trait SubscriptionValidator { - type Error: Into; - - fn validate(&self, subscription: Subscription) -> Result; - } - - #[derive(Debug, thiserror::Error)] - #[error("invalid option '{name}'. Reason: {reason}")] - pub struct ValidationError { - name: &'static str, - reason: &'static str, - } - - impl SubscriptionValidator for IngressOptions { - type Error = ValidationError; - - fn validate(&self, mut subscription: Subscription) -> Result { - // Retrieve the cluster option and merge them with subscription metadata - let Source::Kafka { cluster, .. } = subscription.source(); - let cluster_options = &self.get_kafka_cluster(cluster).ok_or(ValidationError { - name: "source", - reason: "specified cluster in the source URI does not exist. Make sure it is defined in the KafkaOptions", - })?.additional_options; - - if cluster_options.contains_key("enable.auto.commit") - || subscription.metadata().contains_key("enable.auto.commit") - { - warn!("The configuration option enable.auto.commit should not be set and it will be ignored."); - } - if cluster_options.contains_key("enable.auto.offset.store") - || subscription - .metadata() - .contains_key("enable.auto.offset.store") - { - warn!("The configuration option enable.auto.offset.store should not be set and it will be ignored."); - } - - // Set the group.id if unset - if !(cluster_options.contains_key("group.id") - || subscription.metadata().contains_key("group.id")) - { - let group_id = subscription.id().to_string(); - - subscription - .metadata_mut() - .insert("group.id".to_string(), group_id); - } - - // Set client.id if unset - if !(cluster_options.contains_key("client.id") - || subscription.metadata().contains_key("client.id")) - { - subscription - .metadata_mut() - .insert("client.id".to_string(), "restate".to_string()); - } - - Ok(subscription) - } - } - - #[cfg(feature = "mocks")] - pub mod mocks { - use std::str::FromStr; - - use super::*; - - impl Subscription { - pub fn mock() -> Self { - let id = SubscriptionId::from_str("sub_15VqmTOnXH3Vv2pl5HOG7Ua") - .expect("stable valid subscription id"); - Subscription { - id, - source: Source::Kafka { - cluster: "my-cluster".to_string(), - topic: "my-topic".to_string(), - }, - sink: Sink::Service { - name: "MySvc".to_string(), - handler: "MyMethod".to_string(), - ty: EventReceiverServiceType::Service, - }, - metadata: Default::default(), - } - } - } - } -} diff --git a/crates/schema/Cargo.toml b/crates/schema/Cargo.toml deleted file mode 100644 index d5ac3b9b99..0000000000 --- a/crates/schema/Cargo.toml +++ /dev/null @@ -1,28 +0,0 @@ -[package] -name = "restate-schema" -version.workspace = true -authors.workspace = true -edition.workspace = true -rust-version.workspace = true -license.workspace = true -publish = false - -[features] -test-util = ["dep:restate-test-util"] - -[dependencies] -restate-schema-api = { workspace = true, features = ["service", "deployment", "subscription", "invocation_target", "serde"] } -restate-types = { workspace = true } -restate-test-util = { workspace = true, optional = true } - -arc-swap = { workspace = true } -bytes = { workspace = true } -derive_more = { workspace = true } -flexbuffers = { workspace = true } -http = { workspace = true } -serde = { workspace = true } -serde_with = { workspace = true } -strum_macros = { workspace = true } - -[dev-dependencies] - diff --git a/crates/schema/src/deployment.rs b/crates/schema/src/deployment.rs deleted file mode 100644 index fd60e941a6..0000000000 --- a/crates/schema/src/deployment.rs +++ /dev/null @@ -1,107 +0,0 @@ -// Copyright (c) 2023 - Restate Software, Inc., Restate GmbH. -// All rights reserved. -// -// Use of this software is governed by the Business Source License -// included in the LICENSE file. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0. - -use super::{Schema, UpdateableSchema}; -use restate_schema_api::deployment::{Deployment, DeploymentMetadata, DeploymentResolver}; -use restate_schema_api::service::ServiceMetadata; -use restate_types::identifiers::{DeploymentId, ServiceRevision}; - -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] -pub struct DeploymentSchemas { - pub metadata: DeploymentMetadata, - - // We need to store ServiceMetadata here only for queries - // We could optimize the memory impact of this by reading these info from disk - pub services: Vec, -} - -impl DeploymentResolver for Schema { - fn resolve_latest_deployment_for_service( - &self, - service_name: impl AsRef, - ) -> Option { - let service = self.services.get(service_name.as_ref())?; - self.deployments - .get(&service.location.latest_deployment) - .map(|schemas| Deployment { - id: service.location.latest_deployment, - metadata: schemas.metadata.clone(), - }) - } - - fn get_deployment(&self, deployment_id: &DeploymentId) -> Option { - self.deployments - .get(deployment_id) - .map(|schemas| Deployment { - id: *deployment_id, - metadata: schemas.metadata.clone(), - }) - } - - fn get_deployment_and_services( - &self, - deployment_id: &DeploymentId, - ) -> Option<(Deployment, Vec)> { - self.deployments.get(deployment_id).map(|schemas| { - ( - Deployment { - id: *deployment_id, - metadata: schemas.metadata.clone(), - }, - schemas.services.clone(), - ) - }) - } - - fn get_deployments(&self) -> Vec<(Deployment, Vec<(String, ServiceRevision)>)> { - self.deployments - .iter() - .map(|(deployment_id, schemas)| { - ( - Deployment { - id: *deployment_id, - metadata: schemas.metadata.clone(), - }, - schemas - .services - .iter() - .map(|s| (s.name.clone(), s.revision)) - .collect(), - ) - }) - .collect() - } -} - -impl DeploymentResolver for UpdateableSchema { - fn resolve_latest_deployment_for_service( - &self, - service_name: impl AsRef, - ) -> Option { - self.0 - .load() - .resolve_latest_deployment_for_service(service_name) - } - - fn get_deployment(&self, deployment_id: &DeploymentId) -> Option { - self.0.load().get_deployment(deployment_id) - } - - fn get_deployment_and_services( - &self, - deployment_id: &DeploymentId, - ) -> Option<(Deployment, Vec)> { - self.0.load().get_deployment_and_services(deployment_id) - } - - fn get_deployments(&self) -> Vec<(Deployment, Vec<(String, ServiceRevision)>)> { - self.0.load().get_deployments() - } -} diff --git a/crates/schema/src/invocation_target.rs b/crates/schema/src/invocation_target.rs deleted file mode 100644 index 4aa01b3b58..0000000000 --- a/crates/schema/src/invocation_target.rs +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright (c) 2024 - Restate Software, Inc., Restate GmbH. -// All rights reserved. -// -// Use of this software is governed by the Business Source License -// included in the LICENSE file. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0. - -use super::*; - -use restate_schema_api::invocation_target::{InvocationTargetMetadata, InvocationTargetResolver}; - -impl InvocationTargetResolver for Schema { - fn resolve_latest_invocation_target( - &self, - service_name: impl AsRef, - handler_name: impl AsRef, - ) -> Option { - self.use_service_schema(service_name.as_ref(), |service_schemas| { - service_schemas - .handlers - .get(handler_name.as_ref()) - .map(|handler_schemas| handler_schemas.target_meta.clone()) - }) - .flatten() - } -} - -impl InvocationTargetResolver for UpdateableSchema { - fn resolve_latest_invocation_target( - &self, - service_name: impl AsRef, - handler_name: impl AsRef, - ) -> Option { - self.0 - .load() - .resolve_latest_invocation_target(service_name, handler_name) - } -} diff --git a/crates/schema/src/service.rs b/crates/schema/src/service.rs deleted file mode 100644 index 86ee6ca5d6..0000000000 --- a/crates/schema/src/service.rs +++ /dev/null @@ -1,97 +0,0 @@ -// Copyright (c) 2023 - Restate Software, Inc., Restate GmbH. -// All rights reserved. -// -// Use of this software is governed by the Business Source License -// included in the LICENSE file. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0. - -use super::*; -use std::time::Duration; - -use restate_schema_api::invocation_target::InvocationTargetMetadata; -use restate_schema_api::service::ServiceMetadataResolver; -use restate_types::invocation::ServiceType; - -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] -pub struct HandlerSchemas { - pub target_meta: InvocationTargetMetadata, -} - -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] -pub struct ServiceSchemas { - pub revision: ServiceRevision, - pub handlers: HashMap, - pub ty: ServiceType, - pub location: ServiceLocation, - pub idempotency_retention: Duration, - pub workflow_completion_retention: Option, -} - -impl ServiceSchemas { - pub fn as_service_metadata(&self, name: String) -> ServiceMetadata { - ServiceMetadata { - name, - handlers: self - .handlers - .iter() - .map(|(h_name, h_schemas)| HandlerMetadata { - name: h_name.clone(), - ty: h_schemas.target_meta.target_ty.into(), - input_description: h_schemas.target_meta.input_rules.to_string(), - output_description: h_schemas.target_meta.output_rules.to_string(), - }) - .collect(), - ty: self.ty, - deployment_id: self.location.latest_deployment, - revision: self.revision, - public: self.location.public, - idempotency_retention: self.idempotency_retention.into(), - workflow_completion_retention: self.workflow_completion_retention.map(Into::into), - } - } -} - -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] -pub struct ServiceLocation { - pub latest_deployment: DeploymentId, - pub public: bool, -} - -impl ServiceMetadataResolver for Schema { - fn resolve_latest_service(&self, service_name: impl AsRef) -> Option { - let name = service_name.as_ref(); - self.use_service_schema(name, |service_schemas| { - service_schemas.as_service_metadata(name.to_owned()) - }) - } - - fn resolve_latest_service_type(&self, service_name: impl AsRef) -> Option { - self.use_service_schema(service_name.as_ref(), |service_schemas| service_schemas.ty) - } - - fn list_services(&self) -> Vec { - self.services - .iter() - .map(|(service_name, service_schemas)| { - service_schemas.as_service_metadata(service_name.clone()) - }) - .collect() - } -} - -impl ServiceMetadataResolver for UpdateableSchema { - fn resolve_latest_service(&self, service_name: impl AsRef) -> Option { - self.0.load().resolve_latest_service(service_name) - } - - fn resolve_latest_service_type(&self, service_name: impl AsRef) -> Option { - self.0.load().resolve_latest_service_type(service_name) - } - - fn list_services(&self) -> Vec { - self.0.load().list_services() - } -} diff --git a/crates/schema/src/subscriptions.rs b/crates/schema/src/subscriptions.rs deleted file mode 100644 index a588bb9fc4..0000000000 --- a/crates/schema/src/subscriptions.rs +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright (c) 2023 - Restate Software, Inc., Restate GmbH. -// All rights reserved. -// -// Use of this software is governed by the Business Source License -// included in the LICENSE file. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0. - -use super::{Schema, UpdateableSchema}; - -use restate_schema_api::subscription::{ - ListSubscriptionFilter, Subscription, SubscriptionResolver, -}; -use restate_types::identifiers::SubscriptionId; - -impl SubscriptionResolver for Schema { - fn get_subscription(&self, id: SubscriptionId) -> Option { - self.subscriptions.get(&id).cloned() - } - - fn list_subscriptions(&self, filters: &[ListSubscriptionFilter]) -> Vec { - self.subscriptions - .values() - .filter(|sub| { - for f in filters { - if !f.matches(sub) { - return false; - } - } - true - }) - .cloned() - .collect() - } -} - -impl SubscriptionResolver for UpdateableSchema { - fn get_subscription(&self, id: SubscriptionId) -> Option { - self.0.load().get_subscription(id) - } - - fn list_subscriptions(&self, filters: &[ListSubscriptionFilter]) -> Vec { - self.0.load().list_subscriptions(filters) - } -} diff --git a/crates/service-protocol/Cargo.toml b/crates/service-protocol/Cargo.toml index a80f8179ff..1c1cb8dd22 100644 --- a/crates/service-protocol/Cargo.toml +++ b/crates/service-protocol/Cargo.toml @@ -12,14 +12,13 @@ default = [] awakeable-id = ["dep:base64", "dep:restate-base64-util", "dep:restate-types"] codec = ["dep:restate-types", "dep:paste"] -discovery = ["dep:serde", "dep:serde_json", "dep:regress", "dep:tracing", "dep:codederror", "dep:restate-errors", "dep:restate-schema-api", "dep:hyper", "dep:restate-service-client", "dep:restate-types", "dep:tokio"] +discovery = ["dep:serde", "dep:serde_json", "dep:regress", "dep:tracing", "dep:codederror", "dep:restate-errors", "dep:hyper", "dep:restate-service-client", "dep:restate-types", "dep:tokio"] message = ["dep:restate-types", "dep:bytes-utils", "dep:codederror", "dep:restate-errors", "dep:size", "dep:tracing"] test-util = ["awakeable-id"] [dependencies] restate-base64-util = { workspace = true, optional = true } restate-errors = { workspace = true, optional = true } -restate-schema-api = { workspace = true, optional = true, features = ["deployment"] } restate-service-client = { workspace = true, optional = true } restate-types = { workspace = true, optional = true } diff --git a/crates/service-protocol/src/discovery.rs b/crates/service-protocol/src/discovery.rs index b8e2cf887e..3eddd87bab 100644 --- a/crates/service-protocol/src/discovery.rs +++ b/crates/service-protocol/src/discovery.rs @@ -18,10 +18,10 @@ use hyper::{Body, HeaderMap, StatusCode}; use itertools::Itertools; use once_cell::sync::Lazy; use restate_errors::{META0003, META0012, META0013}; -use restate_schema_api::deployment::ProtocolType; use restate_service_client::{Endpoint, Method, Parts, Request, ServiceClient, ServiceClientError}; use restate_types::endpoint_manifest; use restate_types::retries::{RetryIter, RetryPolicy}; +use restate_types::schema::deployment::ProtocolType; use restate_types::service_discovery::{ ServiceDiscoveryProtocolVersion, MAX_SERVICE_DISCOVERY_PROTOCOL_VERSION, MIN_SERVICE_DISCOVERY_PROTOCOL_VERSION, diff --git a/crates/storage-query-datafusion/Cargo.toml b/crates/storage-query-datafusion/Cargo.toml index a4f363757f..7dc26a0806 100644 --- a/crates/storage-query-datafusion/Cargo.toml +++ b/crates/storage-query-datafusion/Cargo.toml @@ -16,7 +16,6 @@ table_docs = [] restate-core = { workspace = true } restate-invoker-api = { workspace = true } restate-partition-store = { workspace = true } -restate-schema-api = { workspace = true, features = ["deployment"] } restate-service-protocol = { workspace = true, features = ["codec"] } restate-storage-api = { workspace = true } restate-types = { workspace = true } @@ -44,7 +43,6 @@ tracing = { workspace = true } restate-core = { workspace = true, features = ["test-util"] } restate-invoker-api = { workspace = true, features = ["test-util"] } restate-rocksdb = { workspace = true, features = ["test-util"] } -restate-schema-api = { workspace = true, features = ["test-util"] } restate-storage-api = { workspace = true, features = ["test-util"] } restate-service-protocol = { workspace = true, features = ["test-util"] } restate-types = { workspace = true, features = ["test-util"] } diff --git a/crates/storage-query-datafusion/src/context.rs b/crates/storage-query-datafusion/src/context.rs index b360292b09..adb0f43ecb 100644 --- a/crates/storage-query-datafusion/src/context.rs +++ b/crates/storage-query-datafusion/src/context.rs @@ -23,11 +23,11 @@ use datafusion::prelude::{SessionConfig, SessionContext}; use restate_core::worker_api::ProcessorsManagerHandle; use restate_invoker_api::StatusHandle; use restate_partition_store::PartitionStoreManager; -use restate_schema_api::deployment::DeploymentResolver; -use restate_schema_api::service::ServiceMetadataResolver; use restate_types::config::QueryEngineOptions; use restate_types::errors::GenericError; use restate_types::identifiers::PartitionId; +use restate_types::schema::deployment::DeploymentResolver; +use restate_types::schema::service::ServiceMetadataResolver; use crate::{analyzer, physical_optimizer}; diff --git a/crates/storage-query-datafusion/src/deployment/row.rs b/crates/storage-query-datafusion/src/deployment/row.rs index 623ec5226f..2be71ac62e 100644 --- a/crates/storage-query-datafusion/src/deployment/row.rs +++ b/crates/storage-query-datafusion/src/deployment/row.rs @@ -10,7 +10,7 @@ use super::schema::SysDeploymentBuilder; use crate::table_util::format_using; -use restate_schema_api::deployment::{Deployment, DeploymentType}; +use restate_types::schema::deployment::{Deployment, DeploymentType}; #[inline] pub(crate) fn append_deployment_row( diff --git a/crates/storage-query-datafusion/src/deployment/table.rs b/crates/storage-query-datafusion/src/deployment/table.rs index f66011d7fc..5ac3fcf043 100644 --- a/crates/storage-query-datafusion/src/deployment/table.rs +++ b/crates/storage-query-datafusion/src/deployment/table.rs @@ -18,8 +18,8 @@ use datafusion::physical_plan::stream::RecordBatchReceiverStream; use datafusion::physical_plan::SendableRecordBatchStream; use tokio::sync::mpsc::Sender; -use restate_schema_api::deployment::{Deployment, DeploymentResolver}; use restate_types::identifiers::ServiceRevision; +use restate_types::schema::deployment::{Deployment, DeploymentResolver}; use super::schema::SysDeploymentBuilder; use crate::context::QueryContext; diff --git a/crates/storage-query-datafusion/src/mocks.rs b/crates/storage-query-datafusion/src/mocks.rs index 65d22ea682..b5e197dc35 100644 --- a/crates/storage-query-datafusion/src/mocks.rs +++ b/crates/storage-query-datafusion/src/mocks.rs @@ -8,32 +8,33 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -use crate::context::SelectPartitions; - -use super::context::QueryContext; +use std::fmt::Debug; +use std::marker::PhantomData; +use std::ops::RangeInclusive; use async_trait::async_trait; use datafusion::arrow::array::ArrayRef; use datafusion::arrow::record_batch::RecordBatch; use datafusion::execution::SendableRecordBatchStream; use googletest::matcher::{Matcher, MatcherResult}; + use restate_core::task_center; use restate_invoker_api::status_handle::test_util::MockStatusHandle; use restate_invoker_api::StatusHandle; use restate_partition_store::{OpenMode, PartitionStore, PartitionStoreManager}; use restate_rocksdb::RocksDbManager; -use restate_schema_api::deployment::test_util::MockDeploymentMetadataRegistry; -use restate_schema_api::deployment::{Deployment, DeploymentResolver}; -use restate_schema_api::service::test_util::MockServiceMetadataResolver; -use restate_schema_api::service::{ServiceMetadata, ServiceMetadataResolver}; use restate_types::arc_util::Constant; use restate_types::config::{CommonOptions, QueryEngineOptions, WorkerOptions}; use restate_types::errors::GenericError; use restate_types::identifiers::{DeploymentId, PartitionId, PartitionKey, ServiceRevision}; use restate_types::invocation::ServiceType; -use std::fmt::Debug; -use std::marker::PhantomData; -use std::ops::RangeInclusive; +use restate_types::schema::deployment::test_util::MockDeploymentMetadataRegistry; +use restate_types::schema::deployment::{Deployment, DeploymentResolver}; +use restate_types::schema::service::test_util::MockServiceMetadataResolver; +use restate_types::schema::service::{ServiceMetadata, ServiceMetadataResolver}; + +use super::context::QueryContext; +use crate::context::SelectPartitions; #[derive(Default, Clone, Debug)] pub(crate) struct MockSchemas( diff --git a/crates/storage-query-datafusion/src/service/row.rs b/crates/storage-query-datafusion/src/service/row.rs index 08d4928ba3..4961448e24 100644 --- a/crates/storage-query-datafusion/src/service/row.rs +++ b/crates/storage-query-datafusion/src/service/row.rs @@ -11,8 +11,8 @@ use super::schema::SysServiceBuilder; use crate::table_util::format_using; -use restate_schema_api::service::ServiceMetadata; use restate_types::invocation::ServiceType; +use restate_types::schema::service::ServiceMetadata; #[inline] pub(crate) fn append_service_row( diff --git a/crates/storage-query-datafusion/src/service/table.rs b/crates/storage-query-datafusion/src/service/table.rs index 3364ac436b..59dd934ca1 100644 --- a/crates/storage-query-datafusion/src/service/table.rs +++ b/crates/storage-query-datafusion/src/service/table.rs @@ -18,7 +18,7 @@ use datafusion::physical_plan::stream::RecordBatchReceiverStream; use datafusion::physical_plan::SendableRecordBatchStream; use tokio::sync::mpsc::Sender; -use restate_schema_api::service::{ServiceMetadata, ServiceMetadataResolver}; +use restate_types::schema::service::{ServiceMetadata, ServiceMetadataResolver}; use super::schema::SysServiceBuilder; use crate::context::QueryContext; diff --git a/crates/types/Cargo.toml b/crates/types/Cargo.toml index 5d17317955..f979c42e24 100644 --- a/crates/types/Cargo.toml +++ b/crates/types/Cargo.toml @@ -11,11 +11,12 @@ publish = false default = [] schemars = ["dep:schemars", "restate-serde-util/schema"] -test-util = ["dep:tempfile"] +test-util = ["dep:tempfile", "dep:restate-test-util"] [dependencies] restate-base64-util = { workspace = true } restate-serde-util = { workspace = true } +restate-test-util = { workspace = true, optional = true } anyhow = { workspace = true } arc-swap = { workspace = true } @@ -32,10 +33,12 @@ flexbuffers = { workspace = true } hostname = { workspace = true } http = { workspace = true } humantime = { workspace = true } +itertools = { workspace = true } num-traits = { version = "0.2.17" } once_cell = { workspace = true } opentelemetry = { workspace = true } prost = { workspace = true } +prost-dto = { workspace = true } prost-types = { workspace = true } rand = { workspace = true } regress = { version = "0.9" } diff --git a/crates/types/build.rs b/crates/types/build.rs index d7d60869ff..3a282c9d4a 100644 --- a/crates/types/build.rs +++ b/crates/types/build.rs @@ -11,10 +11,12 @@ use jsonptr::Pointer; use std::env; use std::fs::File; -use std::path::Path; +use std::path::{Path, PathBuf}; use typify::{TypeSpace, TypeSpaceSettings}; fn main() -> std::io::Result<()> { + let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap()); + prost_build::Config::new() .bytes(["."]) .protoc_arg("--experimental_allow_proto3_optional") @@ -34,6 +36,9 @@ fn main() -> std::io::Result<()> { &["service-protocol"], )?; + // Common proto types for internal use + build_restate_proto(&out_dir)?; + let mut parsed_content: serde_json::Value = serde_json::from_reader( File::open("./service-protocol/endpoint_manifest_schema.json").unwrap(), ) @@ -75,7 +80,31 @@ fn main() -> std::io::Result<()> { prettyplease::unparse(&syn::parse2::(type_space.to_stream()).unwrap()) ); - let mut out_file = Path::new(&env::var("OUT_DIR").unwrap()).to_path_buf(); - out_file.push("endpoint_manifest.rs"); - std::fs::write(out_file, contents) + std::fs::write(out_dir.join("endpoint_manifest.rs"), contents) +} + +fn build_restate_proto(out_dir: &Path) -> std::io::Result<()> { + prost_build::Config::new() + .bytes(["."]) + .enum_attribute( + "TargetName", + "#[derive(::enum_map::Enum, ::strum_macros::EnumIs, ::strum_macros::Display)]", + ) + .enum_attribute("Message.body", "#[derive(::strum_macros::EnumIs)]") + .btree_map([ + ".restate.cluster.ClusterState", + ".restate.cluster.AliveNode", + ]) + .file_descriptor_set_path(out_dir.join("common_descriptor.bin")) + // allow older protobuf compiler to be used + .protoc_arg("--experimental_allow_proto3_optional") + .compile_protos( + &[ + "./protobuf/restate/common.proto", + "./protobuf/restate/cluster.proto", + "./protobuf/restate/node.proto", + ], + &["protobuf"], + )?; + Ok(()) } diff --git a/crates/types/protobuf/restate/cluster.proto b/crates/types/protobuf/restate/cluster.proto new file mode 100644 index 0000000000..ab8764acb0 --- /dev/null +++ b/crates/types/protobuf/restate/cluster.proto @@ -0,0 +1,68 @@ +// Copyright (c) 2024 - Restate Software, Inc., Restate GmbH. +// All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +syntax = "proto3"; + +import "restate/common.proto"; +import "google/protobuf/empty.proto"; +import "google/protobuf/duration.proto"; +import "google/protobuf/timestamp.proto"; + +package restate.cluster; + +message ClusterState { + google.protobuf.Duration last_refreshed = 1; + restate.common.Version nodes_config_version = 2; + restate.common.Version partition_table_version = 3; + map nodes = 4; +} + +message NodeState { + oneof state { + AliveNode alive = 1; + DeadNode dead = 2; + } +} + +message AliveNode { + restate.common.NodeId generational_node_id = 1; + google.protobuf.Timestamp last_heartbeat_at = 2; + map partitions = 3; +} + +message DeadNode { google.protobuf.Timestamp last_seen_alive = 1; } + +enum RunMode { + RunMode_UNKNOWN = 0; + LEADER = 1; + FOLLOWER = 2; +} + +enum ReplayStatus { + ReplayStatus_UNKNOWN = 0; + STARTING = 1; + ACTIVE = 2; + CATCHING_UP = 3; +} + +message PartitionProcessorStatus { + google.protobuf.Timestamp updated_at = 1; + RunMode planned_mode = 2; + optional RunMode effective_mode = 3; + optional restate.common.LeaderEpoch last_observed_leader_epoch = 4; + optional restate.common.NodeId last_observed_leader_node = 5; + optional restate.common.Lsn last_applied_log_lsn = 6; + optional google.protobuf.Timestamp last_record_applied_at = 7; + uint64 num_skipped_records = 8; + ReplayStatus replay_status = 9; + optional restate.common.Lsn last_persisted_log_lsn = 10; + // Set if replay_status is CATCHING_UP + optional restate.common.Lsn target_tail_lsn = 11; +} diff --git a/crates/node-protocol/proto/common.proto b/crates/types/protobuf/restate/common.proto similarity index 84% rename from crates/node-protocol/proto/common.proto rename to crates/types/protobuf/restate/common.proto index 589f4ce4c7..769e0e1209 100644 --- a/crates/node-protocol/proto/common.proto +++ b/crates/types/protobuf/restate/common.proto @@ -9,7 +9,7 @@ syntax = "proto3"; -package dev.restate.common; +package restate.common; enum ProtocolVersion { ProtocolVersion_UNKNOWN = 0; @@ -43,3 +43,12 @@ enum TargetName { GET_PROCESSORS_STATE_REQUEST = 7; PROCESSORS_STATE_RESPONSE = 8; } + +enum NodeStatus { + NodeStatus_UNKNOWN = 0; + ALIVE = 1; + // The node is not fully running yet. + STARTING_UP = 2; + // The node is performing a graceful shutdown. + SHUTTING_DOWN = 3; +} diff --git a/crates/node-protocol/proto/node.proto b/crates/types/protobuf/restate/node.proto similarity index 74% rename from crates/node-protocol/proto/node.proto rename to crates/types/protobuf/restate/node.proto index 9514653c0c..5b960143c4 100644 --- a/crates/node-protocol/proto/node.proto +++ b/crates/types/protobuf/restate/node.proto @@ -9,30 +9,30 @@ syntax = "proto3"; -import "common.proto"; +import "restate/common.proto"; -package dev.restate.node; +package restate.node; // // # Wire Protocol Of Streaming Connections // ------------------------------------- // -message Header { dev.restate.common.Version my_nodes_config_version = 1; } +message Header { restate.common.Version my_nodes_config_version = 1; } // First message sent to an ingress after starting the connection. The message // must be sent before any other message. message Hello { - dev.restate.common.ProtocolVersion min_protocol_version = 1; - dev.restate.common.ProtocolVersion max_protocol_version = 2; + restate.common.ProtocolVersion min_protocol_version = 1; + restate.common.ProtocolVersion max_protocol_version = 2; // generational node id of sender (who am I) - dev.restate.common.NodeId my_node_id = 3; + restate.common.NodeId my_node_id = 3; string cluster_name = 4; } message Welcome { - dev.restate.common.ProtocolVersion protocol_version = 2; + restate.common.ProtocolVersion protocol_version = 2; // generational node id of sender - dev.restate.common.NodeId my_node_id = 3; + restate.common.NodeId my_node_id = 3; } // Bidirectional Communication @@ -50,7 +50,7 @@ message Message { } message BinaryMessage { - dev.restate.common.TargetName target = 1; + restate.common.TargetName target = 1; bytes payload = 2; } diff --git a/crates/types/src/cluster/cluster_state.rs b/crates/types/src/cluster/cluster_state.rs new file mode 100644 index 0000000000..244e34bfb5 --- /dev/null +++ b/crates/types/src/cluster/cluster_state.rs @@ -0,0 +1,128 @@ +// Copyright (c) 2024 - Restate Software, Inc., Restate GmbH. +// All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use std::collections::BTreeMap; +use std::time::Instant; + +use prost_dto::IntoProto; +use serde::{Deserialize, Serialize}; + +use crate::identifiers::{LeaderEpoch, PartitionId}; +use crate::logs::Lsn; +use crate::time::MillisSinceEpoch; +use crate::{GenerationalNodeId, PlainNodeId, Version}; + +/// A container for health information about every node and partition in the +/// cluster. +#[derive(Debug, Clone, IntoProto)] +#[proto(target = "crate::protobuf::cluster::ClusterState")] +pub struct ClusterState { + #[into_proto(map = "instant_to_proto")] + pub last_refreshed: Option, + #[proto(required)] + pub nodes_config_version: Version, + #[proto(required)] + pub partition_table_version: Version, + pub nodes: BTreeMap, +} + +impl ClusterState { + pub fn is_reliable(&self) -> bool { + // todo: make this configurable + // If the cluster state is older than 10 seconds, then it is not reliable. + self.last_refreshed + .map(|last_refreshed| last_refreshed.elapsed().as_secs() < 10) + .unwrap_or(false) + } +} + +fn instant_to_proto(t: Instant) -> prost_types::Duration { + t.elapsed().try_into().unwrap() +} + +#[derive(Debug, Clone, IntoProto)] +#[proto(target = "crate::protobuf::cluster::NodeState", oneof = "state")] +pub enum NodeState { + Alive(AliveNode), + Dead(DeadNode), +} + +#[derive(Debug, Clone, IntoProto)] +#[proto(target = "crate::protobuf::cluster::AliveNode")] +pub struct AliveNode { + #[proto(required)] + pub last_heartbeat_at: MillisSinceEpoch, + #[proto(required)] + pub generational_node_id: GenerationalNodeId, + pub partitions: BTreeMap, +} + +#[derive(Debug, Clone, IntoProto)] +#[proto(target = "crate::protobuf::cluster::DeadNode")] +pub struct DeadNode { + pub last_seen_alive: Option, +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, Eq, PartialEq, IntoProto)] +#[proto(target = "crate::protobuf::cluster::RunMode")] +pub enum RunMode { + Leader, + Follower, +} + +#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize, IntoProto)] +#[proto(target = "crate::protobuf::cluster::ReplayStatus")] +pub enum ReplayStatus { + Starting, + Active, + CatchingUp, +} + +#[derive(Debug, Clone, Serialize, Deserialize, IntoProto)] +#[proto(target = "crate::protobuf::cluster::PartitionProcessorStatus")] +pub struct PartitionProcessorStatus { + #[proto(required)] + pub updated_at: MillisSinceEpoch, + pub planned_mode: RunMode, + pub effective_mode: Option, + pub last_observed_leader_epoch: Option, + pub last_observed_leader_node: Option, + pub last_applied_log_lsn: Option, + pub last_record_applied_at: Option, + pub num_skipped_records: u64, + pub replay_status: ReplayStatus, + pub last_persisted_log_lsn: Option, + // Set if replay_status is CatchingUp + pub target_tail_lsn: Option, +} + +impl PartitionProcessorStatus { + pub fn is_effective_leader(&self) -> bool { + self.effective_mode + .map(|m| m == RunMode::Leader) + .unwrap_or(false) + } + + pub fn new(planned_mode: RunMode) -> Self { + Self { + updated_at: MillisSinceEpoch::now(), + planned_mode, + effective_mode: None, + last_observed_leader_epoch: None, + last_observed_leader_node: None, + last_applied_log_lsn: None, + last_record_applied_at: None, + num_skipped_records: 0, + replay_status: ReplayStatus::Starting, + last_persisted_log_lsn: None, + target_tail_lsn: None, + } + } +} diff --git a/crates/types/src/cluster/mod.rs b/crates/types/src/cluster/mod.rs new file mode 100644 index 0000000000..cb370610fd --- /dev/null +++ b/crates/types/src/cluster/mod.rs @@ -0,0 +1,11 @@ +// Copyright (c) 2024 - Restate Software, Inc., Restate GmbH. +// All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. +// +pub mod cluster_state; diff --git a/crates/types/src/identifiers.rs b/crates/types/src/identifiers.rs index 6719927a5e..cd44f87c30 100644 --- a/crates/types/src/identifiers.rs +++ b/crates/types/src/identifiers.rs @@ -60,6 +60,19 @@ impl Default for LeaderEpoch { } } +impl From for LeaderEpoch { + fn from(epoch: crate::protobuf::common::LeaderEpoch) -> Self { + Self::from(epoch.value) + } +} + +impl From for crate::protobuf::common::LeaderEpoch { + fn from(epoch: LeaderEpoch) -> Self { + let value: u64 = epoch.into(); + Self { value } + } +} + /// Identifying the partition #[derive( Copy, diff --git a/crates/types/src/lib.rs b/crates/types/src/lib.rs index d062d34ae7..af875cb493 100644 --- a/crates/types/src/lib.rs +++ b/crates/types/src/lib.rs @@ -18,6 +18,7 @@ mod version; pub mod arc_util; pub mod art; +pub mod cluster; pub mod config; pub mod deployment; pub mod endpoint_manifest; @@ -33,8 +34,9 @@ pub mod metadata_store; pub mod net; pub mod nodes_config; pub mod partition_table; -pub mod processors; +pub mod protobuf; pub mod retries; +pub mod schema; pub mod service_discovery; pub mod service_protocol; pub mod state_mut; diff --git a/crates/types/src/logs/mod.rs b/crates/types/src/logs/mod.rs index d3624f45f7..817ef2d154 100644 --- a/crates/types/src/logs/mod.rs +++ b/crates/types/src/logs/mod.rs @@ -77,6 +77,19 @@ impl Lsn { } } +impl From for Lsn { + fn from(lsn: crate::protobuf::common::Lsn) -> Self { + Self::from(lsn.value) + } +} + +impl From for crate::protobuf::common::Lsn { + fn from(lsn: Lsn) -> Self { + let value: u64 = lsn.into(); + Self { value } + } +} + impl SequenceNumber for Lsn { /// The maximum possible sequence number, this is useful when creating a read stream /// with an open ended tail. diff --git a/crates/types/src/net.rs b/crates/types/src/net.rs deleted file mode 100644 index ba5b5aab31..0000000000 --- a/crates/types/src/net.rs +++ /dev/null @@ -1,112 +0,0 @@ -// Copyright (c) 2024 - Restate Software, Inc., Restate GmbH. -// All rights reserved. -// -// Use of this software is governed by the Business Source License -// included in the LICENSE file. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0. - -use http::Uri; -use std::net::{AddrParseError, SocketAddr}; -use std::path::PathBuf; -use std::str::FromStr; - -#[derive( - Debug, - Clone, - Eq, - PartialEq, - Hash, - derive_more::Display, - serde_with::SerializeDisplay, - serde_with::DeserializeFromStr, -)] -pub enum AdvertisedAddress { - /// Unix domain socket - #[display(fmt = "unix:{}", "_0.display()")] - Uds(PathBuf), - /// Hostname or host:port pair, or any unrecognizable string. - #[display(fmt = "{}", _0)] - Http(Uri), -} - -impl FromStr for AdvertisedAddress { - type Err = http::uri::InvalidUri; - - fn from_str(s: &str) -> Result { - if let Some(stripped_address) = s.strip_prefix("unix:") { - Ok(AdvertisedAddress::Uds( - stripped_address.parse().expect("infallible"), - )) - } else { - // try to parse as a URI - Ok(AdvertisedAddress::Http(s.parse()?)) - } - } -} - -#[derive( - Debug, - Clone, - Eq, - PartialEq, - Hash, - derive_more::Display, - serde_with::SerializeDisplay, - serde_with::DeserializeFromStr, -)] -pub enum BindAddress { - /// Unix domain socket - #[display(fmt = "unix:{}", "_0.display()")] - Uds(PathBuf), - /// Socket addr. - #[display(fmt = "{}", _0)] - Socket(SocketAddr), -} - -impl FromStr for BindAddress { - type Err = AddrParseError; - - fn from_str(s: &str) -> Result { - if let Some(stripped_address) = s.strip_prefix("unix:") { - Ok(BindAddress::Uds( - stripped_address.parse().expect("infallible"), - )) - } else { - // try to parse as a URI - Ok(BindAddress::Socket(s.parse()?)) - } - } -} - -#[cfg(test)] -mod tests { - use crate::net::AdvertisedAddress; - use http::Uri; - - // test parsing [`AdvertisedAddress`] - #[test] - fn test_parse_network_address() -> anyhow::Result<()> { - let tcp: AdvertisedAddress = "127.0.0.1:5123".parse()?; - restate_test_util::assert_eq!(tcp, AdvertisedAddress::Http("127.0.0.1:5123".parse()?)); - - let tcp: AdvertisedAddress = "unix:/tmp/unix.socket".parse()?; - restate_test_util::assert_eq!( - tcp, - AdvertisedAddress::Uds("/tmp/unix.socket".parse().unwrap()) - ); - - let tcp: AdvertisedAddress = "localhost:5123".parse()?; - restate_test_util::assert_eq!(tcp, AdvertisedAddress::Http("localhost:5123".parse()?)); - - let tcp: AdvertisedAddress = "https://localhost:5123".parse()?; - restate_test_util::assert_eq!( - tcp, - AdvertisedAddress::Http(Uri::from_static("https://localhost:5123")) - ); - - Ok(()) - } -} diff --git a/crates/node-protocol/src/cluster_controller.rs b/crates/types/src/net/cluster_controller.rs similarity index 85% rename from crates/node-protocol/src/cluster_controller.rs rename to crates/types/src/net/cluster_controller.rs index 4de6b1b8f1..c5d3be2394 100644 --- a/crates/node-protocol/src/cluster_controller.rs +++ b/crates/types/src/net/cluster_controller.rs @@ -10,11 +10,12 @@ use serde::{Deserialize, Serialize}; -use restate_types::identifiers::PartitionId; -use restate_types::processors::RunMode; +use crate::cluster::cluster_state::RunMode; +use crate::identifiers::PartitionId; +use crate::net::{RequestId, TargetName}; +use crate::partition_table::KeyRange; -use crate::common::{KeyRange, RequestId, TargetName}; -use crate::define_rpc; +use crate::net::define_rpc; define_rpc! { @request = AttachRequest, diff --git a/crates/node-protocol/src/codec.rs b/crates/types/src/net/codec.rs similarity index 94% rename from crates/node-protocol/src/codec.rs rename to crates/types/src/net/codec.rs index 4e5e37a463..e8b2135b42 100644 --- a/crates/node-protocol/src/codec.rs +++ b/crates/types/src/net/codec.rs @@ -11,15 +11,15 @@ use std::sync::Arc; use bytes::{Buf, BufMut, BytesMut}; -use restate_types::storage::{decode_from_flexbuffers, encode_as_flexbuffers}; use serde::de::DeserializeOwned; use serde::Serialize; -use crate::common::ProtocolVersion; -use crate::common::TargetName; -use crate::node::message; -use crate::node::message::BinaryMessage; -use crate::CodecError; +use crate::net::CodecError; +use crate::protobuf::common::ProtocolVersion; +use crate::protobuf::common::TargetName; +use crate::protobuf::node::message; +use crate::protobuf::node::message::BinaryMessage; +use crate::storage::{decode_from_flexbuffers, encode_as_flexbuffers}; pub trait Targeted { const TARGET: TargetName; diff --git a/crates/node-protocol/src/error.rs b/crates/types/src/net/error.rs similarity index 93% rename from crates/node-protocol/src/error.rs rename to crates/types/src/net/error.rs index 7ac2c9fd6b..959610fcf2 100644 --- a/crates/node-protocol/src/error.rs +++ b/crates/types/src/net/error.rs @@ -8,7 +8,7 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -use restate_types::errors::GenericError; +use crate::errors::GenericError; #[derive(Debug, thiserror::Error)] pub enum CodecError { diff --git a/crates/node-protocol/src/ingress.rs b/crates/types/src/net/ingress.rs similarity index 84% rename from crates/node-protocol/src/ingress.rs rename to crates/types/src/net/ingress.rs index c3c3ea209e..4d6d816ece 100644 --- a/crates/node-protocol/src/ingress.rs +++ b/crates/types/src/net/ingress.rs @@ -8,11 +8,10 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -use restate_types::ingress::{InvocationResponse, SubmittedInvocationNotification}; use serde::{Deserialize, Serialize}; -use crate::common::TargetName; -use crate::define_message; +use crate::ingress::{InvocationResponse, SubmittedInvocationNotification}; +use crate::net::{define_message, TargetName}; #[derive( Debug, diff --git a/crates/node-protocol/src/metadata.rs b/crates/types/src/net/metadata.rs similarity index 86% rename from crates/node-protocol/src/metadata.rs rename to crates/types/src/net/metadata.rs index 30fbb744ca..acf2f3449c 100644 --- a/crates/node-protocol/src/metadata.rs +++ b/crates/types/src/net/metadata.rs @@ -9,15 +9,16 @@ // by the Apache License, Version 2.0. use enum_map::Enum; -pub use restate_schema::{Schema, UpdateableSchema}; -use restate_types::logs::metadata::Logs; -use restate_types::nodes_config::NodesConfiguration; -use restate_types::partition_table::FixedPartitionTable; use serde::{Deserialize, Serialize}; use strum_macros::EnumIter; -use crate::common::TargetName; -use crate::define_message; +use crate::logs::metadata::Logs; +use crate::net::TargetName; +use crate::nodes_config::NodesConfiguration; +use crate::partition_table::FixedPartitionTable; +use crate::schema::Schema; + +use crate::net::define_message; #[derive( Debug, @@ -70,7 +71,7 @@ pub enum MetadataContainer { #[derive(Debug, Clone, Serialize, Deserialize)] pub struct GetMetadataRequest { pub metadata_kind: MetadataKind, - pub min_version: Option, + pub min_version: Option, } #[derive(Debug, Clone, Serialize, Deserialize)] diff --git a/crates/types/src/net/mod.rs b/crates/types/src/net/mod.rs new file mode 100644 index 0000000000..c266c51375 --- /dev/null +++ b/crates/types/src/net/mod.rs @@ -0,0 +1,328 @@ +// Copyright (c) 2024 - Restate Software, Inc., Restate GmbH. +// All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +pub mod cluster_controller; +pub mod codec; +mod error; +pub mod ingress; +pub mod metadata; +pub mod partition_processor_manager; + +// re-exports for convenience +pub use error::*; + +use http::Uri; +use std::net::{AddrParseError, SocketAddr}; +use std::path::PathBuf; +use std::str::FromStr; + +use super::GenerationalNodeId; +pub use crate::protobuf::common::ProtocolVersion; +pub use crate::protobuf::common::TargetName; + +use self::codec::Targeted; +use self::codec::WireDecode; + +pub static MIN_SUPPORTED_PROTOCOL_VERSION: ProtocolVersion = ProtocolVersion::Flexbuffers; +pub static CURRENT_PROTOCOL_VERSION: ProtocolVersion = ProtocolVersion::Flexbuffers; + +/// Used to identify a request in a RPC-style call going through Networking. +#[derive( + Debug, + derive_more::Display, + PartialEq, + Eq, + Clone, + Copy, + Hash, + PartialOrd, + Ord, + serde::Serialize, + serde::Deserialize, +)] +pub struct RequestId(u64); +impl RequestId { + pub fn new() -> Self { + Default::default() + } +} + +impl Default for RequestId { + fn default() -> Self { + use std::sync::atomic::AtomicUsize; + static NEXT_REQUEST_ID: AtomicUsize = AtomicUsize::new(1); + RequestId( + NEXT_REQUEST_ID + .fetch_add(1, std::sync::atomic::Ordering::Relaxed) + .try_into() + .unwrap(), + ) + } +} + +#[derive( + Debug, + Clone, + Eq, + PartialEq, + Hash, + derive_more::Display, + serde_with::SerializeDisplay, + serde_with::DeserializeFromStr, +)] +pub enum AdvertisedAddress { + /// Unix domain socket + #[display(fmt = "unix:{}", "_0.display()")] + Uds(PathBuf), + /// Hostname or host:port pair, or any unrecognizable string. + #[display(fmt = "{}", _0)] + Http(Uri), +} + +impl FromStr for AdvertisedAddress { + type Err = http::uri::InvalidUri; + + fn from_str(s: &str) -> Result { + if let Some(stripped_address) = s.strip_prefix("unix:") { + Ok(AdvertisedAddress::Uds( + stripped_address.parse().expect("infallible"), + )) + } else { + // try to parse as a URI + Ok(AdvertisedAddress::Http(s.parse()?)) + } + } +} + +#[derive( + Debug, + Clone, + Eq, + PartialEq, + Hash, + derive_more::Display, + serde_with::SerializeDisplay, + serde_with::DeserializeFromStr, +)] +pub enum BindAddress { + /// Unix domain socket + #[display(fmt = "unix:{}", "_0.display()")] + Uds(PathBuf), + /// Socket addr. + #[display(fmt = "{}", _0)] + Socket(SocketAddr), +} + +impl FromStr for BindAddress { + type Err = AddrParseError; + + fn from_str(s: &str) -> Result { + if let Some(stripped_address) = s.strip_prefix("unix:") { + Ok(BindAddress::Uds( + stripped_address.parse().expect("infallible"), + )) + } else { + // try to parse as a URI + Ok(BindAddress::Socket(s.parse()?)) + } + } +} + +/// A wrapper for a message that includes the sender id +pub struct MessageEnvelope { + peer: GenerationalNodeId, + connection_id: u64, + body: M, +} + +impl MessageEnvelope { + pub fn new(peer: GenerationalNodeId, connection_id: u64, body: M) -> Self { + Self { + peer, + connection_id, + body, + } + } +} + +impl MessageEnvelope { + pub fn connection_id(&self) -> u64 { + self.connection_id + } + + pub fn split(self) -> (GenerationalNodeId, M) { + (self.peer, self.body) + } + + pub fn body(&self) -> &M { + &self.body + } +} + +impl MessageEnvelope { + /// A unique identifier used by RPC-style messages to correlated requests and responses + pub fn correlation_id(&self) -> M::CorrelationId { + self.body.correlation_id() + } +} + +pub trait RpcMessage { + type CorrelationId: Clone + Send + Eq + PartialEq + std::fmt::Debug + std::hash::Hash; + fn correlation_id(&self) -> Self::CorrelationId; +} + +pub trait RpcRequest: RpcMessage + Targeted { + type Response: RpcMessage + Targeted; +} + +// to define a message, we need +// - Message type +// - message target +// +// Example: +// ``` +// define_message! { +// @message = IngressMessage, +// @target = TargetName::Ingress, +// } +// ``` +macro_rules! define_message { + ( + @message = $message:ty, + @target = $target:expr, + ) => { + impl $crate::net::Targeted for $message { + const TARGET: $crate::net::TargetName = $target; + fn kind(&self) -> &'static str { + stringify!($message) + } + } + + impl $crate::net::codec::WireEncode for $message { + fn encode( + &self, + buf: &mut B, + protocol_version: $crate::net::ProtocolVersion, + ) -> Result<(), $crate::net::CodecError> { + // serialize message into buf + $crate::net::codec::encode_default(self, buf, protocol_version) + } + } + + impl $crate::net::codec::WireDecode for $message { + fn decode( + buf: &mut B, + protocol_version: $crate::net::ProtocolVersion, + ) -> Result + where + Self: Sized, + { + $crate::net::codec::decode_default(buf, protocol_version) + } + } + }; +} + +// to define an RPC, we need +// - Request type +// - request target +// - Response type +// - response Target +// +// Example: +// ``` +// define_rpc! { +// @request = AttachRequest, +// @response = AttachResponse, +// @request_target = TargetName::ClusterController, +// @response_target = TargetName::AttachResponse, +// } +// ``` +#[allow(unused_macros)] +macro_rules! define_rpc { + ( + @request = $request:ty, + @response = $response:ty, + @request_target = $request_target:expr, + @response_target = $response_target:expr, + ) => { + impl $crate::net::RpcRequest for $request { + type Response = $response; + } + + impl $crate::net::RpcMessage for $request { + type CorrelationId = $crate::net::RequestId; + + fn correlation_id(&self) -> Self::CorrelationId { + self.request_id + } + } + + impl $crate::net::RpcMessage for $response { + type CorrelationId = $crate::net::RequestId; + + fn correlation_id(&self) -> Self::CorrelationId { + self.request_id + } + } + + $crate::net::define_message! { + @message = $request, + @target = $request_target, + } + + $crate::net::define_message! { + @message = $response, + @target = $response_target, + } + }; +} + +#[allow(unused_imports)] +use {define_message, define_rpc}; + +#[cfg(test)] +mod tests { + use http::Uri; + + use super::*; + + // test parsing [`AdvertisedAddress`] + #[test] + fn test_parse_network_address() -> anyhow::Result<()> { + let tcp: AdvertisedAddress = "127.0.0.1:5123".parse()?; + restate_test_util::assert_eq!(tcp, AdvertisedAddress::Http("127.0.0.1:5123".parse()?)); + + let tcp: AdvertisedAddress = "unix:/tmp/unix.socket".parse()?; + restate_test_util::assert_eq!( + tcp, + AdvertisedAddress::Uds("/tmp/unix.socket".parse().unwrap()) + ); + + let tcp: AdvertisedAddress = "localhost:5123".parse()?; + restate_test_util::assert_eq!(tcp, AdvertisedAddress::Http("localhost:5123".parse()?)); + + let tcp: AdvertisedAddress = "https://localhost:5123".parse()?; + restate_test_util::assert_eq!( + tcp, + AdvertisedAddress::Http(Uri::from_static("https://localhost:5123")) + ); + + Ok(()) + } + + #[test] + fn test_request_id() { + let request_id1 = RequestId::new(); + let request_id2 = RequestId::new(); + let request_id3 = RequestId::default(); + assert!(request_id1.0 < request_id2.0 && request_id2.0 < request_id3.0); + } +} diff --git a/crates/node-protocol/src/partition_processor_manager.rs b/crates/types/src/net/partition_processor_manager.rs similarity index 86% rename from crates/node-protocol/src/partition_processor_manager.rs rename to crates/types/src/net/partition_processor_manager.rs index 2b552c7a48..d62f5ef65e 100644 --- a/crates/node-protocol/src/partition_processor_manager.rs +++ b/crates/types/src/net/partition_processor_manager.rs @@ -10,13 +10,14 @@ use std::collections::BTreeMap; -use restate_types::identifiers::PartitionId; -use restate_types::processors::PartitionProcessorStatus; use serde::{Deserialize, Serialize}; use serde_with::serde_as; -use crate::common::{RequestId, TargetName}; -use crate::define_rpc; +use crate::cluster::cluster_state::PartitionProcessorStatus; +use crate::identifiers::PartitionId; +use crate::net::{RequestId, TargetName}; + +use crate::net::define_rpc; define_rpc! { @request = GetProcessorsState, diff --git a/crates/node-protocol/src/status.rs b/crates/types/src/net/status.rs similarity index 100% rename from crates/node-protocol/src/status.rs rename to crates/types/src/net/status.rs diff --git a/crates/types/src/node_id.rs b/crates/types/src/node_id.rs index d6ec92885a..882b7a90bc 100644 --- a/crates/types/src/node_id.rs +++ b/crates/types/src/node_id.rs @@ -144,6 +144,40 @@ impl PartialEq for NodeId { } } +impl From for NodeId { + fn from(node_id: crate::protobuf::common::NodeId) -> Self { + NodeId::new(node_id.id, node_id.generation) + } +} + +impl From for crate::protobuf::common::NodeId { + fn from(node_id: NodeId) -> Self { + crate::protobuf::common::NodeId { + id: node_id.id().into(), + generation: node_id.as_generational().map(|g| g.generation()), + } + } +} + +impl From for crate::protobuf::common::NodeId { + fn from(node_id: PlainNodeId) -> Self { + let id: u32 = node_id.into(); + crate::protobuf::common::NodeId { + id, + generation: None, + } + } +} + +impl From for crate::protobuf::common::NodeId { + fn from(node_id: GenerationalNodeId) -> Self { + crate::protobuf::common::NodeId { + id: node_id.raw_id(), + generation: Some(node_id.generation()), + } + } +} + impl PlainNodeId { pub fn with_generation(self, generation: u32) -> GenerationalNodeId { GenerationalNodeId(self, generation) diff --git a/crates/types/src/partition_table.rs b/crates/types/src/partition_table.rs index 02b33ef6e9..4a84c721f4 100644 --- a/crates/types/src/partition_table.rs +++ b/crates/types/src/partition_table.rs @@ -8,11 +8,12 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -use crate::identifiers::{PartitionId, PartitionKey}; -use crate::{flexbuffers_storage_encode_decode, Version, Versioned}; use std::borrow::Borrow; use std::ops::RangeInclusive; +use crate::identifiers::{PartitionId, PartitionKey}; +use crate::{flexbuffers_storage_encode_decode, Version, Versioned}; + #[derive(Debug, thiserror::Error)] #[error("Cannot find target peer for partition key {0}")] pub struct PartitionTableError(PartitionKey); @@ -24,6 +25,18 @@ pub trait FindPartition { ) -> Result; } +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct KeyRange { + pub from: PartitionKey, + pub to: PartitionKey, +} + +impl From for RangeInclusive { + fn from(val: KeyRange) -> Self { + RangeInclusive::new(val.from, val.to) + } +} + #[derive(Debug, Clone, Eq, PartialEq, serde::Serialize, serde::Deserialize)] pub struct FixedPartitionTable { version: Version, diff --git a/crates/types/src/processors/mod.rs b/crates/types/src/processors/mod.rs deleted file mode 100644 index 142d0edb33..0000000000 --- a/crates/types/src/processors/mod.rs +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright (c) 2024 - Restate Software, Inc., Restate GmbH. -// All rights reserved. -// -// Use of this software is governed by the Business Source License -// included in the LICENSE file. -// -// As of the Change Date specified in that file, in accordance with -// the Business Source License, use of this software will be governed -// by the Apache License, Version 2.0. - -use serde::{Deserialize, Serialize}; - -use crate::identifiers::LeaderEpoch; -use crate::logs::Lsn; -use crate::time::MillisSinceEpoch; -use crate::GenerationalNodeId; - -#[derive(Debug, Clone, Copy, Serialize, Deserialize, Eq, PartialEq)] -pub enum RunMode { - Leader, - Follower, -} - -#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] -pub enum ReplayStatus { - Starting, - Active, - CatchingUp { target_tail_lsn: Lsn }, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct PartitionProcessorStatus { - pub updated_at: MillisSinceEpoch, - pub planned_mode: RunMode, - pub effective_mode: Option, - pub last_observed_leader_epoch: Option, - pub last_observed_leader_node: Option, - pub last_applied_log_lsn: Option, - pub last_record_applied_at: Option, - pub num_skipped_records: u64, - pub replay_status: ReplayStatus, - pub last_persisted_log_lsn: Option, -} - -impl PartitionProcessorStatus { - pub fn is_effective_leader(&self) -> bool { - self.effective_mode - .map(|m| m == RunMode::Leader) - .unwrap_or(false) - } - - pub fn new(planned_mode: RunMode) -> Self { - Self { - updated_at: MillisSinceEpoch::now(), - planned_mode, - effective_mode: None, - last_observed_leader_epoch: None, - last_observed_leader_node: None, - last_applied_log_lsn: None, - last_record_applied_at: None, - num_skipped_records: 0, - replay_status: ReplayStatus::Starting, - last_persisted_log_lsn: None, - } - } -} diff --git a/crates/types/src/protobuf.rs b/crates/types/src/protobuf.rs new file mode 100644 index 0000000000..851eeb4f5e --- /dev/null +++ b/crates/types/src/protobuf.rs @@ -0,0 +1,159 @@ +// Copyright (c) 2024 - Restate Software, Inc., Restate GmbH. +// All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +pub const FILE_DESCRIPTOR_SET: &[u8] = + include_bytes!(concat!(env!("OUT_DIR"), "/common_descriptor.bin")); + +pub mod common { + use crate::net::{CURRENT_PROTOCOL_VERSION, MIN_SUPPORTED_PROTOCOL_VERSION}; + + include!(concat!(env!("OUT_DIR"), "/restate.common.rs")); + + impl ProtocolVersion { + pub fn is_supported(&self) -> bool { + *self >= MIN_SUPPORTED_PROTOCOL_VERSION && *self <= CURRENT_PROTOCOL_VERSION + } + } + + impl std::fmt::Display for NodeId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + if let Some(generation) = self.generation { + write!(f, "N{}:{}", self.id, generation) + } else { + write!(f, "N{}", self.id) + } + } + } + + impl std::fmt::Display for Lsn { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.value) + } + } + + impl std::fmt::Display for LeaderEpoch { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "e{}", self.value) + } + } +} + +pub mod cluster { + include!(concat!(env!("OUT_DIR"), "/restate.cluster.rs")); + + impl std::fmt::Display for RunMode { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let o = match self { + RunMode::Unknown => "UNKNOWN", + RunMode::Leader => "Leader", + RunMode::Follower => "Follower", + }; + write!(f, "{}", o) + } + } +} + +pub mod node { + use crate::GenerationalNodeId; + + use crate::net::{ProtocolVersion, CURRENT_PROTOCOL_VERSION, MIN_SUPPORTED_PROTOCOL_VERSION}; + + use self::message::{BinaryMessage, ConnectionControl, Signal}; + + include!(concat!(env!("OUT_DIR"), "/restate.node.rs")); + + impl Hello { + pub fn new(my_node_id: GenerationalNodeId, cluster_name: String) -> Self { + Self { + min_protocol_version: MIN_SUPPORTED_PROTOCOL_VERSION.into(), + max_protocol_version: CURRENT_PROTOCOL_VERSION.into(), + my_node_id: Some(my_node_id.into()), + cluster_name, + } + } + } + + impl Header { + pub fn new(nodes_config_version: crate::Version) -> Self { + Self { + my_nodes_config_version: Some(nodes_config_version.into()), + } + } + } + + impl Welcome { + pub fn new(my_node_id: GenerationalNodeId, protocol_version: ProtocolVersion) -> Self { + Self { + my_node_id: Some(my_node_id.into()), + protocol_version: protocol_version.into(), + } + } + } + + impl Message { + pub fn new(header: Header, body: impl Into) -> Self { + Self { + header: Some(header), + body: Some(body.into()), + } + } + } + + impl From for message::Body { + fn from(value: Hello) -> Self { + message::Body::Hello(value) + } + } + + impl From for message::Body { + fn from(value: Welcome) -> Self { + message::Body::Welcome(value) + } + } + + impl From for message::Body { + fn from(value: ConnectionControl) -> Self { + message::Body::ConnectionControl(value) + } + } + + impl From for message::Body { + fn from(value: BinaryMessage) -> Self { + message::Body::Encoded(value) + } + } + + impl ConnectionControl { + pub fn connection_reset() -> Self { + Self { + signal: message::Signal::DrainConnection.into(), + message: "Connection is draining and will be dropped".to_owned(), + } + } + pub fn shutdown() -> Self { + Self { + signal: message::Signal::Shutdown.into(), + message: "Node is shutting down".to_owned(), + } + } + pub fn codec_error(message: impl Into) -> Self { + Self { + signal: message::Signal::CodecError.into(), + message: message.into(), + } + } + } + + impl std::fmt::Display for Signal { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.as_str_name()) + } + } +} diff --git a/crates/types/src/schema/deployment.rs b/crates/types/src/schema/deployment.rs new file mode 100644 index 0000000000..5c4ea55365 --- /dev/null +++ b/crates/types/src/schema/deployment.rs @@ -0,0 +1,376 @@ +// Copyright (c) 2023 - Restate Software, Inc., Restate GmbH. +// All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. +// +use std::collections::HashMap; +use std::fmt; +use std::fmt::{Display, Formatter}; +use std::ops::RangeInclusive; + +use bytestring::ByteString; +use http::header::{HeaderName, HeaderValue}; +use http::Uri; +use serde::{Deserialize, Serialize}; +use serde_with::serde_as; + +use crate::identifiers::{DeploymentId, LambdaARN, ServiceRevision}; +use crate::schema::service::ServiceMetadata; +use crate::schema::{Schema, UpdateableSchema}; +use crate::time::MillisSinceEpoch; + +#[derive(Debug, Copy, Clone, Eq, PartialEq, Serialize, Deserialize)] +#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] +pub enum ProtocolType { + RequestResponse, + BidiStream, +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] +pub struct DeliveryOptions { + #[serde( + with = "serde_with::As::>" + )] + #[cfg_attr(feature = "schemars", schemars(with = "HashMap"))] + pub additional_headers: HashMap, +} + +impl DeliveryOptions { + pub fn new(additional_headers: HashMap) -> Self { + Self { additional_headers } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] +pub struct Deployment { + pub id: DeploymentId, + pub metadata: DeploymentMetadata, +} + +#[serde_as] +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] +pub struct DeploymentMetadata { + pub ty: DeploymentType, + pub delivery_options: DeliveryOptions, + pub supported_protocol_versions: RangeInclusive, + pub created_at: MillisSinceEpoch, +} + +#[serde_as] +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] +pub enum DeploymentType { + Http { + #[serde(with = "serde_with::As::")] + #[cfg_attr(feature = "schemars", schemars(with = "String"))] + address: Uri, + protocol_type: ProtocolType, + }, + Lambda { + arn: LambdaARN, + #[cfg_attr(feature = "schemars", schemars(with = "Option"))] + assume_role_arn: Option, + }, +} + +impl DeploymentType { + pub fn protocol_type(&self) -> ProtocolType { + match self { + DeploymentType::Http { protocol_type, .. } => *protocol_type, + DeploymentType::Lambda { .. } => ProtocolType::RequestResponse, + } + } + + pub fn normalized_address(&self) -> String { + match self { + DeploymentType::Http { address, .. } => { + // We use only authority and path, as those uniquely identify the deployment. + format!( + "{}{}", + address.authority().expect("Must have authority"), + address.path() + ) + } + DeploymentType::Lambda { arn, .. } => arn.to_string(), + } + } +} + +impl DeploymentMetadata { + pub fn new_http( + address: Uri, + protocol_type: ProtocolType, + delivery_options: DeliveryOptions, + supported_protocol_versions: RangeInclusive, + ) -> Self { + Self { + ty: DeploymentType::Http { + address, + protocol_type, + }, + delivery_options, + created_at: MillisSinceEpoch::now(), + supported_protocol_versions, + } + } + + pub fn new_lambda( + arn: LambdaARN, + assume_role_arn: Option, + delivery_options: DeliveryOptions, + supported_protocol_versions: RangeInclusive, + ) -> Self { + Self { + ty: DeploymentType::Lambda { + arn, + assume_role_arn, + }, + delivery_options, + created_at: MillisSinceEpoch::now(), + supported_protocol_versions, + } + } + + // address_display returns a Displayable identifier for the endpoint; for http endpoints this is a URI, + // and for Lambda deployments its the ARN + pub fn address_display(&self) -> impl Display + '_ { + struct Wrapper<'a>(&'a DeploymentType); + impl<'a> Display for Wrapper<'a> { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + match self { + Wrapper(DeploymentType::Http { address, .. }) => address.fmt(f), + Wrapper(DeploymentType::Lambda { arn, .. }) => arn.fmt(f), + } + } + } + Wrapper(&self.ty) + } + + pub fn created_at(&self) -> MillisSinceEpoch { + self.created_at + } +} + +pub trait DeploymentResolver { + fn resolve_latest_deployment_for_service( + &self, + service_name: impl AsRef, + ) -> Option; + + fn get_deployment(&self, deployment_id: &DeploymentId) -> Option; + + fn get_deployment_and_services( + &self, + deployment_id: &DeploymentId, + ) -> Option<(Deployment, Vec)>; + + fn get_deployments(&self) -> Vec<(Deployment, Vec<(String, ServiceRevision)>)>; +} + +#[cfg(feature = "test-util")] +pub mod test_util { + use super::*; + + use crate::service_protocol::MAX_SERVICE_PROTOCOL_VERSION_VALUE; + use std::collections::HashMap; + + impl Deployment { + pub fn mock() -> Deployment { + let id = "dp_15VqmTOnXH3Vv2pl5HOG7UB" + .parse() + .expect("valid stable deployment id"); + let metadata = DeploymentMetadata::new_http( + "http://localhost:9080".parse().unwrap(), + ProtocolType::BidiStream, + Default::default(), + 1..=MAX_SERVICE_PROTOCOL_VERSION_VALUE, + ); + + Deployment { id, metadata } + } + + pub fn mock_with_uri(uri: &str) -> Deployment { + let id = DeploymentId::new(); + let metadata = DeploymentMetadata::new_http( + uri.parse().unwrap(), + ProtocolType::BidiStream, + Default::default(), + 1..=MAX_SERVICE_PROTOCOL_VERSION_VALUE, + ); + Deployment { id, metadata } + } + } + + #[derive(Default, Clone, Debug)] + pub struct MockDeploymentMetadataRegistry { + pub deployments: HashMap, + pub latest_deployment: HashMap, + } + + impl MockDeploymentMetadataRegistry { + pub fn mock_service(&mut self, service: &str) { + self.mock_service_with_metadata(service, Deployment::mock()); + } + + pub fn mock_service_with_metadata(&mut self, service: &str, deployment: Deployment) { + self.latest_deployment + .insert(service.to_string(), deployment.id); + self.deployments.insert(deployment.id, deployment.metadata); + } + } + + impl DeploymentResolver for MockDeploymentMetadataRegistry { + fn resolve_latest_deployment_for_service( + &self, + service_name: impl AsRef, + ) -> Option { + self.latest_deployment + .get(service_name.as_ref()) + .and_then(|deployment_id| self.get_deployment(deployment_id)) + } + + fn get_deployment(&self, deployment_id: &DeploymentId) -> Option { + self.deployments + .get(deployment_id) + .cloned() + .map(|metadata| Deployment { + id: *deployment_id, + metadata, + }) + } + + fn get_deployment_and_services( + &self, + deployment_id: &DeploymentId, + ) -> Option<(Deployment, Vec)> { + self.deployments + .get(deployment_id) + .cloned() + .map(|metadata| { + ( + Deployment { + id: *deployment_id, + metadata, + }, + vec![], + ) + }) + } + + fn get_deployments(&self) -> Vec<(Deployment, Vec<(String, ServiceRevision)>)> { + self.deployments + .iter() + .map(|(id, metadata)| { + ( + Deployment { + id: *id, + metadata: metadata.clone(), + }, + vec![], + ) + }) + .collect() + } + } +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct DeploymentSchemas { + pub metadata: DeploymentMetadata, + + // We need to store ServiceMetadata here only for queries + // We could optimize the memory impact of this by reading these info from disk + pub services: Vec, +} + +impl DeploymentResolver for Schema { + fn resolve_latest_deployment_for_service( + &self, + service_name: impl AsRef, + ) -> Option { + let service = self.services.get(service_name.as_ref())?; + self.deployments + .get(&service.location.latest_deployment) + .map(|schemas| Deployment { + id: service.location.latest_deployment, + metadata: schemas.metadata.clone(), + }) + } + + fn get_deployment(&self, deployment_id: &DeploymentId) -> Option { + self.deployments + .get(deployment_id) + .map(|schemas| Deployment { + id: *deployment_id, + metadata: schemas.metadata.clone(), + }) + } + + fn get_deployment_and_services( + &self, + deployment_id: &DeploymentId, + ) -> Option<(Deployment, Vec)> { + self.deployments.get(deployment_id).map(|schemas| { + ( + Deployment { + id: *deployment_id, + metadata: schemas.metadata.clone(), + }, + schemas.services.clone(), + ) + }) + } + + fn get_deployments(&self) -> Vec<(Deployment, Vec<(String, ServiceRevision)>)> { + self.deployments + .iter() + .map(|(deployment_id, schemas)| { + ( + Deployment { + id: *deployment_id, + metadata: schemas.metadata.clone(), + }, + schemas + .services + .iter() + .map(|s| (s.name.clone(), s.revision)) + .collect(), + ) + }) + .collect() + } +} + +impl DeploymentResolver for UpdateableSchema { + fn resolve_latest_deployment_for_service( + &self, + service_name: impl AsRef, + ) -> Option { + self.0 + .load() + .resolve_latest_deployment_for_service(service_name) + } + + fn get_deployment(&self, deployment_id: &DeploymentId) -> Option { + self.0.load().get_deployment(deployment_id) + } + + fn get_deployment_and_services( + &self, + deployment_id: &DeploymentId, + ) -> Option<(Deployment, Vec)> { + self.0.load().get_deployment_and_services(deployment_id) + } + + fn get_deployments(&self) -> Vec<(Deployment, Vec<(String, ServiceRevision)>)> { + self.0.load().get_deployments() + } +} diff --git a/crates/schema-api/src/invocation_target.rs b/crates/types/src/schema/invocation_target.rs similarity index 93% rename from crates/schema-api/src/invocation_target.rs rename to crates/types/src/schema/invocation_target.rs index 84e3f9a701..d9257378b8 100644 --- a/crates/schema-api/src/invocation_target.rs +++ b/crates/types/src/schema/invocation_target.rs @@ -8,19 +8,22 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -use bytes::Bytes; -use bytestring::ByteString; -use itertools::Itertools; -use restate_types::invocation::InvocationTargetType; use std::str::FromStr; use std::time::Duration; use std::{cmp, fmt}; +use bytes::Bytes; +use bytestring::ByteString; +use itertools::Itertools; +use serde::{Deserialize, Serialize}; + +use super::{Schema, UpdateableSchema}; +use crate::invocation::InvocationTargetType; + pub const DEFAULT_IDEMPOTENCY_RETENTION: Duration = Duration::from_secs(60 * 60 * 24); pub const DEFAULT_WORKFLOW_COMPLETION_RETENTION: Duration = Duration::from_secs(60 * 60 * 24); -#[derive(Debug, Clone)] -#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct InvocationTargetMetadata { pub public: bool, /// Retention timer to be used for the completion. See [`InvocationTargetMetadata::compute_retention`] for more details. @@ -55,6 +58,34 @@ pub trait InvocationTargetResolver { ) -> Option; } +impl InvocationTargetResolver for Schema { + fn resolve_latest_invocation_target( + &self, + service_name: impl AsRef, + handler_name: impl AsRef, + ) -> Option { + self.use_service_schema(service_name.as_ref(), |service_schemas| { + service_schemas + .handlers + .get(handler_name.as_ref()) + .map(|handler_schemas| handler_schemas.target_meta.clone()) + }) + .flatten() + } +} + +impl InvocationTargetResolver for UpdateableSchema { + fn resolve_latest_invocation_target( + &self, + service_name: impl AsRef, + handler_name: impl AsRef, + ) -> Option { + self.0 + .load() + .resolve_latest_invocation_target(service_name, handler_name) + } +} + // --- Input rules #[derive(Debug, thiserror::Error)] @@ -71,8 +102,7 @@ pub enum InputValidationError { ContentTypeNotMatching(String, InputContentType), } -#[derive(Debug, Clone, PartialEq, Eq)] -#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct InputRules { /// Input validation will try each of these rules. Validation passes if at least one rule matches. pub input_validation_rules: Vec, @@ -127,8 +157,7 @@ impl fmt::Display for InputRules { } } -#[derive(Debug, Clone, PartialEq, Eq)] -#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub enum InputValidationRule { // Input and content-type must be empty NoBodyAndContentType, @@ -200,8 +229,7 @@ impl InputValidationRule { } /// Describes a content type in the same format of the [`Accept` header](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Accept). -#[derive(Debug, Clone, PartialEq, Eq, Default)] -#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)] pub enum InputContentType { /// `*/*` #[default] @@ -304,8 +332,7 @@ impl FromStr for InputContentType { // --- Output rules -#[derive(Debug, Clone, Default, PartialEq, Eq)] -#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] pub struct OutputRules { pub content_type_rule: OutputContentTypeRule, } @@ -335,15 +362,11 @@ impl fmt::Display for OutputRules { } } -#[derive(Debug, Clone, PartialEq, Eq)] -#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub enum OutputContentTypeRule { None, Set { - #[cfg_attr( - feature = "serde", - serde(with = "serde_with::As::") - )] + #[serde(with = "serde_with::As::")] content_type: http::HeaderValue, // If true, sets the content-type even if the output is empty. // Otherwise, don't set the content-type. diff --git a/crates/schema/src/lib.rs b/crates/types/src/schema/mod.rs similarity index 86% rename from crates/schema/src/lib.rs rename to crates/types/src/schema/mod.rs index 589f21beb9..413d60ca68 100644 --- a/crates/schema/src/lib.rs +++ b/crates/types/src/schema/mod.rs @@ -8,23 +8,24 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -use arc_swap::ArcSwap; -use restate_schema_api::deployment::DeploymentType; -use restate_schema_api::service::{HandlerMetadata, ServiceMetadata}; -use restate_schema_api::subscription::Subscription; -use restate_types::identifiers::{DeploymentId, ServiceRevision, SubscriptionId}; -use serde_with::serde_as; +pub mod deployment; +pub mod invocation_target; +pub mod service; +pub mod subscriptions; + use std::collections::HashMap; use std::sync::Arc; -pub mod deployment; -mod invocation_target; -pub mod service; -mod subscriptions; +use arc_swap::ArcSwap; +use serde_with::serde_as; -use crate::deployment::DeploymentSchemas; -use crate::service::ServiceSchemas; -use restate_types::{Version, Versioned}; +use self::deployment::DeploymentSchemas; +use self::deployment::DeploymentType; +use self::service::ServiceSchemas; +use self::subscriptions::Subscription; +use crate::identifiers::{DeploymentId, SubscriptionId}; +use crate::Version; +use crate::Versioned; /// Schema information which automatically loads the latest version when accessing it. /// @@ -96,18 +97,22 @@ impl Versioned for Schema { } pub mod storage { - use crate::Schema; - use restate_types::flexbuffers_storage_encode_decode; + use crate::flexbuffers_storage_encode_decode; + + use super::Schema; flexbuffers_storage_encode_decode!(Schema); } #[cfg(feature = "test-util")] mod test_util { + use super::*; - use restate_schema_api::invocation_target::InvocationTargetResolver; - use restate_schema_api::service::ServiceMetadataResolver; + use super::invocation_target::InvocationTargetResolver; + use super::service::ServiceMetadata; + use super::service::ServiceMetadataResolver; + use crate::identifiers::ServiceRevision; use restate_test_util::{assert, assert_eq}; impl Schema { diff --git a/crates/types/src/schema/service.rs b/crates/types/src/schema/service.rs new file mode 100644 index 0000000000..318ad259f4 --- /dev/null +++ b/crates/types/src/schema/service.rs @@ -0,0 +1,291 @@ +// Copyright (c) 2023 - Restate Software, Inc., Restate GmbH. +// All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use std::collections::HashMap; +use std::time::Duration; + +use serde::Deserialize; +use serde::Serialize; +use serde_with::serde_as; + +use super::invocation_target::InvocationTargetMetadata; +use super::Schema; +use super::UpdateableSchema; +use crate::identifiers::{DeploymentId, ServiceRevision}; +use crate::invocation::{ + InvocationTargetType, ServiceType, VirtualObjectHandlerType, WorkflowHandlerType, +}; + +#[serde_as] +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] +pub struct ServiceMetadata { + /// # Name + /// + /// Fully qualified name of the service + pub name: String, + + pub handlers: Vec, + + pub ty: ServiceType, + + /// # Deployment Id + /// + /// Deployment exposing the latest revision of the service. + #[cfg_attr(feature = "schemars", schemars(with = "String"))] + pub deployment_id: DeploymentId, + + /// # Revision + /// + /// Latest revision of the service. + pub revision: ServiceRevision, + + /// # Public + /// + /// If true, the service can be invoked through the ingress. + /// If false, the service can be invoked only from another Restate service. + pub public: bool, + + /// # Idempotency retention + /// + /// The retention duration of idempotent requests for this service. + #[serde(with = "serde_with::As::")] + #[cfg_attr(feature = "schemars", schemars(with = "String"))] + pub idempotency_retention: humantime::Duration, + + /// # Workflow completion retention + /// + /// The retention duration of workflows. Only available on workflow services. + #[serde( + with = "serde_with::As::>", + skip_serializing_if = "Option::is_none", + default + )] + #[cfg_attr(feature = "schemars", schemars(with = "Option"))] + pub workflow_completion_retention: Option, +} + +// This type is used only for exposing the handler metadata, and not internally. See [ServiceAndHandlerType]. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] +pub enum HandlerMetadataType { + Exclusive, + Shared, + Workflow, +} + +impl From for HandlerMetadataType { + fn from(value: InvocationTargetType) -> Self { + match value { + InvocationTargetType::Service => HandlerMetadataType::Shared, + InvocationTargetType::VirtualObject(h_ty) => match h_ty { + VirtualObjectHandlerType::Exclusive => HandlerMetadataType::Exclusive, + VirtualObjectHandlerType::Shared => HandlerMetadataType::Shared, + }, + InvocationTargetType::Workflow(h_ty) => match h_ty { + WorkflowHandlerType::Workflow => HandlerMetadataType::Workflow, + WorkflowHandlerType::Shared => HandlerMetadataType::Shared, + }, + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] +pub struct HandlerMetadata { + pub name: String, + + pub ty: HandlerMetadataType, + + // # Human readable input description + // + // If empty, no schema was provided by the user at discovery time. + pub input_description: String, + + // # Human readable output description + // + // If empty, no schema was provided by the user at discovery time. + pub output_description: String, +} + +/// This API will return services registered by the user. +pub trait ServiceMetadataResolver { + fn resolve_latest_service(&self, service_name: impl AsRef) -> Option; + + fn resolve_latest_service_type(&self, service_name: impl AsRef) -> Option; + + fn list_services(&self) -> Vec; +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct HandlerSchemas { + pub target_meta: InvocationTargetMetadata, +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct ServiceSchemas { + pub revision: ServiceRevision, + pub handlers: HashMap, + pub ty: ServiceType, + pub location: ServiceLocation, + pub idempotency_retention: Duration, + pub workflow_completion_retention: Option, +} + +impl ServiceSchemas { + pub fn as_service_metadata(&self, name: String) -> ServiceMetadata { + ServiceMetadata { + name, + handlers: self + .handlers + .iter() + .map(|(h_name, h_schemas)| HandlerMetadata { + name: h_name.clone(), + ty: h_schemas.target_meta.target_ty.into(), + input_description: h_schemas.target_meta.input_rules.to_string(), + output_description: h_schemas.target_meta.output_rules.to_string(), + }) + .collect(), + ty: self.ty, + deployment_id: self.location.latest_deployment, + revision: self.revision, + public: self.location.public, + idempotency_retention: self.idempotency_retention.into(), + workflow_completion_retention: self.workflow_completion_retention.map(Into::into), + } + } +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct ServiceLocation { + pub latest_deployment: DeploymentId, + pub public: bool, +} + +impl ServiceMetadataResolver for Schema { + fn resolve_latest_service(&self, service_name: impl AsRef) -> Option { + let name = service_name.as_ref(); + self.use_service_schema(name, |service_schemas| { + service_schemas.as_service_metadata(name.to_owned()) + }) + } + + fn resolve_latest_service_type(&self, service_name: impl AsRef) -> Option { + self.use_service_schema(service_name.as_ref(), |service_schemas| service_schemas.ty) + } + + fn list_services(&self) -> Vec { + self.services + .iter() + .map(|(service_name, service_schemas)| { + service_schemas.as_service_metadata(service_name.clone()) + }) + .collect() + } +} + +impl ServiceMetadataResolver for UpdateableSchema { + fn resolve_latest_service(&self, service_name: impl AsRef) -> Option { + self.0.load().resolve_latest_service(service_name) + } + + fn resolve_latest_service_type(&self, service_name: impl AsRef) -> Option { + self.0.load().resolve_latest_service_type(service_name) + } + + fn list_services(&self) -> Vec { + self.0.load().list_services() + } +} + +#[cfg(feature = "test-util")] +#[allow(dead_code)] +pub mod test_util { + use super::*; + + use std::collections::HashMap; + + #[derive(Debug, Default, Clone)] + pub struct MockServiceMetadataResolver(HashMap); + + impl MockServiceMetadataResolver { + pub fn add(&mut self, service_metadata: ServiceMetadata) { + self.0 + .insert(service_metadata.name.clone(), service_metadata); + } + } + + impl ServiceMetadataResolver for MockServiceMetadataResolver { + fn resolve_latest_service(&self, service_name: impl AsRef) -> Option { + self.0.get(service_name.as_ref()).cloned() + } + + fn resolve_latest_service_type( + &self, + service_name: impl AsRef, + ) -> Option { + self.0.get(service_name.as_ref()).map(|c| c.ty) + } + + fn list_services(&self) -> Vec { + self.0.values().cloned().collect() + } + } + + impl ServiceMetadata { + pub fn mock_service( + name: impl AsRef, + handlers: impl IntoIterator>, + ) -> Self { + Self { + name: name.as_ref().to_string(), + handlers: handlers + .into_iter() + .map(|s| HandlerMetadata { + name: s.as_ref().to_string(), + ty: HandlerMetadataType::Shared, + input_description: "any".to_string(), + output_description: "any".to_string(), + }) + .collect(), + ty: ServiceType::Service, + deployment_id: Default::default(), + revision: 0, + public: true, + idempotency_retention: std::time::Duration::from_secs(60).into(), + workflow_completion_retention: None, + } + } + + pub fn mock_virtual_object( + name: impl AsRef, + handlers: impl IntoIterator>, + ) -> Self { + Self { + name: name.as_ref().to_string(), + handlers: handlers + .into_iter() + .map(|s| HandlerMetadata { + name: s.as_ref().to_string(), + ty: HandlerMetadataType::Exclusive, + input_description: "any".to_string(), + output_description: "any".to_string(), + }) + .collect(), + ty: ServiceType::VirtualObject, + deployment_id: Default::default(), + revision: 0, + public: true, + idempotency_retention: std::time::Duration::from_secs(60).into(), + workflow_completion_retention: None, + } + } + } +} diff --git a/crates/types/src/schema/subscriptions.rs b/crates/types/src/schema/subscriptions.rs new file mode 100644 index 0000000000..5ac5f11f00 --- /dev/null +++ b/crates/types/src/schema/subscriptions.rs @@ -0,0 +1,262 @@ +// Copyright (c) 2023 - Restate Software, Inc., Restate GmbH. +// All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +use std::collections::HashMap; +use std::fmt; + +use serde::Deserialize; +use serde::Serialize; +use tracing::warn; + +use super::{Schema, UpdateableSchema}; +use crate::config::IngressOptions; +use crate::errors::GenericError; +use crate::identifiers::SubscriptionId; + +#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] +#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] +pub enum Source { + Kafka { cluster: String, topic: String }, +} + +impl fmt::Display for Source { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Source::Kafka { cluster, topic, .. } => { + write!(f, "kafka://{}/{}", cluster, topic) + } + } + } +} + +impl PartialEq<&str> for Source { + fn eq(&self, other: &&str) -> bool { + self.to_string().as_str() == *other + } +} + +/// Specialized version of [super::service::ServiceType] +#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] +#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] +pub enum EventReceiverServiceType { + VirtualObject, + Workflow, + Service, +} + +#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] +#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] +pub enum Sink { + Service { + name: String, + handler: String, + ty: EventReceiverServiceType, + }, +} + +impl fmt::Display for Sink { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Sink::Service { name, handler, .. } => { + write!(f, "service://{}/{}", name, handler) + } + } + } +} + +impl PartialEq<&str> for Sink { + fn eq(&self, other: &&str) -> bool { + self.to_string().as_str() == *other + } +} + +#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] +#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] +pub struct Subscription { + id: SubscriptionId, + source: Source, + sink: Sink, + metadata: HashMap, +} + +impl Subscription { + pub fn new( + id: SubscriptionId, + source: Source, + sink: Sink, + metadata: HashMap, + ) -> Self { + Self { + id, + source, + sink, + metadata, + } + } + + pub fn id(&self) -> SubscriptionId { + self.id + } + + pub fn source(&self) -> &Source { + &self.source + } + + pub fn sink(&self) -> &Sink { + &self.sink + } + + pub fn metadata(&self) -> &HashMap { + &self.metadata + } + + pub fn metadata_mut(&mut self) -> &mut HashMap { + &mut self.metadata + } +} + +pub enum ListSubscriptionFilter { + ExactMatchSink(String), + ExactMatchSource(String), +} + +impl ListSubscriptionFilter { + pub fn matches(&self, sub: &Subscription) -> bool { + match self { + ListSubscriptionFilter::ExactMatchSink(sink) => sub.sink == sink.as_str(), + ListSubscriptionFilter::ExactMatchSource(source) => sub.source == source.as_str(), + } + } +} + +pub trait SubscriptionResolver { + fn get_subscription(&self, id: SubscriptionId) -> Option; + + fn list_subscriptions(&self, filters: &[ListSubscriptionFilter]) -> Vec; +} + +impl SubscriptionResolver for Schema { + fn get_subscription(&self, id: SubscriptionId) -> Option { + self.subscriptions.get(&id).cloned() + } + + fn list_subscriptions(&self, filters: &[ListSubscriptionFilter]) -> Vec { + self.subscriptions + .values() + .filter(|sub| { + for f in filters { + if !f.matches(sub) { + return false; + } + } + true + }) + .cloned() + .collect() + } +} + +impl SubscriptionResolver for UpdateableSchema { + fn get_subscription(&self, id: SubscriptionId) -> Option { + self.0.load().get_subscription(id) + } + + fn list_subscriptions(&self, filters: &[ListSubscriptionFilter]) -> Vec { + self.0.load().list_subscriptions(filters) + } +} + +pub trait SubscriptionValidator { + type Error: Into; + + fn validate(&self, subscription: Subscription) -> Result; +} + +#[derive(Debug, thiserror::Error)] +#[error("invalid option '{name}'. Reason: {reason}")] +pub struct ValidationError { + name: &'static str, + reason: &'static str, +} + +impl SubscriptionValidator for IngressOptions { + type Error = ValidationError; + + fn validate(&self, mut subscription: Subscription) -> Result { + // Retrieve the cluster option and merge them with subscription metadata + let Source::Kafka { cluster, .. } = subscription.source(); + let cluster_options = &self.get_kafka_cluster(cluster).ok_or(ValidationError { + name: "source", + reason: "specified cluster in the source URI does not exist. Make sure it is defined in the KafkaOptions", + })?.additional_options; + + if cluster_options.contains_key("enable.auto.commit") + || subscription.metadata().contains_key("enable.auto.commit") + { + warn!("The configuration option enable.auto.commit should not be set and it will be ignored."); + } + if cluster_options.contains_key("enable.auto.offset.store") + || subscription + .metadata() + .contains_key("enable.auto.offset.store") + { + warn!("The configuration option enable.auto.offset.store should not be set and it will be ignored."); + } + + // Set the group.id if unset + if !(cluster_options.contains_key("group.id") + || subscription.metadata().contains_key("group.id")) + { + let group_id = subscription.id().to_string(); + + subscription + .metadata_mut() + .insert("group.id".to_string(), group_id); + } + + // Set client.id if unset + if !(cluster_options.contains_key("client.id") + || subscription.metadata().contains_key("client.id")) + { + subscription + .metadata_mut() + .insert("client.id".to_string(), "restate".to_string()); + } + + Ok(subscription) + } +} + +#[cfg(feature = "test-util")] +pub mod mocks { + use std::str::FromStr; + + use super::*; + + impl Subscription { + pub fn mock() -> Self { + let id = SubscriptionId::from_str("sub_15VqmTOnXH3Vv2pl5HOG7Ua") + .expect("stable valid subscription id"); + Subscription { + id, + source: Source::Kafka { + cluster: "my-cluster".to_string(), + topic: "my-topic".to_string(), + }, + sink: Sink::Service { + name: "MySvc".to_string(), + handler: "MyMethod".to_string(), + ty: EventReceiverServiceType::Service, + }, + metadata: Default::default(), + } + } + } +} diff --git a/crates/types/src/version.rs b/crates/types/src/version.rs index f3c7529d93..4b64199802 100644 --- a/crates/types/src/version.rs +++ b/crates/types/src/version.rs @@ -42,6 +42,20 @@ impl Default for Version { } } +impl From for Version { + fn from(version: crate::protobuf::common::Version) -> Self { + crate::Version::from(version.value) + } +} + +impl From for crate::protobuf::common::Version { + fn from(version: Version) -> Self { + crate::protobuf::common::Version { + value: version.into(), + } + } +} + /// A trait for all metadata types that have a version. pub trait Versioned { /// Returns the version of the versioned value diff --git a/crates/worker/Cargo.toml b/crates/worker/Cargo.toml index 52a59f67a5..2cb4ebf589 100644 --- a/crates/worker/Cargo.toml +++ b/crates/worker/Cargo.toml @@ -31,11 +31,8 @@ restate-invoker-api = { workspace = true } restate-invoker-impl = { workspace = true } restate-metadata-store = { workspace = true } restate-network = { workspace = true } -restate-node-protocol = { workspace = true } restate-partition-store = { workspace = true } restate-rocksdb = { workspace = true } -restate-schema = { workspace = true } -restate-schema-api = { workspace = true, features = [ "service", "subscription"] } restate-serde-util = { workspace = true, features = ["proto"] } restate-service-client = { workspace = true } restate-service-protocol = { workspace = true, features = [ "codec", "awakeable-id", "message" ] } @@ -77,7 +74,6 @@ tracing-opentelemetry = { workspace = true } restate-bifrost = { workspace = true, features = ["test-util"] } restate-core = { workspace = true, features = ["test-util"] } restate-rocksdb = { workspace = true, features = ["test-util"] } -restate-schema-api = { workspace = true, features = ["test-util"] } restate-service-protocol = { workspace = true, features = ["test-util"] } restate-storage-api = { workspace = true, features = ["test-util"] } restate-test-util = { workspace = true, features = ["prost"] } diff --git a/crates/worker/src/invoker_integration.rs b/crates/worker/src/invoker_integration.rs index cb62df4148..a6cc93ea02 100644 --- a/crates/worker/src/invoker_integration.rs +++ b/crates/worker/src/invoker_integration.rs @@ -8,10 +8,14 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. +use std::marker::PhantomData; +use std::str::FromStr; + use anyhow::anyhow; use assert2::let_assert; use bytes::Bytes; use bytestring::ByteString; + use restate_service_protocol::awakeable_id::AwakeableIdentifier; use restate_types::errors::{codes, InvocationError}; use restate_types::identifiers::InvocationId; @@ -24,8 +28,7 @@ use restate_types::journal::enriched::{ use restate_types::journal::raw::{PlainEntryHeader, PlainRawEntry, RawEntry, RawEntryCodec}; use restate_types::journal::{CompleteAwakeableEntry, Entry, InvokeEntry, OneWayCallEntry}; use restate_types::journal::{EntryType, InvokeRequest}; -use std::marker::PhantomData; -use std::str::FromStr; +use restate_types::schema::invocation_target::InvocationTargetResolver; #[derive(Debug, Clone)] pub(super) struct EntryEnricher { @@ -45,7 +48,7 @@ impl EntryEnricher { impl EntryEnricher where - Schemas: restate_schema_api::invocation_target::InvocationTargetResolver, + Schemas: InvocationTargetResolver, Codec: RawEntryCodec, { fn resolve_service_invocation_target( @@ -111,7 +114,7 @@ where impl restate_invoker_api::EntryEnricher for EntryEnricher where - Schemas: restate_schema_api::invocation_target::InvocationTargetResolver, + Schemas: InvocationTargetResolver, Codec: RawEntryCodec, { fn enrich_entry( diff --git a/crates/worker/src/lib.rs b/crates/worker/src/lib.rs index ab2019fad3..101e606b0e 100644 --- a/crates/worker/src/lib.rs +++ b/crates/worker/src/lib.rs @@ -20,14 +20,11 @@ mod partition_processor_manager; mod subscription_controller; mod subscription_integration; -pub use error::*; -pub use handle::*; use restate_types::arc_util::ArcSwapExt; use restate_types::config::UpdateableConfiguration; pub use subscription_controller::SubscriptionController; pub use subscription_integration::SubscriptionControllerHandle; -use crate::ingress_integration::InvocationStorageReaderImpl; use codederror::CodedError; use restate_bifrost::Bifrost; use restate_core::network::MessageRouterBuilder; @@ -41,11 +38,14 @@ use restate_invoker_impl::{ use restate_metadata_store::MetadataStoreClient; use restate_network::Networking; use restate_partition_store::{PartitionStore, PartitionStoreManager}; -use restate_schema::UpdateableSchema; use restate_service_protocol::codec::ProtobufRawEntryCodec; use restate_storage_query_datafusion::context::QueryContext; use restate_storage_query_postgres::service::PostgresQueryService; +use restate_types::schema::UpdateableSchema; +pub use self::error::*; +pub use self::handle::*; +use crate::ingress_integration::InvocationStorageReaderImpl; use crate::invoker_integration::EntryEnricher; use crate::partition::storage::invoker::InvokerStorageReader; use crate::partition_processor_manager::PartitionProcessorManager; diff --git a/crates/worker/src/partition/leadership/mod.rs b/crates/worker/src/partition/leadership/mod.rs index baf33d4ff4..48f9c7cd80 100644 --- a/crates/worker/src/partition/leadership/mod.rs +++ b/crates/worker/src/partition/leadership/mod.rs @@ -20,8 +20,8 @@ use restate_core::{ }; use restate_invoker_api::InvokeInputJournal; use restate_network::Networking; -use restate_node_protocol::ingress; use restate_timer::TokioClock; +use restate_types::net::ingress; use std::fmt::Debug; use std::ops::RangeInclusive; use std::pin::Pin; diff --git a/crates/worker/src/partition/mod.rs b/crates/worker/src/partition/mod.rs index 53d635b663..d62300e6c8 100644 --- a/crates/worker/src/partition/mod.rs +++ b/crates/worker/src/partition/mod.rs @@ -21,8 +21,8 @@ use metrics::{counter, histogram}; use restate_core::metadata; use restate_network::Networking; use restate_partition_store::{PartitionStore, RocksDBTransaction}; +use restate_types::cluster::cluster_state::{PartitionProcessorStatus, ReplayStatus, RunMode}; use restate_types::identifiers::{PartitionId, PartitionKey}; -use restate_types::processors::{PartitionProcessorStatus, ReplayStatus, RunMode}; use restate_types::time::MillisSinceEpoch; use std::fmt::Debug; use std::marker::PhantomData; @@ -141,9 +141,8 @@ where self.status.replay_status = ReplayStatus::Active; } else { // catching up. - self.status.replay_status = ReplayStatus::CatchingUp { - target_tail_lsn: current_tail.unwrap(), - } + self.status.target_tail_lsn = current_tail; + self.status.replay_status = ReplayStatus::CatchingUp; } let mut log_reader = bifrost @@ -324,10 +323,8 @@ where status.last_applied_log_lsn = Some(record.0); status.last_record_applied_at = Some(MillisSinceEpoch::now()); match status.replay_status { - ReplayStatus::CatchingUp { + ReplayStatus::CatchingUp if status.target_tail_lsn.is_some_and(|v| record.0 >= v) => { // finished catching up - target_tail_lsn, - } if record.0 >= target_tail_lsn => { status.replay_status = ReplayStatus::Active; } _ => {} diff --git a/crates/worker/src/partition_processor_manager.rs b/crates/worker/src/partition_processor_manager.rs index fa9cabbf74..51cab822f7 100644 --- a/crates/worker/src/partition_processor_manager.rs +++ b/crates/worker/src/partition_processor_manager.rs @@ -17,14 +17,6 @@ use futures::future::OptionFuture; use futures::stream::BoxStream; use futures::stream::StreamExt; use metrics::gauge; -use restate_core::network::NetworkSender; -use restate_core::TaskCenter; -use restate_network::rpc_router::{RpcError, RpcRouter}; -use restate_node_protocol::partition_processor_manager::GetProcessorsState; -use restate_node_protocol::partition_processor_manager::ProcessorsStateResponse; -use restate_node_protocol::RpcMessage; -use restate_types::processors::ReplayStatus; -use restate_types::processors::{PartitionProcessorStatus, RunMode}; use tokio::sync::{mpsc, watch}; use tokio::time; use tokio::time::MissedTickBehavior; @@ -32,17 +24,19 @@ use tracing::{debug, info, trace, warn}; use restate_bifrost::Bifrost; use restate_core::network::MessageRouterBuilder; +use restate_core::network::NetworkSender; use restate_core::worker_api::{ProcessorsManagerCommand, ProcessorsManagerHandle}; +use restate_core::TaskCenter; use restate_core::{cancellation_watcher, Metadata, ShutdownError, TaskId, TaskKind}; use restate_invoker_impl::InvokerHandle; use restate_metadata_store::{MetadataStoreClient, ReadModifyWriteError}; +use restate_network::rpc_router::{RpcError, RpcRouter}; use restate_network::Networking; -use restate_node_protocol::cluster_controller::AttachRequest; -use restate_node_protocol::cluster_controller::{Action, AttachResponse}; -use restate_node_protocol::MessageEnvelope; use restate_partition_store::{OpenMode, PartitionStore, PartitionStoreManager}; use restate_storage_api::StorageError; use restate_types::arc_util::{ArcSwapExt, Updateable}; +use restate_types::cluster::cluster_state::ReplayStatus; +use restate_types::cluster::cluster_state::{PartitionProcessorStatus, RunMode}; use restate_types::config::{ Configuration, StorageOptions, UpdateableConfiguration, WorkerOptions, }; @@ -50,6 +44,12 @@ use restate_types::epoch::EpochMetadata; use restate_types::identifiers::{LeaderEpoch, PartitionId, PartitionKey}; use restate_types::logs::{LogId, Lsn, Payload, SequenceNumber}; use restate_types::metadata_store::keys::partition_processor_epoch_key; +use restate_types::net::cluster_controller::AttachRequest; +use restate_types::net::cluster_controller::{Action, AttachResponse}; +use restate_types::net::partition_processor_manager::GetProcessorsState; +use restate_types::net::partition_processor_manager::ProcessorsStateResponse; +use restate_types::net::MessageEnvelope; +use restate_types::net::RpcMessage; use restate_types::time::MillisSinceEpoch; use restate_types::GenerationalNodeId; use restate_wal_protocol::control::AnnounceLeader; diff --git a/crates/worker/src/subscription_controller.rs b/crates/worker/src/subscription_controller.rs index f1f8133800..2294baeea5 100644 --- a/crates/worker/src/subscription_controller.rs +++ b/crates/worker/src/subscription_controller.rs @@ -10,8 +10,8 @@ use std::future::Future; -use restate_schema_api::subscription::Subscription; use restate_types::identifiers::SubscriptionId; +use restate_types::schema::subscriptions::Subscription; use crate::WorkerHandleError; diff --git a/crates/worker/src/subscription_integration.rs b/crates/worker/src/subscription_integration.rs index a960f0378d..cf2e8df00d 100644 --- a/crates/worker/src/subscription_integration.rs +++ b/crates/worker/src/subscription_integration.rs @@ -8,13 +8,15 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. -use crate::{SubscriptionController, WorkerHandleError}; +use std::ops::Deref; +use std::sync::Arc; + use restate_ingress_kafka::SubscriptionCommandSender; -use restate_schema_api::subscription::{Subscription, SubscriptionValidator}; use restate_types::config::IngressOptions; use restate_types::identifiers::SubscriptionId; -use std::ops::Deref; -use std::sync::Arc; +use restate_types::schema::subscriptions::{Subscription, SubscriptionValidator}; + +use crate::{SubscriptionController, WorkerHandleError}; #[derive(Debug, Clone)] pub struct SubscriptionControllerHandle(Arc, SubscriptionCommandSender); diff --git a/tools/restatectl/Cargo.toml b/tools/restatectl/Cargo.toml index eb27e63ee7..5a71b1b78a 100644 --- a/tools/restatectl/Cargo.toml +++ b/tools/restatectl/Cargo.toml @@ -8,9 +8,8 @@ license.workspace = true publish = false [dependencies] +restate-admin = { workspace = true, features = ["clients"] } restate-cli-util = { workspace = true } -restate-node-protocol = { workspace = true } -restate-node-services = { workspace = true, features = ["clients"] } restate-types = { workspace = true } anyhow = { workspace = true } diff --git a/tools/restatectl/src/commands/dump/cluster_state.rs b/tools/restatectl/src/commands/dump/cluster_state.rs index 3d16d6e0e7..391bff989e 100644 --- a/tools/restatectl/src/commands/dump/cluster_state.rs +++ b/tools/restatectl/src/commands/dump/cluster_state.rs @@ -14,15 +14,16 @@ use std::time::SystemTime; use anyhow::Context; use chrono::{DateTime, Local}; use cling::prelude::*; +use restate_admin::cluster_controller::protobuf::cluster_ctrl_svc_client::ClusterCtrlSvcClient; +use restate_admin::cluster_controller::protobuf::ClusterStateRequest; use restate_cli_util::_comfy_table::{Attribute, Cell, Color, Table}; use restate_cli_util::ui::console::StyledTable; use restate_cli_util::ui::{timestamp_as_human_duration, Tense}; -use restate_node_protocol::common::Lsn; -use restate_node_services::cluster_ctrl::cluster_ctrl_svc_client::ClusterCtrlSvcClient; +use restate_types::logs::Lsn; use restate_cli_util::{c_println, c_title}; -use restate_node_services::cluster_ctrl::{ - node_state, ClusterStateRequest, DeadNode, PartitionProcessorStatus, ReplayStatus, RunMode, +use restate_types::protobuf::cluster::{ + node_state, DeadNode, PartitionProcessorStatus, ReplayStatus, RunMode, }; use restate_types::{GenerationalNodeId, PlainNodeId}; use tonic::codec::CompressionEncoding; @@ -56,7 +57,12 @@ async fn dump_cluster_state( ClusterCtrlSvcClient::new(channel).accept_compressed(CompressionEncoding::Gzip); let req = ClusterStateRequest::default(); - let state = client.get_cluster_state(req).await?.into_inner(); + let state = client + .get_cluster_state(req) + .await? + .into_inner() + .cluster_state + .ok_or_else(|| anyhow::anyhow!("no cluster state returned"))?; let mut processors: BTreeMap = BTreeMap::new(); let mut dead_nodes: BTreeMap = BTreeMap::new(); @@ -102,7 +108,7 @@ async fn dump_cluster_state( ), render_replay_status( details.status.replay_status(), - details.status.target_tail_lsn, + details.status.target_tail_lsn.map(Into::into), ), Cell::new( details diff --git a/tools/xtask/Cargo.toml b/tools/xtask/Cargo.toml index 203f7cf2a1..e687a8571f 100644 --- a/tools/xtask/Cargo.toml +++ b/tools/xtask/Cargo.toml @@ -12,8 +12,7 @@ restate-admin = { workspace = true, features = ["options_schema"] } restate-bifrost = { workspace = true, features = ["options_schema", "test-util"] } restate-core = { workspace = true, features = ["test-util"] } restate-metadata-store = { workspace = true, features = ["test-util"] } -restate-node-services = { workspace = true, features = ["clients"] } -restate-schema-api = { workspace = true, features = ["subscription"] } +restate-network = { workspace = true } restate-service-client = { workspace = true } restate-service-protocol = { workspace = true, features = ["discovery"]} restate-storage-query-datafusion = { workspace = true, features = ["table_docs"] } diff --git a/tools/xtask/src/main.rs b/tools/xtask/src/main.rs index c083f6bdb9..5e9db89b3a 100644 --- a/tools/xtask/src/main.rs +++ b/tools/xtask/src/main.rs @@ -8,14 +8,20 @@ // the Business Source License, use of this software will be governed // by the Apache License, Version 2.0. +use std::io::Write; +use std::time::Duration; +use std::{env, io}; + use anyhow::bail; use reqwest::header::ACCEPT; +use schemars::gen::SchemaSettings; +use tonic::transport::{Channel, Uri}; + use restate_admin::service::AdminService; use restate_bifrost::Bifrost; use restate_core::TaskKind; use restate_core::TestCoreEnv; -use restate_node_services::node_svc::node_svc_client::NodeSvcClient; -use restate_schema_api::subscription::Subscription; +use restate_network::protobuf::node_svc::node_svc_client::NodeSvcClient; use restate_service_client::{AssumeRoleCacheMode, ServiceClient}; use restate_service_protocol::discovery::ServiceDiscovery; use restate_storage_query_datafusion::table_docs; @@ -24,15 +30,12 @@ use restate_types::config::Configuration; use restate_types::identifiers::SubscriptionId; use restate_types::invocation::InvocationTermination; use restate_types::retries::RetryPolicy; +use restate_types::schema::subscriptions::Subscription; +use restate_types::schema::subscriptions::SubscriptionValidator; use restate_types::state_mut::ExternalStateMutation; use restate_worker::SubscriptionController; use restate_worker::WorkerHandle; use restate_worker::WorkerHandleError; -use schemars::gen::SchemaSettings; -use std::io::Write; -use std::time::Duration; -use std::{env, io}; -use tonic::transport::{Channel, Uri}; fn generate_config_schema() -> anyhow::Result<()> { let schema = SchemaSettings::draft2019_09() @@ -85,7 +88,7 @@ impl SubscriptionController for Mock { } } -impl restate_schema_api::subscription::SubscriptionValidator for Mock { +impl SubscriptionValidator for Mock { type Error = WorkerHandleError; fn validate(&self, _: Subscription) -> Result {