From 643ac1b27c82ea8b0ea63947c7e629c60cef8916 Mon Sep 17 00:00:00 2001 From: Steven Johnson Date: Thu, 31 Aug 2023 02:40:15 +0700 Subject: [PATCH] feat(cat-data-service): More wip initial poem integration work --- Cargo.lock | 55 +++++- Cargo.toml | 8 +- src/cat-data-service/Cargo.toml | 24 ++- .../src/service/api/health/live_get.rs | 0 .../src/service/api/health/mod.rs | 41 ++++ .../src/service/api/health/ready_get.rs | 1 + .../src/service/{api.rs => api/mod.rs} | 12 +- src/cat-data-service/src/service/docs/mod.rs | 5 +- .../src/service/generic/mod.rs | 4 + .../src/service/generic/objects/mod.rs | 1 + .../src/service/generic/responses/mod.rs | 6 + .../src/service/generic/responses/resp_2xx.rs | 12 ++ .../src/service/generic/responses/resp_4xx.rs | 37 ++++ .../src/service/generic/responses/resp_5xx.rs | 64 +++++++ src/cat-data-service/src/service/mod.rs | 2 +- .../src/service/poem_service.rs | 92 ++++++++- .../src/service/utilities/metrics_tracing.rs | 22 +-- src/cat-data-service/src/settings.rs | 179 ++++++++++++++++++ src/cat-data-service/src/state/mod.rs | 11 +- src/event-db/Cargo.toml | 2 +- 20 files changed, 526 insertions(+), 52 deletions(-) create mode 100644 src/cat-data-service/src/service/api/health/live_get.rs create mode 100644 src/cat-data-service/src/service/api/health/mod.rs create mode 100644 src/cat-data-service/src/service/api/health/ready_get.rs rename src/cat-data-service/src/service/{api.rs => api/mod.rs} (77%) create mode 100644 src/cat-data-service/src/service/generic/mod.rs create mode 100644 src/cat-data-service/src/service/generic/objects/mod.rs create mode 100644 src/cat-data-service/src/service/generic/responses/mod.rs create mode 100644 src/cat-data-service/src/service/generic/responses/resp_2xx.rs create mode 100644 src/cat-data-service/src/service/generic/responses/resp_4xx.rs create mode 100644 src/cat-data-service/src/service/generic/responses/resp_5xx.rs diff --git a/Cargo.lock b/Cargo.lock index 324893535e..3893d41afe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -894,14 +894,18 @@ dependencies = [ "chrono", "clap 4.2.1", "cryptoxide 0.4.4", + "dotenvy", "event-db", "jormungandr-lib", + "lazy_static", "metrics", "metrics-exporter-prometheus", "once_cell", "opentelemetry", "opentelemetry-prometheus 0.13.0", + "panic-message", "poem", + "poem-extensions", "poem-openapi", "quickcheck", "quickcheck_macros", @@ -915,6 +919,7 @@ dependencies = [ "tower-http", "tracing", "tracing-subscriber", + "url", "uuid 1.3.1", ] @@ -2519,9 +2524,9 @@ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] name = "form_urlencoded" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9c384f161156f5260c24a097c56119f9be8c798586aecc13afbcbe7b7e26bf8" +checksum = "a62bc1cf6f830c2ec14a513a9fb124d0a213a629668a4186f329db21fe045652" dependencies = [ "percent-encoding", ] @@ -3302,9 +3307,9 @@ dependencies = [ [[package]] name = "idna" -version = "0.3.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6" +checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" dependencies = [ "unicode-bidi", "unicode-normalization", @@ -5079,6 +5084,12 @@ version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f" +[[package]] +name = "panic-message" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384e52fd8fbd4cbe3c317e8216260c21a0f9134de108cea8a4dd4e7e152c472d" + [[package]] name = "parity-multiaddr" version = "0.11.2" @@ -5206,9 +5217,9 @@ dependencies = [ [[package]] name = "percent-encoding" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" +checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" [[package]] name = "pest" @@ -5417,6 +5428,30 @@ dependencies = [ "syn 2.0.16", ] +[[package]] +name = "poem-extensions" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97ba49b672c1b387c724a20fd6fd6d629a5423c17c5f1c381763b1c899fec35e" +dependencies = [ + "poem", + "poem-extensions-macro", + "poem-openapi", +] + +[[package]] +name = "poem-extensions-macro" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f06eadfdec3a3a90626f990f2bf1668b857f0219b07eedabba3bf368620ce8e7" +dependencies = [ + "darling 0.20.3", + "proc-macro2", + "quote", + "syn 2.0.16", + "thiserror", +] + [[package]] name = "poem-openapi" version = "3.0.3" @@ -5439,6 +5474,8 @@ dependencies = [ "serde_yaml 0.9.25", "thiserror", "tokio", + "url", + "uuid 1.3.1", ] [[package]] @@ -8183,12 +8220,12 @@ dependencies = [ [[package]] name = "url" -version = "2.3.1" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d68c799ae75762b8c3fe375feb6600ef5602c883c5d21eb51c09f22b83c4643" +checksum = "143b538f18257fac9cad154828a57c6bf5157e1aa604d4816b5995bf6de87ae5" dependencies = [ "form_urlencoded", - "idna 0.3.0", + "idna 0.4.0", "percent-encoding", "serde", ] diff --git a/Cargo.toml b/Cargo.toml index a796a5c5b1..bcc3a77d71 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -119,4 +119,10 @@ warp-reverse-proxy = { version = "0.3", default-features = false, features = ["r once_cell = "1" # Use this for Blake2 -cryptoxide = "0.4.4" \ No newline at end of file +cryptoxide = "0.4.4" + +lazy_static = "1.4" + +url = "2.4.1" + +dotenvy = "0.15" \ No newline at end of file diff --git a/src/cat-data-service/Cargo.toml b/src/cat-data-service/Cargo.toml index 5f17030951..30995d9dea 100644 --- a/src/cat-data-service/Cargo.toml +++ b/src/cat-data-service/Cargo.toml @@ -10,7 +10,7 @@ event-db = { path = "../event-db" } clap = { workspace = true } tracing = { workspace = true } -tracing-subscriber = { workspace = true, features = ["fmt", "json"]} +tracing-subscriber = { workspace = true, features = ["fmt", "json"] } serde = { version = "1.0", features = ["derive"] } serde_json = { version = "1.0" } @@ -24,7 +24,7 @@ metrics-exporter-prometheus = { version = "0.12.1" } tower-http = { version = "0.4", features = ["cors"] } -rust_decimal = { workspace = true } +rust_decimal = { workspace = true } chrono = { workspace = true } @@ -32,10 +32,18 @@ jormungandr-lib = { workspace = true, optional = true } chain-impl-mockchain = { workspace = true, optional = true } poem = { version = "1.3.57", features = ["opentelemetry", "prometheus"] } -poem-openapi = { version = "3.0.3", features = ["openapi-explorer", "rapidoc", "redoc", "swagger-ui"] } +poem-openapi = { version = "3.0.3", features = [ + "openapi-explorer", + "rapidoc", + "redoc", + "swagger-ui", + "uuid", + "url", +] } +poem-extensions = { version = "0.7.2" } # Metrics - Poem -opentelemetry-prometheus = { version = "0.13"} +opentelemetry-prometheus = { version = "0.13" } opentelemetry = { workspace = true } once_cell = { workspace = true } @@ -44,6 +52,14 @@ cryptoxide = { workspace = true } uuid = { workspace = true } +lazy_static = { workspace = true } + +url = { workspace = true } + +dotenvy = { workspace = true } + +panic-message = "0.3" + [dev-dependencies] tower = { version = "0.4", features = ["util"] } quickcheck = { version = "0.9" } diff --git a/src/cat-data-service/src/service/api/health/live_get.rs b/src/cat-data-service/src/service/api/health/live_get.rs new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/cat-data-service/src/service/api/health/mod.rs b/src/cat-data-service/src/service/api/health/mod.rs new file mode 100644 index 0000000000..07bc5275db --- /dev/null +++ b/src/cat-data-service/src/service/api/health/mod.rs @@ -0,0 +1,41 @@ +//mod live_get; +//mod ready_get; + +use crate::service::generic::responses::resp_5xx::ServiceUnavailable; +use crate::service::generic::responses::{resp_2xx::NoContent, resp_5xx::ServerError}; + +use poem_openapi::OpenApi; + +use poem_extensions::{ + response, + UniResponse::{T204, T503}, +}; + +pub(crate) struct HealthApi; + +#[OpenApi] +impl HealthApi { + #[oai(path = "/health/ready", method = "get")] + async fn health_get( + &self, + ) -> response! { + 204: NoContent, + 500: ServerError, + 503: ServiceUnavailable, + } { + T204(NoContent) + } + + #[oai(path = "/health/live", method = "get")] + async fn live_get( + &self, + ) -> response! { + 204: NoContent, + 500: ServerError, + 503: ServiceUnavailable, + } { + // Return No Content unless any endpoint panics. + // If there are x panics in a time frame, say the service is unavailable to force a restart. + T503(ServiceUnavailable) + } +} diff --git a/src/cat-data-service/src/service/api/health/ready_get.rs b/src/cat-data-service/src/service/api/health/ready_get.rs new file mode 100644 index 0000000000..7db7d7f98d --- /dev/null +++ b/src/cat-data-service/src/service/api/health/ready_get.rs @@ -0,0 +1 @@ +//pub fn endpoint() -> \ No newline at end of file diff --git a/src/cat-data-service/src/service/api.rs b/src/cat-data-service/src/service/api/mod.rs similarity index 77% rename from src/cat-data-service/src/service/api.rs rename to src/cat-data-service/src/service/api/mod.rs index 2580df54d8..ab8c37ac07 100644 --- a/src/cat-data-service/src/service/api.rs +++ b/src/cat-data-service/src/service/api/mod.rs @@ -2,11 +2,15 @@ //! //! This defines all endpoints for the Catalyst Data Service API. //! It however does NOT contain any processing for them, that is defined elsewhere. -use poem::Route; +use health::HealthApi; use poem_openapi::{param::Query, payload::PlainText, OpenApi, OpenApiService}; use std::net::SocketAddr; -pub struct Api; +mod health; + +pub(crate) type OpenApiServiceT = OpenApiService<(Api, HealthApi), ()>; + +pub(crate) struct Api; #[OpenApi] impl Api { @@ -22,9 +26,9 @@ impl Api { } } -pub(crate) fn mk_api(addr: &SocketAddr) -> OpenApiService { +pub(crate) fn mk_api(addr: &SocketAddr) -> OpenApiServiceT { // This should be the actual hostname of the service. But in the absence of that, the IP address/port will do. let server_host = format!("http://{}:{}/api", addr.ip(), addr.port()); - OpenApiService::new(Api, "Hello World 2", "1.0").server(server_host) + OpenApiService::new((Api, HealthApi), "Catalyst Data Service", "1.2").server(server_host) } diff --git a/src/cat-data-service/src/service/docs/mod.rs b/src/cat-data-service/src/service/docs/mod.rs index 970bfe409e..8a1a6562a8 100644 --- a/src/cat-data-service/src/service/docs/mod.rs +++ b/src/cat-data-service/src/service/docs/mod.rs @@ -1,10 +1,9 @@ mod stoplight_elements; use poem::{get, Route}; -use poem_openapi::OpenApiService; -use super::api::Api; +use super::api::OpenApiServiceT; -pub(crate) fn docs(api_service: &OpenApiService) -> Route { +pub(crate) fn docs(api_service: &OpenApiServiceT) -> Route { let spec = api_service.spec(); let swagger_ui = api_service.swagger_ui(); diff --git a/src/cat-data-service/src/service/generic/mod.rs b/src/cat-data-service/src/service/generic/mod.rs new file mode 100644 index 0000000000..a9df442684 --- /dev/null +++ b/src/cat-data-service/src/service/generic/mod.rs @@ -0,0 +1,4 @@ +//! Define generic reusable api components here. +//! these components should be structured into their own sub modules. +//! +pub(crate) mod responses; diff --git a/src/cat-data-service/src/service/generic/objects/mod.rs b/src/cat-data-service/src/service/generic/objects/mod.rs new file mode 100644 index 0000000000..cdcb79c8a0 --- /dev/null +++ b/src/cat-data-service/src/service/generic/objects/mod.rs @@ -0,0 +1 @@ +//! This module contains generic re-usable objects. diff --git a/src/cat-data-service/src/service/generic/responses/mod.rs b/src/cat-data-service/src/service/generic/responses/mod.rs new file mode 100644 index 0000000000..4c3fa4d191 --- /dev/null +++ b/src/cat-data-service/src/service/generic/responses/mod.rs @@ -0,0 +1,6 @@ +//! Generic Responses are all contained in their own modules, grouped by response codes. +//! + +pub(crate) mod resp_2xx; +pub(crate) mod resp_4xx; +pub(crate) mod resp_5xx; diff --git a/src/cat-data-service/src/service/generic/responses/resp_2xx.rs b/src/cat-data-service/src/service/generic/responses/resp_2xx.rs new file mode 100644 index 0000000000..ba2dbb6c5f --- /dev/null +++ b/src/cat-data-service/src/service/generic/responses/resp_2xx.rs @@ -0,0 +1,12 @@ +//! This module contains generic re-usable responses with a 2xx response code. +//! + +use poem_extensions::OneResponse; + +#[derive(OneResponse)] +#[oai(status = 200)] +pub(crate) struct EmptyOK; + +#[derive(OneResponse)] +#[oai(status = 204)] +pub(crate) struct NoContent; diff --git a/src/cat-data-service/src/service/generic/responses/resp_4xx.rs b/src/cat-data-service/src/service/generic/responses/resp_4xx.rs new file mode 100644 index 0000000000..66b09c26ca --- /dev/null +++ b/src/cat-data-service/src/service/generic/responses/resp_4xx.rs @@ -0,0 +1,37 @@ +//! This module contains generic re-usable responses with a 4xx response code. +//! + +use poem::IntoResponse; +use poem_extensions::OneResponse; +use poem_openapi::payload::Payload; + +#[derive(OneResponse)] +#[oai(status = 400)] +pub(crate) struct BadRequest(T); + +#[derive(OneResponse)] +#[oai(status = 401)] +pub(crate) struct Unauthorized; + +#[derive(OneResponse)] +#[oai(status = 403)] +pub(crate) struct Forbidden; + +#[derive(OneResponse)] +#[oai(status = 404)] +pub(crate) struct NotFound; + +#[derive(OneResponse)] +#[oai(status = 405)] +pub(crate) struct MethodNotAllowed; + +#[derive(OneResponse)] +#[oai(status = 406)] +pub(crate) struct NotAcceptable; + +#[derive(OneResponse)] +#[oai(status = 422)] +/// Common automatically produced validation error for every endpoint. +/// Is generated automatically when any of the OpenAPI validation rules fail. +/// Can also be generated manually. +pub(crate) struct ValidationError; diff --git a/src/cat-data-service/src/service/generic/responses/resp_5xx.rs b/src/cat-data-service/src/service/generic/responses/resp_5xx.rs new file mode 100644 index 0000000000..a53373db19 --- /dev/null +++ b/src/cat-data-service/src/service/generic/responses/resp_5xx.rs @@ -0,0 +1,64 @@ +//! This module contains generic re-usable responses with a 4xx response code. +//! + +use poem_extensions::OneResponse; +use poem_openapi::payload::Json; +use poem_openapi::types::Example; +use poem_openapi::Object; +use url::Url; +use uuid::Uuid; + +use crate::settings::generate_github_issue_url; + +#[derive(Debug, Object)] +#[oai(example, skip_serializing_if_is_none)] +/// Response payload to a Bad request. +pub(crate) struct ServerErrorPayload { + /// Unique ID of this Server Error so that it can be located easily for debugging. + pub id: Uuid, + /// *Optional* SHORT Error message. + /// Will not contain sensitive information, internal details or backtraces. + msg: Option, + /// A URL to report an issue. + issue: Option, +} + +impl ServerErrorPayload { + /// Create a new Server Error Response Payload. + pub(crate) fn new(msg: Option) -> Self { + let id = Uuid::new_v4(); + let issue_title = format!("Internal Server Error - {id}"); + let issue = generate_github_issue_url(&issue_title); + + Self { id, msg, issue } + } +} + +impl Example for ServerErrorPayload { + /// Example for the Server Error Payload. + fn example() -> Self { + Self::new(Some("Server Error".to_string())) + } +} + +#[derive(OneResponse)] +#[oai(status = 500)] +/// Internal Server Error +pub(crate) struct ServerError(Json); + +impl ServerError { + /// Create a new Server Error Response. + pub(crate) fn new(msg: Option) -> Self { + Self(Json(ServerErrorPayload::new(msg))) + } + + /// Get the id of this Server Error. + pub(crate) fn id(&self) -> Uuid { + self.0.id + } +} + +#[derive(OneResponse)] +#[oai(status = 503)] +/// The service is not available. +pub(crate) struct ServiceUnavailable; diff --git a/src/cat-data-service/src/service/mod.rs b/src/cat-data-service/src/service/mod.rs index 04224a1475..1078153b38 100644 --- a/src/cat-data-service/src/service/mod.rs +++ b/src/cat-data-service/src/service/mod.rs @@ -10,13 +10,13 @@ use axum::{ Json, Router, }; use metrics_exporter_prometheus::{Matcher, PrometheusBuilder, PrometheusHandle}; -use poem::{middleware::OpenTelemetryMetrics, EndpointExt, IntoEndpoint}; use serde::Serialize; use std::{future::ready, net::SocketAddr, sync::Arc, time::Instant}; use tower_http::cors::{Any, CorsLayer}; mod api; mod docs; +mod generic; mod health; mod poem_service; mod utilities; diff --git a/src/cat-data-service/src/service/poem_service.rs b/src/cat-data-service/src/service/poem_service.rs index 9307b71a7e..7536ede84d 100644 --- a/src/cat-data-service/src/service/poem_service.rs +++ b/src/cat-data-service/src/service/poem_service.rs @@ -6,11 +6,92 @@ use crate::service::docs::docs; use crate::service::Error; use crate::service::api::mk_api; +use crate::service::generic::responses::resp_5xx::ServerError; use crate::service::utilities::metrics_tracing::{init_prometheus, log_requests}; -use poem::middleware::{Cors, OpenTelemetryMetrics}; -use poem::{endpoint::PrometheusExporter, listener::TcpListener, EndpointExt, Route}; +use panic_message::panic_message; +use poem::endpoint::PrometheusExporter; +use poem::listener::TcpListener; +use poem::middleware::{CatchPanic, Cors, OpenTelemetryMetrics, PanicHandler}; +use poem::{EndpointExt, Route}; +use std::any::Any; +use std::cell::RefCell; use std::net::SocketAddr; +use tracing::log::error; + +use std::backtrace::Backtrace; + +/// Customized Panic handler. +/// Catches all panics, and turns them into 500. +/// Does not crash the service, BUT will set it to NOT LIVE. +/// Logs the panic as an error. +/// This should cause Kubernetes to restart the service. +#[derive(Clone)] +struct ServicePanicHandler; + +// Customized Panic handler - data storage. +// Allows us to catch the backtrace so we can include it in logs. +thread_local! { + static BACKTRACE: RefCell> = RefCell::new(None); + static LOCATION: RefCell> = RefCell::new(None); +} + +/// Sets a custom panic hook to capture the Backtrace and Panic Location for logging purposes. +/// This hook gets called BEFORE we catch it. So the thread local variables stored here are +/// valid when processing the panic capture. +fn set_panic_hook() { + std::panic::set_hook(Box::new(|panic_info| { + // Get the backtrace and format it. + let raw_trace = Backtrace::force_capture(); + let trace = format!("{raw_trace}"); + BACKTRACE.with(move |b| b.borrow_mut().replace(trace)); + + // Get the location and format it. + let location = match panic_info.location() { + Some(location) => format!("{location}"), + None => "Unknown".to_string(), + }; + LOCATION.with(move |l| l.borrow_mut().replace(location)); + })); +} + +impl PanicHandler for ServicePanicHandler { + type Response = ServerError; + + fn get_response(&self, err: Box) -> ServerError { + let response = ServerError::new(Some( + "Internal Server Error. Please report the issue to the service owner.".to_string(), + )); + + // Get the unique identifier for this panic, so we can find it in the logs. + let panic_identifier = response.id().to_string(); + + // Get the message from the panic as best we can. + let err_msg = panic_message(&err); + + // This is the location of the panic. + let location = match LOCATION.with(|l| l.borrow_mut().take()) { + Some(location) => location, + None => "Unknown".to_string(), + }; + + // This is the backtrace of the panic. + let backtrace = match BACKTRACE.with(|b| b.borrow_mut().take()) { + Some(backtrace) => backtrace, + None => "Unknown".to_string(), + }; + + error!( + panic = panic_identifier, + error = err_msg, + loc = location, + bt = backtrace; + "PANIC" + ); + + response + } +} /// Run the Poem Service /// @@ -21,6 +102,12 @@ pub async fn run_service(addr: &SocketAddr) -> Result<(), Error> { tracing::info!("Starting Poem Service ..."); tracing::info!("Listening on {addr}"); + // Set a custom panic hook, so we can catch panics and not crash the service. + // And also get data from the panic so we can log it. + // Panics will cause a 500 to be sent with minimal information we can use to + // help find them in the logs if they happen in production. + set_panic_hook(); + let api_service = mk_api(addr); let docs = docs(&api_service); @@ -35,6 +122,7 @@ pub async fn run_service(addr: &SocketAddr) -> Result<(), Error> { ) .with(Cors::new()) .with(OpenTelemetryMetrics::new()) + .with(CatchPanic::new().with_handler(ServicePanicHandler)) .around(|ep, req| async move { Ok(log_requests(ep, req).await) }); poem::Server::new(TcpListener::bind(addr)) diff --git a/src/cat-data-service/src/service/utilities/metrics_tracing.rs b/src/cat-data-service/src/service/utilities/metrics_tracing.rs index 7564deed7f..6d03af1595 100644 --- a/src/cat-data-service/src/service/utilities/metrics_tracing.rs +++ b/src/cat-data-service/src/service/utilities/metrics_tracing.rs @@ -6,19 +6,16 @@ use opentelemetry::sdk::{ processors, selectors, }, }; -use poem::Route; -use poem::{ - middleware::{CorsEndpoint, OpenTelemetryMetricsEndpoint}, - Endpoint, Request, Response, -}; +use poem::{Endpoint, Request, Response}; use poem_openapi::OperationId; use std::time::Instant; -use std::{env, sync::Arc}; use tracing::{info, span}; use cryptoxide::blake2b::Blake2b; use cryptoxide::digest::Digest; +use crate::settings::CLIENT_ID_KEY; + /// Get an anonymized client ID from the request. /// /// This simply takes the clients IP address, @@ -27,16 +24,10 @@ use cryptoxide::digest::Digest; /// The Hash is unique per client IP, but not able to /// be reversed or analysed without both the client IP and the key. fn anonymous_client_id(req: &Request) -> String { - // Get the Anonymous Client ID Key. - // In production this is a secret key. - // In development this is the default value specified here. - let client_anonymous_key = env::var("CLIENT_ID_KEY") - .unwrap_or_else(|_| String::from("3db5301e-40f2-47ed-ab11-55b37674631a")); - let mut b2b = Blake2b::new(16); // We are going to represent it as a UUID. let mut out = [0; 16]; - b2b.input_str(&client_anonymous_key); + b2b.input_str(CLIENT_ID_KEY.as_str()); b2b.input_str(&req.remote_addr().to_string()); b2b.result(&mut out); @@ -56,10 +47,7 @@ fn anonymous_client_id(req: &Request) -> String { /// * `ep` - Endpoint of the request being made. /// * `req` - Request being made /// -pub async fn log_requests( - ep: Arc>>, - req: Request, -) -> Response { +pub async fn log_requests(ep: E, req: Request) -> Response { let uri = req.uri().clone(); let client_id = anonymous_client_id(&req); // Get the clients anonymous unique id. diff --git a/src/cat-data-service/src/settings.rs b/src/cat-data-service/src/settings.rs index ce268bce7f..eb8f9bcea1 100644 --- a/src/cat-data-service/src/settings.rs +++ b/src/cat-data-service/src/settings.rs @@ -1,9 +1,35 @@ +//! Command line and environment variable settings for the service +//! use crate::logger::{LogFormat, LogLevel, LOG_FORMAT_DEFAULT, LOG_LEVEL_DEFAULT}; use clap::Args; +use dotenvy::dotenv; +use lazy_static::lazy_static; +use std::env; use std::net::SocketAddr; +use tracing::log::error; +use url::Url; +// Default setting for JORM MOCK Timeout +#[cfg(feature = "jorm-mock")] +use crate::state::jorm_mock::JormState; +#[cfg(feature = "jorm-mock")] +use std::time::Duration; + +/// Default address to start service on. const ADDRESS_DEFAULT: &str = "0.0.0.0:3030"; +/// Default Github repo owner +const GITHUB_REPO_OWNER_DEFAULT: &str = "input-output-hk"; + +/// Default Github repo name +const GITHUB_REPO_NAME_DEFAULT: &str = "catalyst-core"; + +/// Default Github issue template to use +const GITHUB_ISSUE_TEMPLATE_DEFAULT: &str = "bug_report.md"; + +/// Default CLIENT_ID_KEY used in development. +const CLIENT_ID_KEY_DEFAULT: &str = "3db5301e-40f2-47ed-ab11-55b37674631a"; + #[derive(Args, Clone)] pub struct Settings { /// Server binding address @@ -26,3 +52,156 @@ pub struct Settings { #[clap(long, default_value = LOG_LEVEL_DEFAULT)] pub log_level: LogLevel, } + +/// An environment variable read as a string. +pub(crate) struct StringEnvVar(String); + +/// An environment variable read as a string. +impl StringEnvVar { + /// Read the env var from the environment. + /// + /// If not defined, read from a .env file. + /// If still not defined, use the default. + /// + /// # Arguments + /// + /// * `var_name`: &str - the name of the env var + /// * `default_value`: &str - the default value + /// + /// # Returns + /// + /// * Self - the value + /// + /// # Example + /// + /// ```rust,no_run + /// #use cat_data_service::settings::StringEnvVar; + /// + /// let var = StringEnvVar::new("MY_VAR", "default"); + /// assert_eq!(var.as_str(), "default"); + /// ``` + fn new(var_name: &str, default_value: &str) -> Self { + dotenv().ok(); + let value = env::var(var_name).unwrap_or_else(|_| default_value.to_owned()); + Self(value) + } + + /// Get the read env var as a str. + /// + /// # Returns + /// + /// * &str - the value + pub(crate) fn as_str(&self) -> &str { + &self.0 + } +} + +// Lazy intialization of all env vars which are not command line parameters. +// All env vars used by the application should be listed here and all should have a default. +// The default for all NON Secret values should be suitable for Production, and NOT development. +// Secrets however should only be used with the default value in development. +lazy_static! { + /// The github repo owner + pub(crate) static ref GITHUB_REPO_OWNER: StringEnvVar = StringEnvVar::new("GITHUB_REPO_OWNER", GITHUB_REPO_OWNER_DEFAULT); + + /// The github repo name + pub(crate) static ref GITHUB_REPO_NAME: StringEnvVar = StringEnvVar::new("GITHUB_REPO_NAME", GITHUB_REPO_NAME_DEFAULT); + + /// The github issue template to use + pub(crate) static ref GITHUB_ISSUE_TEMPLATE: StringEnvVar = StringEnvVar::new("GITHUB_ISSUE_TEMPLATE", GITHUB_ISSUE_TEMPLATE_DEFAULT); + + /// The client id key used to anonymize client connections. + pub(crate) static ref CLIENT_ID_KEY: StringEnvVar = StringEnvVar::new("CLIENT_ID_KEY", CLIENT_ID_KEY_DEFAULT); +} + +// Jorm cleanup timeout is only used if feature is enabled. +#[cfg(feature = "jorm-mock")] +lazy_static! { + /// The jorm mock timeout, only used if feature is enabled. + pub(crate) static ref JORM_CLEANUP_TIMEOUT: Duration = { + dotenv().ok(); + let value = match env::var("JORM_CLEANUP_TIMEOUT") { + Ok(v) => match v.parse::() { + Ok(v) => Some(v), + Err(e) => { + // The default is fine if we can not parse. Just report the error, and continue. + tracing::error!("Failed to parse JORM_CLEANUP_TIMEOUT: {}. Using Default.", e); + None + } + } + Err(_) => None // Not an error if not set, just default it. + }; + + match value { + Some(value) => Duration::from_secs(value * 60), + None => JormState::CLEANUP_TIMEOUT + } + }; +} + +/// Generate a github issue url with a given title +/// +/// ## Arguments +/// +/// * `title`: &str - the title to give the issue +/// +/// ## Returns +/// +/// * String - the url +/// +/// ## Example +/// +/// ```rust,no_run +/// # use cat_data_service::settings::generate_github_issue_url; +/// assert_eq!( +/// generate_github_issue_url("Hello, World! How are you?"), +/// "https://github.com/input-output-hk/catalyst-core/issues/new?template=bug_report.md&title=Hello%2C%20World%21%20How%20are%20you%3F" +/// ); +/// ``` +pub(crate) fn generate_github_issue_url(title: &str) -> Option { + let path = format!( + "https://github.com/{}/{}/issues/new", + GITHUB_REPO_OWNER.as_str(), + GITHUB_REPO_NAME.as_str() + ); + + match Url::parse_with_params( + &path, + &[ + ("template", GITHUB_ISSUE_TEMPLATE.as_str()), + ("title", title), + ], + ) { + Ok(url) => Some(url), + Err(e) => { + error!(err = e.to_string(); "Failed to generate github issue url"); + None + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::time::Duration; + + #[test] + fn github_repo_name_default() { + assert_eq!(GITHUB_REPO_NAME.as_str(), GITHUB_REPO_NAME_DEFAULT); + } + + #[test] + fn github_repo_name_set() { + env::set_var("GITHUB_REPO_NAME", "test"); + assert_eq!(GITHUB_REPO_NAME.as_str(), GITHUB_REPO_NAME_DEFAULT); + } + + #[test] + fn generate_github_issue_url() { + let title = "Hello, World! How are you?"; + assert_eq!( + super::generate_github_issue_url(title).unwrap().as_str(), + "https://github.com/input-output-hk/catalyst-core/issues/new?template=bug_report.md&title=Hello%2C%20World%21%20How%20are%20you%3F" + ); + } +} diff --git a/src/cat-data-service/src/state/mod.rs b/src/cat-data-service/src/state/mod.rs index f27bb6696b..febdcb4877 100644 --- a/src/cat-data-service/src/state/mod.rs +++ b/src/cat-data-service/src/state/mod.rs @@ -20,16 +20,7 @@ impl State { }; #[cfg(feature = "jorm-mock")] - let jorm = { - if let Ok(arg) = std::env::var("JORM_CLEANUP_TIMEOUT") { - let duration = arg.parse::().map_err(|e| { - Error::Service(crate::service::Error::CannotRunService(e.to_string())) - })?; - jorm_mock::JormState::new(std::time::Duration::from_secs(duration * 60)) - } else { - jorm_mock::JormState::new(jorm_mock::JormState::CLEANUP_TIMEOUT) - } - }; + let jorm = jorm_mock::JormState::new(*crate::settings::JORM_CLEANUP_TIMEOUT); Ok(Self { event_db, diff --git a/src/event-db/Cargo.toml b/src/event-db/Cargo.toml index 83bf57009b..7bb7722052 100644 --- a/src/event-db/Cargo.toml +++ b/src/event-db/Cargo.toml @@ -10,7 +10,7 @@ thiserror = { version = "1.0" } serde_json = { version = "1.0" } -dotenvy = "0.15" +dotenvy = { workspace = true } bb8 = "0.8.0" # Database Connection Pool Manager bb8-postgres = "0.8.1" # BB8 Postgres Connection Support.