issuer proxy, a lot of other changes

Cargo.toml

@@ -63,6 +63,7 @@ serde = "1.0"
 serde_json = "1.0"
 strum = "0.26"
 strum_macros = "0.26"
+sync_wrapper = "1.0"
 thiserror = "1.0"
 tempfile = "3.10"
 tokio = { version = "1.36", features = ["full"] }

src/cli.rs

@@ -14,7 +14,7 @@ use crate::{
     core::{AUTHOR_NAME, SERVICE_NAME},
     http::dns,
     routing::canister::CanisterAlias,
-    tls::acme,
+    tls::{self, acme},
 };
 
 #[derive(Parser)]
@@ -146,12 +146,18 @@ pub struct Cert {
 
 #[derive(Args)]
 pub struct Domain {
-    /// List of domains that we serve system subnets from
-    #[clap(long = "domain-system")]
+    /// Specify domains that will be served. This affects the routing, canister extraction, ACME certificate issuing etc.
+    #[clap(long = "domain")]
+    pub domains: Vec<FQDN>,
+
+    /// List of domains that we serve system subnets from. This enables domain-canister matching for these domains & adds them to the list of served domains above, do not list them there separately.
+    /// Requires --domain-app.
+    #[clap(long = "domain-system", requires = "domains_app")]
     pub domains_system: Vec<FQDN>,
 
-    /// List of domains that we serve app subnets from
-    #[clap(long = "domain-app")]
+    /// List of domains that we serve app subnets from. See --domain-system above for details.
+    /// Requires --domain-system.
+    #[clap(long = "domain-app", requires = "domains_system")]
     pub domains_app: Vec<FQDN>,
 
     /// List of canister aliases in format '<alias>:<canister_id>'
@@ -184,7 +190,9 @@ pub struct Policy {
 
 #[derive(Args)]
 pub struct Acme {
-    /// Type of ACME challenge to use. Currently supported: alpn
+    /// Type of ACME challenge to use. Currently supported: alpn.
+    /// If specified it will try to obtain the certificate that is valid for all specified domains
+    /// (--domain-app & --domain-system). For this to succeed they all should resolve to the hostname where this service is running.
     #[clap(long = "acme-challenge", requires = "acme_cache_path")]
     pub acme_challenge: Option<acme::Challenge>,
 
@@ -193,7 +201,7 @@ pub struct Acme {
     #[clap(long = "acme-cache-path")]
     pub acme_cache_path: Option<PathBuf>,
 
-    /// Whether to use LetsEncrypt staging API to avoid hitting the limits
+    /// Whether to use LetsEncrypt staging API for testing to avoid hitting the limits
     #[clap(long = "acme-staging")]
     pub acme_staging: bool,
 }
@@ -211,3 +219,42 @@ pub struct Misc {
     #[clap(long = "geoip-db")]
     pub geoip_db: Option<PathBuf>,
 }
+
+// Some conversions
+impl From<&HttpServer> for crate::http::server::Options {
+    fn from(c: &HttpServer) -> Self {
+        Self {
+            backlog: c.backlog,
+            http2_keepalive_interval: c.http2_keepalive_interval,
+            http2_keepalive_timeout: c.http2_keepalive_timeout,
+            http2_max_streams: c.http2_max_streams,
+            grace_period: c.grace_period,
+        }
+    }
+}
+
+impl From<&Dns> for crate::http::dns::Options {
+    fn from(c: &Dns) -> Self {
+        Self {
+            protocol: c.protocol,
+            servers: c.servers.clone(),
+            tls_name: c.tls_name.clone(),
+            cache_size: c.cache_size,
+        }
+    }
+}
+
+impl From<&Cli> for crate::http::client::Options {
+    fn from(c: &Cli) -> Self {
+        Self {
+            dns_options: (&c.dns).into(),
+            timeout_connect: c.http_client.timeout_connect,
+            timeout: c.http_client.timeout,
+            tcp_keepalive: Some(c.http_client.tcp_keepalive),
+            http2_keepalive: Some(c.http_client.http2_keepalive),
+            http2_keepalive_timeout: c.http_client.http2_keepalive_timeout,
+            user_agent: crate::core::SERVICE_NAME.into(),
+            tls_config: tls::prepare_client_config(),
+        }
+    }
+}

src/core.rs

@@ -1,15 +1,15 @@
+use std::{error::Error as StdError, sync::Arc};
+
 use anyhow::{anyhow, Error};
 use async_trait::async_trait;
 use prometheus::Registry;
 use rustls::sign::CertifiedKey;
-use std::sync::Arc;
 use tokio_util::{sync::CancellationToken, task::TaskTracker};
 use tracing::{error, warn};
 
 use crate::{
     cli::Cli,
-    http::{server, ReqwestClient, Server},
-    metrics,
+    http, metrics,
     routing::{
         self,
         canister::{CanisterResolver, ResolvesCanister},
@@ -31,6 +31,26 @@ pub trait Run: Send + Sync {
 }
 pub struct Runner(pub String, pub Arc<dyn Run>);
 
+#[async_trait]
+impl Run for http::Server {
+    async fn run(&self, token: CancellationToken) -> Result<(), Error> {
+        self.serve(token).await
+    }
+}
+
+pub fn error_source<E: StdError + 'static>(error: &impl StdError) -> Option<&E> {
+    let mut source = error.source();
+    while let Some(err) = source {
+        if let Some(v) = err.downcast_ref() {
+            return Some(v);
+        }
+
+        source = err.source();
+    }
+
+    None
+}
+
 pub async fn main(cli: &Cli) -> Result<(), Error> {
     // Install crypto-provider
     rustls::crypto::aws_lc_rs::default_provider()
@@ -41,7 +61,7 @@ pub async fn main(cli: &Cli) -> Result<(), Error> {
     let token = CancellationToken::new();
     let tracker = TaskTracker::new();
     let registry = Registry::new();
-    let http_client = Arc::new(ReqwestClient::new(cli)?);
+    let http_client = Arc::new(http::ReqwestClient::new(cli.into())?);
 
     // List of cancellable tasks to execute & track
     let mut runners: Vec<Runner> = vec![];
@@ -52,12 +72,13 @@ pub async fn main(cli: &Cli) -> Result<(), Error> {
     ctrlc::set_handler(move || handler_token.cancel())?;
 
     // Make a list of all supported domains
-    let mut domains = cli.domain.domains_system.clone();
+    let mut domains = cli.domain.domains.clone();
+    domains.extend_from_slice(&cli.domain.domains_system);
     domains.extend_from_slice(&cli.domain.domains_app);
 
     if domains.is_empty() {
         return Err(anyhow!(
-            "No domains specified (use --domain-system and/or --domain-app)"
+            "No domains to serve specified (use --domain/--domain-system/--domain-app)"
         ));
     }
 
@@ -82,13 +103,11 @@ pub async fn main(cli: &Cli) -> Result<(), Error> {
         runners.push(Runner("denylist_updater".into(), v));
     }
 
-    let server_options = server::Options::from(&cli.http_server);
-
     // Set up HTTP
-    let http_server = Arc::new(Server::new(
+    let http_server = Arc::new(http::Server::new(
         cli.http_server.http,
         router.clone(),
-        server_options,
+        (&cli.http_server).into(),
         None,
     )) as Arc<dyn Run>;
     runners.push(Runner("http_server".into(), http_server));
@@ -103,10 +122,10 @@ pub async fn main(cli: &Cli) -> Result<(), Error> {
     )?;
     runners.extend(tls_runners);
 
-    let https_server = Arc::new(Server::new(
+    let https_server = Arc::new(http::Server::new(
         cli.http_server.https,
         router,
-        server_options,
+        (&cli.http_server).into(),
         Some(rustls_cfg),
     )) as Arc<dyn Run>;
     runners.push(Runner("https_server".into(), https_server));
@@ -116,7 +135,12 @@ pub async fn main(cli: &Cli) -> Result<(), Error> {
         let (router, runner) = metrics::setup(&registry);
         runners.push(Runner("metrics_runner".into(), runner));
 
-        let srv = Arc::new(Server::new(addr, router, server_options, None));
+        let srv = Arc::new(http::Server::new(
+            addr,
+            router,
+            (&cli.http_server).into(),
+            None,
+        ));
         runners.push(Runner("metrics_server".into(), srv as Arc<dyn Run>));
     }
 

src/http/client.rs

@@ -1,35 +1,44 @@
-use std::sync::Arc;
+use std::{sync::Arc, time::Duration};
 
 use async_trait::async_trait;
 use mockall::automock;
 
-use crate::{cli, core::SERVICE_NAME, http::dns::Resolver, tls::prepare_client_config};
+use super::dns;
 
 #[automock]
 #[async_trait]
 pub trait Client: Send + Sync {
     async fn execute(&self, req: reqwest::Request) -> Result<reqwest::Response, reqwest::Error>;
 }
 
+pub struct Options {
+    pub dns_options: dns::Options,
+    pub timeout_connect: Duration,
+    pub timeout: Duration,
+    pub tcp_keepalive: Option<Duration>,
+    pub http2_keepalive: Option<Duration>,
+    pub http2_keepalive_timeout: Duration,
+    pub user_agent: String,
+    pub tls_config: rustls::ClientConfig,
+}
+
 #[derive(Clone)]
 pub struct ReqwestClient(reqwest::Client);
 
 impl ReqwestClient {
-    pub fn new(cli: &cli::Cli) -> Result<Self, anyhow::Error> {
-        let http = &cli.http_client;
-
+    pub fn new(opts: Options) -> Result<Self, anyhow::Error> {
         let client = reqwest::Client::builder()
-            .use_preconfigured_tls(prepare_client_config())
-            .dns_resolver(Arc::new(Resolver::new(&cli.dns)))
-            .connect_timeout(http.timeout_connect)
-            .timeout(http.timeout)
+            .use_preconfigured_tls(opts.tls_config)
+            .dns_resolver(Arc::new(dns::Resolver::new(opts.dns_options)))
+            .connect_timeout(opts.timeout_connect)
+            .timeout(opts.timeout)
             .tcp_nodelay(true)
-            .tcp_keepalive(Some(http.tcp_keepalive))
-            .http2_keep_alive_interval(Some(http.http2_keepalive))
-            .http2_keep_alive_timeout(http.http2_keepalive_timeout)
+            .tcp_keepalive(opts.tcp_keepalive)
+            .http2_keep_alive_interval(opts.http2_keepalive)
+            .http2_keep_alive_timeout(opts.http2_keepalive_timeout)
             .http2_keep_alive_while_idle(true)
             .http2_adaptive_window(true)
-            .user_agent(SERVICE_NAME)
+            .user_agent(opts.user_agent)
             .redirect(reqwest::redirect::Policy::none())
             .no_proxy()
             .build()?;

src/http/dns.rs

@@ -1,4 +1,7 @@
-use std::{net::SocketAddr, sync::Arc};
+use std::{
+    net::{IpAddr, SocketAddr},
+    sync::Arc,
+};
 
 use hickory_resolver::{
     config::{NameServerConfigGroup, ResolverConfig, ResolverOpts},
@@ -8,9 +11,7 @@ use hickory_resolver::{
 use reqwest::dns::{Addrs, Name, Resolve, Resolving};
 use strum_macros::EnumString;
 
-use crate::cli::Dns;
-
-#[derive(Clone, Debug, EnumString)]
+#[derive(Clone, Copy, Debug, EnumString)]
 #[strum(serialize_all = "snake_case")]
 pub enum Protocol {
     Clear,
@@ -21,24 +22,29 @@ pub enum Protocol {
 #[derive(Debug, Clone)]
 pub struct Resolver(Arc<TokioAsyncResolver>);
 
+pub struct Options {
+    pub protocol: Protocol,
+    pub servers: Vec<IpAddr>,
+    pub tls_name: String,
+    pub cache_size: usize,
+}
+
 // new() must be called in Tokio context
 impl Resolver {
-    pub fn new(cli: &Dns) -> Self {
-        let name_servers = match cli.protocol {
-            Protocol::Clear => NameServerConfigGroup::from_ips_clear(&cli.servers, 53, true),
-            Protocol::Tls => {
-                NameServerConfigGroup::from_ips_tls(&cli.servers, 853, cli.tls_name.clone(), true)
-            }
+    pub fn new(o: Options) -> Self {
+        let name_servers = match o.protocol {
+            Protocol::Clear => NameServerConfigGroup::from_ips_clear(&o.servers, 53, true),
+            Protocol::Tls => NameServerConfigGroup::from_ips_tls(&o.servers, 853, o.tls_name, true),
             Protocol::Https => {
-                NameServerConfigGroup::from_ips_https(&cli.servers, 443, cli.tls_name.clone(), true)
+                NameServerConfigGroup::from_ips_https(&o.servers, 443, o.tls_name, true)
             }
         };
 
         let cfg = ResolverConfig::from_parts(None, vec![], name_servers);
 
         let mut opts = ResolverOpts::default();
         opts.rotate = true;
-        opts.cache_size = cli.cache_size;
+        opts.cache_size = o.cache_size;
         opts.use_hosts_file = false;
         opts.preserve_intermediates = false;
         opts.try_tcp_on_error = true;

src/http/mod.rs

@@ -7,6 +7,14 @@ use http::{HeaderMap, Version};
 pub use client::{Client, ReqwestClient};
 pub use server::{ConnInfo, Server};
 
+pub const ALPN_H1: &[u8] = b"http/1.1";
+pub const ALPN_H2: &[u8] = b"h2";
+pub const ALPN_HTTP: &[&[u8]] = &[ALPN_H1, ALPN_H2];
+
+pub fn is_http_alpn(alpn: &[u8]) -> bool {
+    ALPN_HTTP.contains(&alpn)
+}
+
 // Calculate very approximate HTTP request/response headers size in bytes.
 // More or less accurate only for http/1.1 since in h2 headers are in HPACK-compressed.
 // But it seems there's no better way.