Skip to content

Commit

Permalink
refactor: remove MixOptions and use StandaloneOptions only (#3893)
Browse files Browse the repository at this point in the history 
* refactor: remove MixOptions and use StandaloneOptions only

* refactor: refactor code by code review comments

1. Use '&self' in frontend_options() and datanode_options();

2. Remove unused 'clone()';

Signed-off-by: zyy17 <zyylsxm@gmail.com>

* ci: fix integration error

---------

Signed-off-by: zyy17 <zyylsxm@gmail.com>
  • Loading branch information
@zyy17
zyy17 authored May 10, 2024
1 parent 115c747 commit b1ef327
Show file tree
Hide file tree
Showing 7 changed files with 122 additions and 198 deletions.
37 changes: 5 additions & 32 deletions src/cmd/src/options.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,51 +13,25 @@
// limitations under the License.

use clap::Parser;
use common_config::KvBackendConfig;
use common_telemetry::logging::{LoggingOptions, TracingOptions};
use common_wal::config::MetasrvWalConfig;
use config::{Config, Environment, File, FileFormat};
use datanode::config::{DatanodeOptions, ProcedureConfig};
use frontend::error::{Result as FeResult, TomlFormatSnafu};
use frontend::frontend::{FrontendOptions, TomlSerializable};
use datanode::config::DatanodeOptions;
use frontend::frontend::FrontendOptions;
use meta_srv::metasrv::MetasrvOptions;
use serde::{Deserialize, Serialize};
use snafu::ResultExt;

use crate::error::{LoadLayeredConfigSnafu, Result, SerdeJsonSnafu};
use crate::standalone::StandaloneOptions;

pub const ENV_VAR_SEP: &str = "__";
pub const ENV_LIST_SEP: &str = ",";

/// Options mixed up from datanode, frontend and metasrv.
#[derive(Serialize, Debug, Clone)]
pub struct MixOptions {
pub data_home: String,
pub procedure: ProcedureConfig,
pub metadata_store: KvBackendConfig,
pub frontend: FrontendOptions,
pub datanode: DatanodeOptions,
pub logging: LoggingOptions,
pub wal_meta: MetasrvWalConfig,
}

impl From<MixOptions> for FrontendOptions {
fn from(value: MixOptions) -> Self {
value.frontend
}
}

impl TomlSerializable for MixOptions {
fn to_toml(&self) -> FeResult<String> {
toml::to_string(self).context(TomlFormatSnafu)
}
}

pub enum Options {
Datanode(Box<DatanodeOptions>),
Frontend(Box<FrontendOptions>),
Metasrv(Box<MetasrvOptions>),
Standalone(Box<MixOptions>),
Standalone(Box<StandaloneOptions>),
Cli(Box<LoggingOptions>),
}

Expand Down Expand Up @@ -158,10 +132,9 @@ impl Options {

pub fn node_id(&self) -> Option<String> {
match self {
Options::Metasrv(_) | Options::Cli(_) => None,
Options::Metasrv(_) | Options::Cli(_) | Options::Standalone(_) => None,
Options::Datanode(opt) => opt.node_id.map(|x| x.to_string()),
Options::Frontend(opt) => opt.node_id.clone(),
Options::Standalone(opt) => opt.frontend.node_id.clone(),
}
}
}
Expand Down
108 changes: 46 additions & 62 deletions src/cmd/src/standalone.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ use common_wal::config::StandaloneWalConfig;
use datanode::config::{DatanodeOptions, ProcedureConfig, RegionEngineConfig, StorageConfig};
use datanode::datanode::{Datanode, DatanodeBuilder};
use file_engine::config::EngineConfig as FileEngineConfig;
use frontend::error::{Result as FeResult, TomlFormatSnafu};
use frontend::frontend::FrontendOptions;
use frontend::instance::builder::FrontendBuilder;
use frontend::instance::{FrontendInstance, Instance as FeInstance, StandaloneDatanodeManager};
Expand All @@ -61,7 +62,7 @@ use crate::error::{
Result, ShutdownDatanodeSnafu, ShutdownFrontendSnafu, StartDatanodeSnafu, StartFrontendSnafu,
StartProcedureManagerSnafu, StartWalOptionsAllocatorSnafu, StopProcedureManagerSnafu,
};
use crate::options::{GlobalOptions, MixOptions, Options};
use crate::options::{GlobalOptions, Options};
use crate::App;

#[derive(Parser)]
Expand All @@ -71,7 +72,7 @@ pub struct Command {
}

impl Command {
pub async fn build(self, opts: MixOptions) -> Result<Instance> {
pub async fn build(self, opts: StandaloneOptions) -> Result<Instance> {
self.subcmd.build(opts).await
}

Expand All @@ -86,7 +87,7 @@ enum SubCommand {
}

impl SubCommand {
async fn build(self, opts: MixOptions) -> Result<Instance> {
async fn build(self, opts: StandaloneOptions) -> Result<Instance> {
match self {
SubCommand::Start(cmd) => cmd.build(opts).await,
}
Expand Down Expand Up @@ -158,37 +159,43 @@ impl Default for StandaloneOptions {
}

impl StandaloneOptions {
fn frontend_options(self) -> FrontendOptions {
pub fn frontend_options(&self) -> FrontendOptions {
let cloned_opts = self.clone();
FrontendOptions {
mode: self.mode,
default_timezone: self.default_timezone,
http: self.http,
grpc: self.grpc,
mysql: self.mysql,
postgres: self.postgres,
opentsdb: self.opentsdb,
influxdb: self.influxdb,
prom_store: self.prom_store,
mode: cloned_opts.mode,
default_timezone: cloned_opts.default_timezone,
http: cloned_opts.http,
grpc: cloned_opts.grpc,
mysql: cloned_opts.mysql,
postgres: cloned_opts.postgres,
opentsdb: cloned_opts.opentsdb,
influxdb: cloned_opts.influxdb,
prom_store: cloned_opts.prom_store,
meta_client: None,
logging: self.logging,
user_provider: self.user_provider,
logging: cloned_opts.logging,
user_provider: cloned_opts.user_provider,
// Handle the export metrics task run by standalone to frontend for execution
export_metrics: self.export_metrics,
export_metrics: cloned_opts.export_metrics,
..Default::default()
}
}

fn datanode_options(self) -> DatanodeOptions {
pub fn datanode_options(&self) -> DatanodeOptions {
let cloned_opts = self.clone();
DatanodeOptions {
node_id: Some(0),
enable_telemetry: self.enable_telemetry,
wal: self.wal.into(),
storage: self.storage,
region_engine: self.region_engine,
rpc_addr: self.grpc.addr,
enable_telemetry: cloned_opts.enable_telemetry,
wal: cloned_opts.wal.into(),
storage: cloned_opts.storage,
region_engine: cloned_opts.region_engine,
rpc_addr: cloned_opts.grpc.addr,
..Default::default()
}
}

pub fn to_toml(&self) -> FeResult<String> {
toml::to_string(self).context(TomlFormatSnafu)
}
}

pub struct Instance {
Expand Down Expand Up @@ -277,20 +284,12 @@ pub struct StartCommand {

impl StartCommand {
fn load_options(&self, global_options: &GlobalOptions) -> Result<Options> {
let opts: StandaloneOptions = Options::load_layered_options(
let mut opts: StandaloneOptions = Options::load_layered_options(
self.config_file.as_deref(),
self.env_prefix.as_ref(),
StandaloneOptions::env_list_keys(),
)?;

self.convert_options(global_options, opts)
}

pub fn convert_options(
&self,
global_options: &GlobalOptions,
mut opts: StandaloneOptions,
) -> Result<Options> {
opts.mode = Mode::Standalone;

if let Some(dir) = &global_options.log_dir {
Expand Down Expand Up @@ -323,8 +322,7 @@ impl StartCommand {
msg: format!(
"gRPC listen address conflicts with datanode reserved gRPC addr: {datanode_grpc_addr}",
),
}
.fail();
}.fail();
}
opts.grpc.addr.clone_from(addr)
}
Expand All @@ -347,47 +345,32 @@ impl StartCommand {

opts.user_provider.clone_from(&self.user_provider);

let metadata_store = opts.metadata_store.clone();
let procedure = opts.procedure.clone();
let frontend = opts.clone().frontend_options();
let logging = opts.logging.clone();
let wal_meta = opts.wal.clone().into();
let datanode = opts.datanode_options().clone();

Ok(Options::Standalone(Box::new(MixOptions {
procedure,
metadata_store,
data_home: datanode.storage.data_home.to_string(),
frontend,
datanode,
logging,
wal_meta,
})))
Ok(Options::Standalone(Box::new(opts)))
}

#[allow(unreachable_code)]
#[allow(unused_variables)]
#[allow(clippy::diverging_sub_expression)]
async fn build(self, opts: MixOptions) -> Result<Instance> {
async fn build(self, opts: StandaloneOptions) -> Result<Instance> {
info!("Standalone start command: {:#?}", self);
info!("Building standalone instance with {opts:#?}");

let mut fe_opts = opts.frontend;
let mut fe_opts = opts.frontend_options();
#[allow(clippy::unnecessary_mut_passed)]
let fe_plugins = plugins::setup_frontend_plugins(&mut fe_opts) // mut ref is MUST, DO NOT change it
.await
.context(StartFrontendSnafu)?;

let dn_opts = opts.datanode;
let dn_opts = opts.datanode_options();

set_default_timezone(fe_opts.default_timezone.as_deref()).context(InitTimezoneSnafu)?;

let data_home = &dn_opts.storage.data_home;
// Ensure the data_home directory exists.
fs::create_dir_all(path::Path::new(&opts.data_home)).context(CreateDirSnafu {
dir: &opts.data_home,
})?;
fs::create_dir_all(path::Path::new(data_home))
.context(CreateDirSnafu { dir: data_home })?;

let metadata_dir = metadata_store_dir(&opts.data_home);
let metadata_dir = metadata_store_dir(data_home);
let (kv_backend, procedure_manager) = FeInstance::try_build_standalone_components(
metadata_dir,
opts.metadata_store.clone(),
Expand Down Expand Up @@ -424,7 +407,7 @@ impl StartCommand {
.build(),
);
let wal_options_allocator = Arc::new(WalOptionsAllocator::new(
opts.wal_meta.clone(),
opts.wal.into(),
kv_backend.clone(),
));
let table_metadata_manager =
Expand Down Expand Up @@ -621,8 +604,8 @@ mod tests {
else {
unreachable!()
};
let fe_opts = options.frontend;
let dn_opts = options.datanode;
let fe_opts = options.frontend_options();
let dn_opts = options.datanode_options();
let logging_opts = options.logging;
assert_eq!(Mode::Standalone, fe_opts.mode);
assert_eq!("127.0.0.1:4000".to_string(), fe_opts.http.addr);
Expand Down Expand Up @@ -759,11 +742,12 @@ mod tests {
assert_eq!(opts.logging.level.as_ref().unwrap(), "debug");

// Should be read from cli, cli > config file > env > default values.
assert_eq!(opts.frontend.http.addr, "127.0.0.1:14000");
assert_eq!(ReadableSize::mb(64), opts.frontend.http.body_limit);
let fe_opts = opts.frontend_options();
assert_eq!(fe_opts.http.addr, "127.0.0.1:14000");
assert_eq!(ReadableSize::mb(64), fe_opts.http.body_limit);

// Should be default value.
assert_eq!(opts.frontend.grpc.addr, GrpcOptions::default().addr);
assert_eq!(fe_opts.grpc.addr, GrpcOptions::default().addr);
},
);
}
Expand Down
19 changes: 19 additions & 0 deletions src/common/wal/src/config.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,25 @@ impl From<StandaloneWalConfig> for MetasrvWalConfig {
}
}

impl From<MetasrvWalConfig> for StandaloneWalConfig {
fn from(config: MetasrvWalConfig) -> Self {
match config {
MetasrvWalConfig::RaftEngine => Self::RaftEngine(RaftEngineConfig::default()),
MetasrvWalConfig::Kafka(config) => Self::Kafka(StandaloneKafkaConfig {
broker_endpoints: config.broker_endpoints,
num_topics: config.num_topics,
selector_type: config.selector_type,
topic_name_prefix: config.topic_name_prefix,
num_partitions: config.num_partitions,
replication_factor: config.replication_factor,
create_topic_timeout: config.create_topic_timeout,
backoff: config.backoff,
..Default::default()
}),
}
}
}

impl From<StandaloneWalConfig> for DatanodeWalConfig {
fn from(config: StandaloneWalConfig) -> Self {
match config {
Expand Down
Loading

0 comments on commit b1ef327

Please sign in to comment.