feat: http debug api for config (#733)

* feat: http api for server config

* fix clippy warning

* Serialize config

* display config in toml format

Author: ShiKaiWi

Cargo.lock

@@ -114,6 +114,7 @@ system_catalog = { path = "system_catalog" }
 table_engine = { path = "table_engine" }
 table_kv = { path = "components/table_kv" }
 tempfile = "3.1.0"
+toml = "0.7"
 tracing_util = { path = "components/tracing_util" }
 trace_metric = { path = "components/trace_metric" }
 trace_metric_derive = { path = "components/trace_metric_derive" }
@@ -143,6 +144,7 @@ server = { workspace = true }
 signal-hook = "0.3"
 sort = "0.8.5"
 table_engine = { workspace = true }
+toml = { workspace = true }
 tracing_util = { workspace = true }
 
 [build-dependencies]

Cargo.toml

@@ -5,7 +5,7 @@
 use std::{collections::HashMap, str::FromStr, sync::Arc};
 
 use common_util::config::{ReadableSize, TimeUnit};
-use serde::Deserialize;
+use serde::{Deserialize, Serialize};
 use snafu::{ensure, Backtrace, GenerateBacktrace, ResultExt, Snafu};
 use tokio::sync::oneshot;
 
@@ -57,15 +57,15 @@ pub enum Error {
     InvalidOption { error: String, backtrace: Backtrace },
 }
 
-#[derive(Debug, Clone, Copy, Deserialize, PartialEq, Default)]
+#[derive(Debug, Clone, Copy, Deserialize, Default, PartialEq, Serialize)]
 pub enum CompactionStrategy {
     #[default]
     Default,
     TimeWindow(TimeWindowCompactionOptions),
     SizeTiered(SizeTieredCompactionOptions),
 }
 
-#[derive(Debug, Clone, Copy, Deserialize, PartialEq)]
+#[derive(Debug, Clone, Copy, Deserialize, PartialEq, Serialize)]
 pub struct SizeTieredCompactionOptions {
     pub bucket_low: f32,
     pub bucket_high: f32,
@@ -75,7 +75,7 @@ pub struct SizeTieredCompactionOptions {
     pub max_input_sstable_size: ReadableSize,
 }
 
-#[derive(Debug, Clone, Copy, Deserialize, PartialEq)]
+#[derive(Debug, Clone, Copy, Deserialize, PartialEq, Serialize)]
 pub struct TimeWindowCompactionOptions {
     pub size_tiered: SizeTieredCompactionOptions,
     // TODO(boyan) In fact right now we only supports TimeUnit::Milliseconds resolution.

analytic_engine/src/compaction/mod.rs

@@ -21,7 +21,7 @@ use common_util::{
     time::DurationExt,
 };
 use log::{debug, error, info, warn};
-use serde::Deserialize;
+use serde::{Deserialize, Serialize};
 use snafu::{ResultExt, Snafu};
 use table_engine::table::TableId;
 use tokio::{
@@ -52,7 +52,7 @@ pub enum Error {
 
 define_result!(Error);
 
-#[derive(Debug, Clone, Deserialize)]
+#[derive(Debug, Clone, Deserialize, Serialize)]
 #[serde(default)]
 pub struct SchedulerConfig {
     pub schedule_channel_len: usize,

analytic_engine/src/compaction/scheduler.rs

@@ -36,7 +36,7 @@ use wal::{
 pub use crate::{compaction::scheduler::SchedulerConfig, table_options::TableOptions};
 
 /// Config of analytic engine
-#[derive(Debug, Clone, Deserialize)]
+#[derive(Debug, Clone, Deserialize, Serialize)]
 #[serde(default)]
 pub struct Config {
     /// Storage options of the engine
@@ -115,7 +115,7 @@ impl Default for Config {
 }
 
 /// Config of wal based on obkv
-#[derive(Debug, Default, Clone, Deserialize)]
+#[derive(Debug, Default, Clone, Deserialize, Serialize)]
 #[serde(default)]
 pub struct ObkvWalConfig {
     /// Obkv client config
@@ -222,7 +222,7 @@ impl From<WalNamespaceConfig> for NamespaceConfig {
 }
 
 /// Config of wal based on obkv
-#[derive(Debug, Default, Clone, Deserialize)]
+#[derive(Debug, Default, Clone, Deserialize, Serialize)]
 #[serde(default)]
 pub struct KafkaWalConfig {
     /// Kafka client config
@@ -235,7 +235,7 @@ pub struct KafkaWalConfig {
 }
 
 /// Config for wal based on RocksDB
-#[derive(Debug, Clone, Deserialize)]
+#[derive(Debug, Clone, Deserialize, Serialize)]
 #[serde(default)]
 pub struct RocksDBConfig {
     /// Data directory used by RocksDB.
@@ -250,7 +250,7 @@ impl Default for RocksDBConfig {
     }
 }
 /// Options for wal storage backend
-#[derive(Debug, Clone, Deserialize)]
+#[derive(Debug, Clone, Deserialize, Serialize)]
 #[serde(tag = "type")]
 pub enum WalStorageConfig {
     RocksDB(Box<RocksDBConfig>),

analytic_engine/src/lib.rs

@@ -22,7 +22,7 @@ use log::{debug, info, warn};
 use object_store::{ObjectStoreRef, Path};
 use parquet::data_type::AsBytes;
 use prost::Message;
-use serde::Deserialize;
+use serde::{Deserialize, Serialize};
 use snafu::{Backtrace, ResultExt, Snafu};
 use table_engine::table::TableId;
 use tokio::sync::Mutex;
@@ -157,7 +157,7 @@ impl MetaUpdateLogEntryIterator for MetaUpdateReaderImpl {
 }
 
 /// Options for manifest
-#[derive(Debug, Clone, Deserialize)]
+#[derive(Debug, Clone, Deserialize, Serialize)]
 pub struct Options {
     /// Steps to do snapshot
     pub snapshot_every_n_updates: usize,

analytic_engine/src/manifest/details.rs

@@ -3,9 +3,9 @@
 use std::time::Duration;
 
 use common_util::config::{ReadableDuration, ReadableSize};
-use serde::Deserialize;
+use serde::{Deserialize, Serialize};
 
-#[derive(Debug, Clone, Deserialize)]
+#[derive(Debug, Clone, Deserialize, Serialize)]
 #[serde(default)]
 /// Options for storage backend
 pub struct StorageOptions {
@@ -37,19 +37,19 @@ impl Default for StorageOptions {
     }
 }
 
-#[derive(Debug, Clone, Deserialize)]
+#[derive(Debug, Clone, Deserialize, Serialize)]
 #[serde(tag = "type")]
 pub enum ObjectStoreOptions {
     Local(LocalOptions),
     Aliyun(AliyunOptions),
 }
 
-#[derive(Debug, Clone, Deserialize)]
+#[derive(Debug, Clone, Deserialize, Serialize)]
 pub struct LocalOptions {
     pub data_dir: String,
 }
 
-#[derive(Debug, Clone, Deserialize)]
+#[derive(Debug, Clone, Deserialize, Serialize)]
 pub struct AliyunOptions {
     pub key_id: String,
     pub key_secret: String,

analytic_engine/src/storage_options.rs

@@ -12,7 +12,7 @@ use common_util::{
     time::DurationExt,
 };
 use datafusion::parquet::basic::Compression as ParquetCompression;
-use serde::Deserialize;
+use serde::{Deserialize, Serialize};
 use snafu::{Backtrace, GenerateBacktrace, OptionExt, ResultExt, Snafu};
 use table_engine::OPTION_KEY_ENABLE_TTL;
 
@@ -125,7 +125,7 @@ pub enum Error {
 
 define_result!(Error);
 
-#[derive(Debug, Clone, Deserialize, PartialEq, Eq)]
+#[derive(Debug, Clone, Deserialize, Eq, PartialEq, Serialize)]
 pub enum UpdateMode {
     Overwrite,
     Append,
@@ -152,7 +152,7 @@ impl ToString for UpdateMode {
     }
 }
 
-#[derive(Debug, Clone, Copy, Deserialize, PartialEq, Eq)]
+#[derive(Debug, Clone, Copy, Deserialize, Eq, PartialEq, Serialize)]
 pub enum Compression {
     Uncompressed,
     Lz4,
@@ -221,7 +221,7 @@ impl From<Compression> for ParquetCompression {
 }
 
 /// A hint for building sst.
-#[derive(Clone, Copy, Debug, Default, Deserialize, PartialEq, Eq)]
+#[derive(Clone, Copy, Debug, Default, Deserialize, PartialEq, Eq, Serialize)]
 pub enum StorageFormatHint {
     /// Which storage format is chosen to encode one sst depends on the data
     /// pattern.
@@ -231,7 +231,7 @@ pub enum StorageFormatHint {
 }
 
 /// StorageFormat specify how records are saved in persistent storage
-#[derive(Debug, Clone, Copy, Deserialize, PartialEq, Eq)]
+#[derive(Debug, Clone, Copy, Deserialize, PartialEq, Eq, Serialize)]
 pub enum StorageFormat {
     /// Traditional columnar format, every column is saved in one exact one
     /// column, for example:
@@ -375,7 +375,7 @@ impl Default for StorageFormat {
 }
 
 /// Options for a table.
-#[derive(Debug, Clone, Deserialize, PartialEq)]
+#[derive(Debug, Clone, Deserialize, PartialEq, Serialize)]
 #[serde(default)]
 pub struct TableOptions {
     // The following options are immutable once table was created.

analytic_engine/src/table_options.rs

@@ -2,7 +2,7 @@
 
 use common_types::schema::TIMESTAMP_COLUMN;
 use meta_client::meta_impl::MetaClientConfig;
-use serde::Deserialize;
+use serde::{Deserialize, Serialize};
 use table_engine::ANALYTIC_ENGINE_TYPE;
 
 #[derive(Debug, Clone, Deserialize)]
@@ -22,7 +22,7 @@ impl Default for SchemaConfig {
     }
 }
 
-#[derive(Default, Clone, Deserialize, Debug)]
+#[derive(Default, Clone, Deserialize, Debug, Serialize)]
 #[serde(default)]
 pub struct ClusterConfig {
     pub cmd_channel_buffer_size: usize,

cluster/src/config.rs

@@ -33,7 +33,7 @@ serde_json = { workspace = true }
 snafu = { workspace = true }
 time = "0.1"
 tokio = { workspace = true }
-toml = "0.5"
+toml = { workspace = true }
 
 [dev-dependencies.slog-global]
 version = "0.1"