feat: add metric engine structure

Author: jiacai2050

Cargo.lock

@@ -29,6 +29,7 @@ members = [
     "integration_tests",
     "integration_tests/sdk/rust",
     "src/analytic_engine",
+    "src/metric_engine",
     "src/benchmarks",
     "src/catalog",
     "src/catalog_impls",
@@ -90,6 +91,7 @@ arrow = { version = "49.0.0", features = ["prettyprint"] }
 arrow_ipc = { version = "49.0.0" }
 arrow_ext = { path = "src/components/arrow_ext" }
 analytic_engine = { path = "src/analytic_engine" }
+metric_engine = { path = "src/metric_engine" }
 anyhow = { version = "1.0" }
 arena = { path = "src/components/arena" }
 async-stream = "0.3.4"

Cargo.toml

@@ -51,6 +51,7 @@ logger          = { workspace = true }
 meta_client     = { workspace = true }
 moka            = { version = "0.10", features = ["future"] }
 panic_ext       = { workspace = true }
+metric_engine = { workspace = true }
 proxy           = { workspace = true }
 query_engine    = { workspace = true }
 router          = { workspace = true }

src/horaedb/Cargo.toml

@@ -31,6 +31,7 @@ use df_operator::registry::{FunctionRegistry, FunctionRegistryImpl};
 use interpreters::table_manipulator::{catalog_based, meta_based};
 use logger::{info, warn, RuntimeLevel};
 use meta_client::{meta_impl, types::NodeMetaInfo};
+use metric_engine::*;
 use proxy::{
     limiter::Limiter,
     schema_config_provider::{

src/horaedb/src/setup.rs

@@ -0,0 +1,100 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[package]
+name = "metric_engine"
+
+[package.license]
+workspace = true
+
+[package.version]
+workspace = true
+
+[package.authors]
+workspace = true
+
+[package.edition]
+workspace = true
+
+[features]
+test = ["tempfile"]
+wal-table-kv = ["wal/wal-table-kv"]
+wal-message-queue = ["wal/wal-message-queue"]
+wal-rocksdb = ["wal/wal-rocksdb"]
+wal-local-storage = ["wal/wal-local-storage"]
+
+[dependencies]
+# In alphabetical order
+anyhow = { workspace = true }
+arc-swap = "1.4.0"
+arena = { workspace = true }
+arrow = { workspace = true }
+async-stream = { workspace = true }
+async-trait = { workspace = true }
+atomic_enum = { workspace = true }
+base64 = { workspace = true }
+bytes_ext = { workspace = true }
+codec = { workspace = true }
+common_types = { workspace = true }
+datafusion = { workspace = true }
+future_ext = { workspace = true }
+futures = { workspace = true }
+generic_error = { workspace = true }
+hash_ext = { workspace = true }
+hex = { workspace = true }
+horaedbproto = { workspace = true }
+hyperloglog = { workspace = true }
+id_allocator = { workspace = true }
+itertools = { workspace = true }
+lazy_static = { workspace = true }
+logger = { workspace = true }
+lru = { workspace = true }
+macros = { workspace = true }
+message_queue = { workspace = true }
+metric_ext = { workspace = true }
+object_store = { workspace = true }
+parquet = { workspace = true }
+parquet_ext = { workspace = true }
+prometheus = { workspace = true }
+prost = { workspace = true }
+remote_engine_client = { workspace = true }
+router = { workspace = true }
+runtime = { workspace = true }
+sampling_cache = { workspace = true }
+serde = { workspace = true }
+size_ext = { workspace = true }
+skiplist = { path = "../components/skiplist" }
+smallvec = { workspace = true }
+snafu = { workspace = true }
+table_engine = { workspace = true }
+table_kv = { workspace = true }
+tempfile = { workspace = true, optional = true }
+thiserror = { workspace = true }
+time_ext = { workspace = true }
+tokio = { workspace = true }
+trace_metric = { workspace = true }
+wal = { workspace = true }
+xorfilter-rs = { workspace = true }
+
+[dev-dependencies]
+common_types = { workspace = true, features = ["test"] }
+env_logger = { workspace = true }
+pin-project-lite = { workspace = true }
+rand = { workspace = true }
+tempfile = { workspace = true }
+test_util = { workspace = true }
+wal = { workspace = true, features = ["wal-message-queue", "wal-rocksdb", "wal-table-kv"] }

src/metric_engine/Cargo.toml

@@ -0,0 +1,3 @@
+//! Storage Engine for metrics.
+
+pub mod table_storage;

src/metric_engine/README.md

@@ -0,0 +1,19 @@
+use wal::manager::WalManagerRef;
+
+pub struct SSTable {
+    id: u64,
+}
+
+/// Storage to represent different components of the system.
+/// Such as: metrics, series, indexes, etc.
+///
+/// Columns for design:
+/// metrics: {MetricName}-{MetricID}-{FieldName}
+/// series: {TSID}-{SeriesKey}
+/// index: {TagKey}-{TagValue}-{TSID}
+pub struct TableStorage {
+    name: String,
+    id: u64,
+    wal: WalManagerRef,
+    sstables: Vec<SSTable>,
+}