feat: sql support dictionary column

components/arrow_ext/src/ipc.rs

@@ -184,12 +184,40 @@ mod tests {
     use std::sync::Arc;
 
     use arrow::{
-        array::{Int32Array, StringArray},
-        datatypes::{DataType, Field, Schema},
+        array::{Int32Array, StringArray, StringDictionaryBuilder},
+        datatypes::{DataType, Field, Int32Type, Schema},
     };
 
     use super::*;
 
+    fn create_dictionary_record_batch() -> RecordBatch {
+        let col1 = Field::new_dict(
+            "dic1",
+            DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)),
+            false,
+            1,
+            false,
+        );
+        let col2 = Field::new_dict(
+            "dic2",
+            DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)),
+            true,
+            0,
+            false,
+        );
+        let schema = Schema::new(vec![col1, col2]);
+        let mut builder = StringDictionaryBuilder::<Int32Type>::new();
+        builder.append_value("d1");
+        builder.append_value("d2");
+        let dic1 = builder.finish();
+        let mut builder = StringDictionaryBuilder::<Int32Type>::new();
+        builder.append_null();
+        builder.append_value("d3");
+        let dic2 = builder.finish();
+
+        RecordBatch::try_new(Arc::new(schema), vec![Arc::new(dic1), Arc::new(dic2)]).unwrap()
+    }
+
     fn create_batch(rows: usize) -> RecordBatch {
         let schema = Schema::new(vec![
             Field::new("a", DataType::Int32, false),
@@ -226,6 +254,18 @@ mod tests {
         }
     }
 
+    #[test]
+    fn test_ipc_encode_decode_with_dicitonary_encode() {
+        let batch = create_dictionary_record_batch();
+        for compression in [CompressionMethod::None, CompressionMethod::Zstd] {
+            let compress_opts = CompressOptions {
+                compress_min_length: 0,
+                method: compression,
+            };
+            ensure_encoding_and_decoding(&batch, compress_opts, compression);
+        }
+    }
+
     #[test]
     fn test_encode_multiple_record_batches() {
         let num_batches = 1000;

integration_tests/Cargo.toml

@@ -8,7 +8,7 @@ workspace = true
 [dependencies]
 anyhow = "1.0.58"
 async-trait = "0.1"
-ceresdb-client = "1.0"
+ceresdb-client = "1.0.2"
 local-ip-address = "0.5"
 reqwest = { workspace = true }
 serde = { workspace = true }

integration_tests/cases/common/dml/case_sensitive.result

@@ -75,10 +75,10 @@ Failed to execute query, err: Server(ServerError { code: 500, msg: "Failed to cr
 
 DESC case_SENSITIVE_table1;
 
-name,type,is_primary,is_nullable,is_tag,
-String("tsid"),String("uint64"),Boolean(true),Boolean(false),Boolean(false),
-String("ts"),String("timestamp"),Boolean(true),Boolean(false),Boolean(false),
-String("VALUE1"),String("double"),Boolean(false),Boolean(true),Boolean(false),
+name,type,is_primary,is_nullable,is_tag,is_dictionary,
+String("tsid"),String("uint64"),Boolean(true),Boolean(false),Boolean(false),Boolean(false),
+String("ts"),String("timestamp"),Boolean(true),Boolean(false),Boolean(false),Boolean(false),
+String("VALUE1"),String("double"),Boolean(false),Boolean(true),Boolean(false),Boolean(false),
 
 
 DESC CASE_SENSITIVE_TABLE1;
@@ -87,10 +87,10 @@ Failed to execute query, err: Server(ServerError { code: 500, msg: "Failed to cr
 
 DESC `case_SENSITIVE_table1`;
 
-name,type,is_primary,is_nullable,is_tag,
-String("tsid"),String("uint64"),Boolean(true),Boolean(false),Boolean(false),
-String("ts"),String("timestamp"),Boolean(true),Boolean(false),Boolean(false),
-String("VALUE1"),String("double"),Boolean(false),Boolean(true),Boolean(false),
+name,type,is_primary,is_nullable,is_tag,is_dictionary,
+String("tsid"),String("uint64"),Boolean(true),Boolean(false),Boolean(false),Boolean(false),
+String("ts"),String("timestamp"),Boolean(true),Boolean(false),Boolean(false),Boolean(false),
+String("VALUE1"),String("double"),Boolean(false),Boolean(true),Boolean(false),Boolean(false),
 
 
 DESC `CASE_SENSITIVE_TABLE1`;

integration_tests/cases/common/dml/insert_mode.result

@@ -5,6 +5,7 @@ affected_rows: 0
 CREATE TABLE `03_dml_insert_mode_table1` (
     `timestamp` timestamp NOT NULL,
     `value` double,
+    `dic` string dictionary,
     timestamp KEY (timestamp)) ENGINE=Analytic
 WITH(
 	 enable_ttl='false',
@@ -13,8 +14,8 @@ WITH(
 
 affected_rows: 0
 
-INSERT INTO `03_dml_insert_mode_table1` (`timestamp`, `value`)
-    VALUES (1, +10), (2, 0), (3, -30);
+INSERT INTO `03_dml_insert_mode_table1` (`timestamp`, `value`, `dic`)
+    VALUES (1, +10, "d1"), (2, 0, "d2"), (3, -30, "d1");
 
 affected_rows: 3
 
@@ -25,10 +26,10 @@ FROM
 ORDER BY
     `value` ASC;
 
-tsid,timestamp,value,
-UInt64(0),Timestamp(3),Double(-30.0),
-UInt64(0),Timestamp(2),Double(0.0),
-UInt64(0),Timestamp(1),Double(10.0),
+tsid,timestamp,value,dic,
+UInt64(0),Timestamp(3),Double(-30.0),String("d1"),
+UInt64(0),Timestamp(2),Double(0.0),String("d2"),
+UInt64(0),Timestamp(1),Double(10.0),String("d1"),
 
 
 INSERT INTO `03_dml_insert_mode_table1` (`timestamp`, `value`)
@@ -43,10 +44,10 @@ FROM
 ORDER BY
     `value` ASC;
 
-tsid,timestamp,value,
-UInt64(0),Timestamp(1),Double(100.0),
-UInt64(0),Timestamp(2),Double(200.0),
-UInt64(0),Timestamp(3),Double(300.0),
+tsid,timestamp,value,dic,
+UInt64(0),Timestamp(1),Double(100.0),String(""),
+UInt64(0),Timestamp(2),Double(200.0),String(""),
+UInt64(0),Timestamp(3),Double(300.0),String(""),
 
 
 DROP TABLE `03_dml_insert_mode_table1`;
@@ -60,6 +61,7 @@ affected_rows: 0
 CREATE TABLE `03_dml_insert_mode_table2` (
     `timestamp` timestamp NOT NULL,
     `value` double,
+    `dic` string dictionary,
     timestamp KEY (timestamp)) ENGINE=Analytic
 WITH(
 	 enable_ttl='false',
@@ -68,8 +70,8 @@ WITH(
 
 affected_rows: 0
 
-INSERT INTO `03_dml_insert_mode_table2` (`timestamp`, `value`)
-    VALUES (1, 10), (2, 20), (3, 30);
+INSERT INTO `03_dml_insert_mode_table2` (`timestamp`, `value`, `dic`)
+    VALUES (1, 10, "d1"), (2, 20, ""), (3, 30, "d2");
 
 affected_rows: 3
 
@@ -80,14 +82,14 @@ FROM
 ORDER BY
     `value` ASC;
 
-tsid,timestamp,value,
-UInt64(0),Timestamp(1),Double(10.0),
-UInt64(0),Timestamp(2),Double(20.0),
-UInt64(0),Timestamp(3),Double(30.0),
+tsid,timestamp,value,dic,
+UInt64(0),Timestamp(1),Double(10.0),String("d1"),
+UInt64(0),Timestamp(2),Double(20.0),String(""),
+UInt64(0),Timestamp(3),Double(30.0),String("d2"),
 
 
-INSERT INTO `03_dml_insert_mode_table2` (`timestamp`, `value`)
-    VALUES (1, 100), (2, 200), (3, 300);
+INSERT INTO `03_dml_insert_mode_table2` (`timestamp`, `value`, `dic`)
+    VALUES (1, 100, "d2"), (2, 200, "d1"), (3, 300, "");
 
 affected_rows: 3
 
@@ -98,13 +100,13 @@ FROM
 ORDER BY
     `value` ASC;
 
-tsid,timestamp,value,
-UInt64(0),Timestamp(1),Double(10.0),
-UInt64(0),Timestamp(2),Double(20.0),
-UInt64(0),Timestamp(3),Double(30.0),
-UInt64(0),Timestamp(1),Double(100.0),
-UInt64(0),Timestamp(2),Double(200.0),
-UInt64(0),Timestamp(3),Double(300.0),
+tsid,timestamp,value,dic,
+UInt64(0),Timestamp(1),Double(10.0),String("d1"),
+UInt64(0),Timestamp(2),Double(20.0),String(""),
+UInt64(0),Timestamp(3),Double(30.0),String("d2"),
+UInt64(0),Timestamp(1),Double(100.0),String("d2"),
+UInt64(0),Timestamp(2),Double(200.0),String("d1"),
+UInt64(0),Timestamp(3),Double(300.0),String(""),
 
 
 DROP TABLE `03_dml_insert_mode_table2`;
@@ -118,15 +120,16 @@ affected_rows: 0
 CREATE TABLE `03_dml_insert_mode_table3` (
     `timestamp` timestamp NOT NULL,
     `value` double,
+    `dic` string dictionary,
     timestamp KEY (timestamp)) ENGINE=Analytic
 WITH(
 	 enable_ttl='false'
 );
 
 affected_rows: 0
 
-INSERT INTO `03_dml_insert_mode_table3` (`timestamp`, `value`)
-    VALUES (1, 10), (2, 20), (3, 30);
+INSERT INTO `03_dml_insert_mode_table3` (`timestamp`, `value`, `dic`)
+    VALUES (1, 100, "d2"), (2, 200, "d1"), (3, 300, "d1");
 
 affected_rows: 3
 
@@ -137,14 +140,14 @@ FROM
 ORDER BY
     `value` ASC;
 
-tsid,timestamp,value,
-UInt64(0),Timestamp(1),Double(10.0),
-UInt64(0),Timestamp(2),Double(20.0),
-UInt64(0),Timestamp(3),Double(30.0),
+tsid,timestamp,value,dic,
+UInt64(0),Timestamp(1),Double(100.0),String("d2"),
+UInt64(0),Timestamp(2),Double(200.0),String("d1"),
+UInt64(0),Timestamp(3),Double(300.0),String("d1"),
 
 
 INSERT INTO `03_dml_insert_mode_table3` (`timestamp`, `value`)
-    VALUES (1, 100), (2, 200), (3, 300);
+    VALUES (1, 100, "d5"), (2, 200, "d6"), (3, 300, "d7");
 
 affected_rows: 3
 
@@ -155,10 +158,10 @@ FROM
 ORDER BY
     `value` ASC;
 
-tsid,timestamp,value,
-UInt64(0),Timestamp(1),Double(100.0),
-UInt64(0),Timestamp(2),Double(200.0),
-UInt64(0),Timestamp(3),Double(300.0),
+tsid,timestamp,value,dic,
+UInt64(0),Timestamp(1),Double(100.0),String(""),
+UInt64(0),Timestamp(2),Double(200.0),String(""),
+UInt64(0),Timestamp(3),Double(300.0),String(""),
 
 
 DROP TABLE `03_dml_insert_mode_table3`;
@@ -176,6 +179,7 @@ CREATE TABLE `03_dml_insert_mode_table4` (
     `c3` uint32 default c1 + 1,
     `c4` uint32 default c3 + 1,
     `c5` uint32 default c3 + 10,
+    `c6` string default "default",
     timestamp KEY (timestamp)) ENGINE=Analytic
 WITH(
 	 enable_ttl='false'
@@ -195,10 +199,10 @@ FROM
 ORDER BY
     `c1` ASC;
 
-tsid,timestamp,c1,c2,c3,c4,c5,
-UInt64(0),Timestamp(1),UInt32(10),String("123"),UInt32(11),UInt32(12),UInt32(3),
-UInt64(0),Timestamp(2),UInt32(20),String("123"),UInt32(21),UInt32(22),UInt32(4),
-UInt64(0),Timestamp(3),UInt32(30),String("123"),UInt32(31),UInt32(32),UInt32(5),
+tsid,timestamp,c1,c2,c3,c4,c5,c6,
+UInt64(0),Timestamp(1),UInt32(10),String("123"),UInt32(11),UInt32(12),UInt32(3),String("default"),
+UInt64(0),Timestamp(2),UInt32(20),String("123"),UInt32(21),UInt32(22),UInt32(4),String("default"),
+UInt64(0),Timestamp(3),UInt32(30),String("123"),UInt32(31),UInt32(32),UInt32(5),String("default"),
 
 
 DROP TABLE IF EXISTS `03_dml_insert_mode_table4`;

integration_tests/cases/common/dml/insert_mode.sql

@@ -4,15 +4,16 @@ DROP TABLE IF EXISTS `03_dml_insert_mode_table1`;
 CREATE TABLE `03_dml_insert_mode_table1` (
     `timestamp` timestamp NOT NULL,
     `value` double,
+    `dic` string dictionary,
     timestamp KEY (timestamp)) ENGINE=Analytic
 WITH(
 	 enable_ttl='false',
 	 update_mode='OVERWRITE'
 );
 
 
-INSERT INTO `03_dml_insert_mode_table1` (`timestamp`, `value`)
-    VALUES (1, +10), (2, 0), (3, -30);
+INSERT INTO `03_dml_insert_mode_table1` (`timestamp`, `value`, `dic`)
+    VALUES (1, +10, "d1"), (2, 0, "d2"), (3, -30, "d1");
 
 
 SELECT
@@ -42,15 +43,16 @@ DROP TABLE IF EXISTS `03_dml_insert_mode_table2`;
 CREATE TABLE `03_dml_insert_mode_table2` (
     `timestamp` timestamp NOT NULL,
     `value` double,
+    `dic` string dictionary,
     timestamp KEY (timestamp)) ENGINE=Analytic
 WITH(
 	 enable_ttl='false',
 	 update_mode='APPEND'
 );
 
 
-INSERT INTO `03_dml_insert_mode_table2` (`timestamp`, `value`)
-    VALUES (1, 10), (2, 20), (3, 30);
+INSERT INTO `03_dml_insert_mode_table2` (`timestamp`, `value`, `dic`)
+    VALUES (1, 10, "d1"), (2, 20, ""), (3, 30, "d2");
 
 SELECT
     *
@@ -59,8 +61,8 @@ FROM
 ORDER BY
     `value` ASC;
 
-INSERT INTO `03_dml_insert_mode_table2` (`timestamp`, `value`)
-    VALUES (1, 100), (2, 200), (3, 300);
+INSERT INTO `03_dml_insert_mode_table2` (`timestamp`, `value`, `dic`)
+    VALUES (1, 100, "d2"), (2, 200, "d1"), (3, 300, "");
 
 SELECT
     *
@@ -77,14 +79,18 @@ DROP TABLE IF EXISTS `03_dml_insert_mode_table3`;
 CREATE TABLE `03_dml_insert_mode_table3` (
     `timestamp` timestamp NOT NULL,
     `value` double,
+    `dic` string dictionary,
     timestamp KEY (timestamp)) ENGINE=Analytic
 WITH(
 	 enable_ttl='false'
 );
 
 
-INSERT INTO `03_dml_insert_mode_table3` (`timestamp`, `value`)
-    VALUES (1, 10), (2, 20), (3, 30);
+INSERT INTO `03_dml_insert_mode_table3` (`timestamp`, `value`, `dic`)
+    VALUES (1, 100, "d2"), (2, 200, "d1"), (3, 300, "d1");
+
+-- TODO support insert Null
+-- INSERT INTO `03_dml_insert_mode_table3` (`timestamp`, `value`, `dic`) VALUES (1, 100, "d2"), (2, 200, "d1"), (3, 300, Null);
 
 SELECT
     *
@@ -94,7 +100,7 @@ ORDER BY
     `value` ASC;
 
 INSERT INTO `03_dml_insert_mode_table3` (`timestamp`, `value`)
-    VALUES (1, 100), (2, 200), (3, 300);
+    VALUES (1, 100, "d5"), (2, 200, "d6"), (3, 300, "d7");
 
 
 SELECT
@@ -116,6 +122,7 @@ CREATE TABLE `03_dml_insert_mode_table4` (
     `c3` uint32 default c1 + 1,
     `c4` uint32 default c3 + 1,
     `c5` uint32 default c3 + 10,
+    `c6` string default "default",
     timestamp KEY (timestamp)) ENGINE=Analytic
 WITH(
 	 enable_ttl='false'