From 3281b2437bf2773fe7d633462e1aa2497d1a9114 Mon Sep 17 00:00:00 2001 From: Maximilian Roos Date: Tue, 28 Mar 2023 19:02:32 -0700 Subject: [PATCH 01/17] fix: Raise an error on unavailable excludes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #2292. It was here deliberately (the old code was _more_ work). I would think it's better to raise here — getting this wrong is still wrong... --- .github/workflows/test-all.yaml | 2 +- prql-compiler/src/sql/gen_projection.rs | 24 ++++++++----------- prql-compiler/src/tests/mod.rs | 1 + .../src/tests/test_error_messages.rs | 12 ++++++++++ 4 files changed, 24 insertions(+), 15 deletions(-) diff --git a/.github/workflows/test-all.yaml b/.github/workflows/test-all.yaml index caf8c30acc52..b63693cc67e0 100644 --- a/.github/workflows/test-all.yaml +++ b/.github/workflows/test-all.yaml @@ -13,7 +13,7 @@ name: test-all # - `test-all` — nightly + on request (the equivalent of `pr-test-all`), with # everything — benchmarks, compilation timing, audits, etc. # -# Also see pull-request.yaml#check-ok-to-merge for other thoughts +# Also see pull-request.yaml#check-ok-to-merge for further ideas. on: push: diff --git a/prql-compiler/src/sql/gen_projection.rs b/prql-compiler/src/sql/gen_projection.rs index 8b7bb862dfca..288d61ad71cb 100644 --- a/prql-compiler/src/sql/gen_projection.rs +++ b/prql-compiler/src/sql/gen_projection.rs @@ -141,8 +141,8 @@ pub(super) fn translate_select_items( // excluded columns let opts = (excluded.remove(&cid)) - .and_then(|excluded| translate_exclude(ctx, excluded)) - .unwrap_or_default(); + .and_then(|x| translate_exclude(ctx, x).transpose()) + .unwrap_or_else(|| Ok(WildcardAdditionalOptions::default()))?; Ok(if ident.len() > 1 { let mut object_name = ident; @@ -158,19 +158,15 @@ pub(super) fn translate_select_items( fn translate_exclude( ctx: &mut Context, excluded: HashSet, -) -> Option { +) -> Result> { let excluded = as_col_names(&excluded, &ctx.anchor); let Some(supported) = ctx.dialect.column_exclude() else { - // TODO: eventually this should throw an error - // I don't want to do this now, because we have no way around it. - // We could also ask the user to add table definitions. - if log::log_enabled!(log::Level::Warn) { - let excluded = excluded.join(", "); - - log::warn!("Columns {excluded} will be included with *, but were not requested.") - } - return None; + let excluded = excluded.join(", "); + // TODO: can we specify `span` here? + // TODO: can we get a nicer name for the dialect? + let dialect = &ctx.dialect; + return Err(Error::new_simple(format!("Excluding columns ({excluded}) is not supported by the current dialect, {dialect:?}")).with_help("Consider specifying the full set of columns prior with a `select`").into()); }; let mut excluded = excluded @@ -178,7 +174,7 @@ fn translate_exclude( .map(|name| translate_ident_part(name.to_string(), ctx)) .collect_vec(); - Some(match supported { + Ok(Some(match supported { ColumnExclude::Exclude => WildcardAdditionalOptions { opt_exclude: Some(ExcludeSelectItem::Multiple(excluded)), ..Default::default() @@ -190,7 +186,7 @@ fn translate_exclude( }), ..Default::default() }, - }) + })) } fn as_col_names<'a>(cids: &'a HashSet, ctx: &'a AnchorContext) -> Vec<&'a str> { diff --git a/prql-compiler/src/tests/mod.rs b/prql-compiler/src/tests/mod.rs index f3c39b54c907..73a82cc0c45b 100644 --- a/prql-compiler/src/tests/mod.rs +++ b/prql-compiler/src/tests/mod.rs @@ -1,4 +1,5 @@ mod test; mod test_bad_error_messages; +mod test_error_messages; pub use test::compile; diff --git a/prql-compiler/src/tests/test_error_messages.rs b/prql-compiler/src/tests/test_error_messages.rs index 8668ac78c33b..c5e99909de6a 100644 --- a/prql-compiler/src/tests/test_error_messages.rs +++ b/prql-compiler/src/tests/test_error_messages.rs @@ -126,3 +126,15 @@ fn test_hint_missing_args() { ───╯ "###) } + +#[test] +fn test_missing_exclude() { + // Not a great error message, would be better to have a span & a better + // name. Not quite bad enough to put in `bad_error_messages` yet though... + assert_display_snapshot!(compile(r###" + from tracks + select ![milliseconds,bytes] + "###).unwrap_err(), @r###" + Error: Excluding columns (milliseconds, bytes) is not supported by the current dialect, GenericDialect + "###) +} From 0a9ce6afa6bdadac59893e6bc246dc535b7b6800 Mon Sep 17 00:00:00 2001 From: Maximilian Roos Date: Wed, 29 Mar 2023 00:54:53 -0700 Subject: [PATCH 02/17] --- .github/workflows/test-all.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-all.yaml b/.github/workflows/test-all.yaml index b63693cc67e0..caf8c30acc52 100644 --- a/.github/workflows/test-all.yaml +++ b/.github/workflows/test-all.yaml @@ -13,7 +13,7 @@ name: test-all # - `test-all` — nightly + on request (the equivalent of `pr-test-all`), with # everything — benchmarks, compilation timing, audits, etc. # -# Also see pull-request.yaml#check-ok-to-merge for further ideas. +# Also see pull-request.yaml#check-ok-to-merge for other thoughts on: push: From 1fa791b89f882bb098bf5a9de3aeaa5e47a5705c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alja=C5=BE=20Mur=20Er=C5=BEen?= Date: Sun, 2 Apr 2023 12:58:55 +0200 Subject: [PATCH 03/17] a test --- prql-compiler/src/tests/test_error_messages.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/prql-compiler/src/tests/test_error_messages.rs b/prql-compiler/src/tests/test_error_messages.rs index c5e99909de6a..278f12403ce9 100644 --- a/prql-compiler/src/tests/test_error_messages.rs +++ b/prql-compiler/src/tests/test_error_messages.rs @@ -136,5 +136,12 @@ fn test_missing_exclude() { select ![milliseconds,bytes] "###).unwrap_err(), @r###" Error: Excluding columns (milliseconds, bytes) is not supported by the current dialect, GenericDialect - "###) + "###); + + assert_display_snapshot!(compile(r###" + from tracks + group title (take 1) + "###).unwrap_err(), @r###" + Error: Excluding columns (_expr_0) is not supported by the current dialect, GenericDialect + "###); } From 0ec22c4d062a1be8c8131b8656431449e2b2cbe4 Mon Sep 17 00:00:00 2001 From: Maximilian Roos <5635139+max-sixty@users.noreply.github.com> Date: Wed, 29 Mar 2023 18:51:27 -0700 Subject: [PATCH 04/17] refactor: Rename `set` to `type` (#2346) --- prql-compiler/src/ast/pl/expr.rs | 6 +- prql-compiler/src/ast/pl/fold.rs | 2 +- prql-compiler/src/ast/pl/types.rs | 56 +++++++++---------- prql-compiler/src/semantic/context.rs | 8 +-- prql-compiler/src/semantic/lowering.rs | 2 +- prql-compiler/src/semantic/resolver.rs | 8 +-- ...semantic__resolver__test__functions_1.snap | 2 +- ...tic__resolver__test__functions_nested.snap | 4 +- ..._resolver__test__functions_pipeline-2.snap | 6 +- ...c__resolver__test__functions_pipeline.snap | 2 +- ..._semantic__resolver__test__named_args.snap | 8 +-- ...semantic__resolver__test__variables_1.snap | 4 +- prql-compiler/src/semantic/transforms.rs | 2 +- prql-compiler/src/semantic/type_resolver.rs | 44 +++++++-------- 14 files changed, 77 insertions(+), 77 deletions(-) diff --git a/prql-compiler/src/ast/pl/expr.rs b/prql-compiler/src/ast/pl/expr.rs index 049495952e8d..ebe2377c0c1c 100644 --- a/prql-compiler/src/ast/pl/expr.rs +++ b/prql-compiler/src/ast/pl/expr.rs @@ -77,7 +77,7 @@ pub enum ExprKind { name: String, args: Vec, }, - Set(SetExpr), + Type(TypeExpr), /// a placeholder for values provided after query is compiled Param(String), @@ -599,8 +599,8 @@ impl Display for Expr { ExprKind::BuiltInFunction { .. } => { f.write_str("")?; } - ExprKind::Set(_) => { - writeln!(f, "")?; + ExprKind::Type(_) => { + writeln!(f, "")?; } ExprKind::Param(id) => { writeln!(f, "${id}")?; diff --git a/prql-compiler/src/ast/pl/fold.rs b/prql-compiler/src/ast/pl/fold.rs index 75bb3c2cf27c..66f38b14fe1e 100644 --- a/prql-compiler/src/ast/pl/fold.rs +++ b/prql-compiler/src/ast/pl/fold.rs @@ -119,7 +119,7 @@ pub fn fold_expr_kind(fold: &mut T, expr_kind: ExprKind) -> Param(id) => Param(id), // None of these capture variables, so we don't need to fold them. - Literal(_) | Set(_) => expr_kind, + Literal(_) | Type(_) => expr_kind, }) } diff --git a/prql-compiler/src/ast/pl/types.rs b/prql-compiler/src/ast/pl/types.rs index 622b3117020a..063189aee770 100644 --- a/prql-compiler/src/ast/pl/types.rs +++ b/prql-compiler/src/ast/pl/types.rs @@ -6,37 +6,37 @@ use serde::{Deserialize, Serialize}; use super::{Frame, Literal}; #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, EnumAsInner)] -pub enum SetExpr { - /// Set of a built-in primitive type +pub enum TypeExpr { + /// Type of a built-in primitive type Primitive(TyLit), - /// Set that contains only a literal value + /// Type that contains only a literal value Singleton(Literal), /// Union of sets (sum) - Union(Vec<(Option, SetExpr)>), + Union(Vec<(Option, TypeExpr)>), - /// Set of tuples (product) + /// Type of tuples (product) Tuple(Vec), - /// Set of arrays - Array(Box), + /// Type of arrays + Array(Box), - /// Set of sets. + /// Type of sets. /// Used for exprs that can be converted to SetExpr and then used as a Ty. - Set, + Type, } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub enum TupleElement { - Single(Option, SetExpr), + Single(Option, TypeExpr), Wildcard, } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, EnumAsInner)] pub enum Ty { /// Value is an element of this [SetExpr] - SetExpr(SetExpr), + TypeExpr(TypeExpr), /// Value is a function described by [TyFunc] // TODO: convert into [Ty::Domain]. @@ -93,7 +93,7 @@ impl Ty { // Not handled here. See type_resolver. (Ty::Infer, _) | (_, Ty::Infer) => false, - (Ty::SetExpr(left), Ty::SetExpr(right)) => left.is_superset_of(right), + (Ty::TypeExpr(left), Ty::TypeExpr(right)) => left.is_superset_of(right), (Ty::Table(_), Ty::Table(_)) => true, @@ -102,17 +102,17 @@ impl Ty { } } -impl SetExpr { - fn is_superset_of(&self, subset: &SetExpr) -> bool { +impl TypeExpr { + fn is_superset_of(&self, subset: &TypeExpr) -> bool { match (self, subset) { // TODO: convert these to array - (SetExpr::Primitive(TyLit::Column), SetExpr::Primitive(TyLit::Column)) => true, - (SetExpr::Primitive(TyLit::Column), SetExpr::Primitive(_)) => true, - (SetExpr::Primitive(_), SetExpr::Primitive(TyLit::Column)) => false, + (TypeExpr::Primitive(TyLit::Column), TypeExpr::Primitive(TyLit::Column)) => true, + (TypeExpr::Primitive(TyLit::Column), TypeExpr::Primitive(_)) => true, + (TypeExpr::Primitive(_), TypeExpr::Primitive(TyLit::Column)) => false, - (SetExpr::Primitive(l0), SetExpr::Primitive(r0)) => l0 == r0, - (SetExpr::Union(many), one) => many.iter().any(|(_, any)| any.is_superset_of(one)), - (one, SetExpr::Union(many)) => many.iter().all(|(_, each)| one.is_superset_of(each)), + (TypeExpr::Primitive(l0), TypeExpr::Primitive(r0)) => l0 == r0, + (TypeExpr::Union(many), one) => many.iter().any(|(_, any)| any.is_superset_of(one)), + (one, TypeExpr::Union(many)) => many.iter().all(|(_, each)| one.is_superset_of(each)), (l, r) => l == r, } @@ -122,7 +122,7 @@ impl SetExpr { impl Display for Ty { fn fmt(&self, f: &mut Formatter<'_>) -> Result { match &self { - Ty::SetExpr(lit) => write!(f, "{:}", lit), + Ty::TypeExpr(lit) => write!(f, "{:}", lit), Ty::Table(frame) => write!(f, "table<{frame}>"), Ty::Infer => write!(f, "infer"), Ty::Function(func) => { @@ -138,11 +138,11 @@ impl Display for Ty { } } -impl Display for SetExpr { +impl Display for TypeExpr { fn fmt(&self, f: &mut Formatter<'_>) -> Result { match &self { - SetExpr::Primitive(lit) => write!(f, "{:}", lit), - SetExpr::Union(ts) => { + TypeExpr::Primitive(lit) => write!(f, "{:}", lit), + TypeExpr::Union(ts) => { for (i, (_, e)) in ts.iter().enumerate() { write!(f, "{e}")?; if i < ts.len() - 1 { @@ -151,8 +151,8 @@ impl Display for SetExpr { } Ok(()) } - SetExpr::Singleton(lit) => write!(f, "{:}", lit), - SetExpr::Tuple(elements) => { + TypeExpr::Singleton(lit) => write!(f, "{:}", lit), + TypeExpr::Tuple(elements) => { write!(f, "[")?; for e in elements { match e { @@ -170,8 +170,8 @@ impl Display for SetExpr { } Ok(()) } - SetExpr::Set => write!(f, "set"), - SetExpr::Array(_) => todo!(), + TypeExpr::Type => write!(f, "set"), + TypeExpr::Array(_) => todo!(), } } } diff --git a/prql-compiler/src/semantic/context.rs b/prql-compiler/src/semantic/context.rs index 06854632f614..ec75b7ce5d73 100644 --- a/prql-compiler/src/semantic/context.rs +++ b/prql-compiler/src/semantic/context.rs @@ -388,16 +388,16 @@ fn get_stdlib_decl(name: &str) -> Option { "timestamp" => TyLit::Timestamp, "table" => { // TODO: this is just a dummy that gets intercepted when resolving types - return Some(ExprKind::Set(SetExpr::Array(Box::new(SetExpr::Singleton( - Literal::Null, - ))))); + return Some(ExprKind::Type(TypeExpr::Array(Box::new( + TypeExpr::Singleton(Literal::Null), + )))); } "column" => TyLit::Column, "list" => TyLit::List, "scalar" => TyLit::Scalar, _ => return None, }; - Some(ExprKind::Set(SetExpr::Primitive(ty_lit))) + Some(ExprKind::Type(TypeExpr::Primitive(ty_lit))) } impl Default for DeclKind { diff --git a/prql-compiler/src/semantic/lowering.rs b/prql-compiler/src/semantic/lowering.rs index 591363f0b5b9..c48fa2c96577 100644 --- a/prql-compiler/src/semantic/lowering.rs +++ b/prql-compiler/src/semantic/lowering.rs @@ -662,7 +662,7 @@ impl Lowerer { | pl::ExprKind::List(_) | pl::ExprKind::Closure(_) | pl::ExprKind::Pipeline(_) - | pl::ExprKind::Set(_) + | pl::ExprKind::Type(_) | pl::ExprKind::TransformCall(_) => { log::debug!("cannot lower {ast:?}"); return Err(Error::new(Reason::Unexpected { diff --git a/prql-compiler/src/semantic/resolver.rs b/prql-compiler/src/semantic/resolver.rs index e1dd506409c8..1a6c4accc7b8 100644 --- a/prql-compiler/src/semantic/resolver.rs +++ b/prql-compiler/src/semantic/resolver.rs @@ -77,7 +77,7 @@ impl AstFold for Resolver { name: ty_def.name, value: Box::new(ty_def.value.unwrap_or_else(|| { let mut e = Expr::null(); - e.ty = Some(Ty::SetExpr(SetExpr::Set)); + e.ty = Some(Ty::TypeExpr(TypeExpr::Type)); e })), }; @@ -448,7 +448,7 @@ impl Resolver { // evaluate let needs_window = (closure.body_ty) .as_ref() - .map(|ty| ty.is_superset_of(&Ty::SetExpr(SetExpr::Primitive(TyLit::Column)))) + .map(|ty| ty.is_superset_of(&Ty::TypeExpr(TypeExpr::Primitive(TyLit::Column)))) .unwrap_or_default(); let mut res = match self.cast_built_in_function(closure)? { @@ -780,11 +780,11 @@ impl Resolver { let set_expr = type_resolver::coerce_to_set(expr, &self.context)?; // TODO: workaround - if let SetExpr::Array(_) = set_expr { + if let TypeExpr::Array(_) = set_expr { return Ok(Some(Ty::Table(Frame::default()))); } - Some(Ty::SetExpr(set_expr)) + Some(Ty::TypeExpr(set_expr)) } None => None, }) diff --git a/prql-compiler/src/semantic/snapshots/prql_compiler__semantic__resolver__test__functions_1.snap b/prql-compiler/src/semantic/snapshots/prql_compiler__semantic__resolver__test__functions_1.snap index c8780eb17a9a..efaa4218bbd2 100644 --- a/prql-compiler/src/semantic/snapshots/prql_compiler__semantic__resolver__test__functions_1.snap +++ b/prql-compiler/src/semantic/snapshots/prql_compiler__semantic__resolver__test__functions_1.snap @@ -22,7 +22,7 @@ expression: "resolve_derive(r#\"\n func subtract a b -> a - b\n\n target_id: 7 ty: Infer ty: - SetExpr: + TypeExpr: Primitive: Column alias: net_salary diff --git a/prql-compiler/src/semantic/snapshots/prql_compiler__semantic__resolver__test__functions_nested.snap b/prql-compiler/src/semantic/snapshots/prql_compiler__semantic__resolver__test__functions_nested.snap index 75477a3642f6..6c352641ad83 100644 --- a/prql-compiler/src/semantic/snapshots/prql_compiler__semantic__resolver__test__functions_nested.snap +++ b/prql-compiler/src/semantic/snapshots/prql_compiler__semantic__resolver__test__functions_nested.snap @@ -40,7 +40,7 @@ expression: "resolve_derive(r#\"\n func lag_day x -> s\"lag_day_todo( Literal: Integer: 1 ty: - SetExpr: + TypeExpr: Primitive: Int ty: Infer op: Add @@ -53,6 +53,6 @@ expression: "resolve_derive(r#\"\n func lag_day x -> s\"lag_day_todo( target_id: 8 ty: Infer ty: - SetExpr: + TypeExpr: Primitive: Column diff --git a/prql-compiler/src/semantic/snapshots/prql_compiler__semantic__resolver__test__functions_pipeline-2.snap b/prql-compiler/src/semantic/snapshots/prql_compiler__semantic__resolver__test__functions_pipeline-2.snap index ea54921c5edc..c8d206fe5494 100644 --- a/prql-compiler/src/semantic/snapshots/prql_compiler__semantic__resolver__test__functions_pipeline-2.snap +++ b/prql-compiler/src/semantic/snapshots/prql_compiler__semantic__resolver__test__functions_pipeline-2.snap @@ -9,7 +9,7 @@ expression: "resolve_derive(r#\"\n func plus_one x -> x + 1\n Literal: Integer: 2 ty: - SetExpr: + TypeExpr: Primitive: Int op: Add right: @@ -34,11 +34,11 @@ expression: "resolve_derive(r#\"\n func plus_one x -> x + 1\n Literal: Integer: 1 ty: - SetExpr: + TypeExpr: Primitive: Int ty: Infer ty: - SetExpr: + TypeExpr: Primitive: Column alias: b diff --git a/prql-compiler/src/semantic/snapshots/prql_compiler__semantic__resolver__test__functions_pipeline.snap b/prql-compiler/src/semantic/snapshots/prql_compiler__semantic__resolver__test__functions_pipeline.snap index 335d1d89e686..a793f584cc1f 100644 --- a/prql-compiler/src/semantic/snapshots/prql_compiler__semantic__resolver__test__functions_pipeline.snap +++ b/prql-compiler/src/semantic/snapshots/prql_compiler__semantic__resolver__test__functions_pipeline.snap @@ -14,7 +14,7 @@ expression: "resolve_derive(r#\"\n from a\n derive one = ( target_id: 6 ty: Infer ty: - SetExpr: + TypeExpr: Primitive: Column alias: one diff --git a/prql-compiler/src/semantic/snapshots/prql_compiler__semantic__resolver__test__named_args.snap b/prql-compiler/src/semantic/snapshots/prql_compiler__semantic__resolver__test__named_args.snap index 5afd102820a2..59ec1d7e4315 100644 --- a/prql-compiler/src/semantic/snapshots/prql_compiler__semantic__resolver__test__named_args.snap +++ b/prql-compiler/src/semantic/snapshots/prql_compiler__semantic__resolver__test__named_args.snap @@ -18,10 +18,10 @@ expression: "resolve_derive(r#\"\n func add x to:1 -> x + to\n\n Literal: Integer: 3 ty: - SetExpr: + TypeExpr: Primitive: Int ty: - SetExpr: + TypeExpr: Primitive: Column alias: added - id: 25 @@ -40,10 +40,10 @@ expression: "resolve_derive(r#\"\n func add x to:1 -> x + to\n\n Literal: Integer: 1 ty: - SetExpr: + TypeExpr: Primitive: Int ty: - SetExpr: + TypeExpr: Primitive: Column alias: added_default diff --git a/prql-compiler/src/semantic/snapshots/prql_compiler__semantic__resolver__test__variables_1.snap b/prql-compiler/src/semantic/snapshots/prql_compiler__semantic__resolver__test__variables_1.snap index ea39d81c2bbc..c48ae4395985 100644 --- a/prql-compiler/src/semantic/snapshots/prql_compiler__semantic__resolver__test__variables_1.snap +++ b/prql-compiler/src/semantic/snapshots/prql_compiler__semantic__resolver__test__variables_1.snap @@ -22,7 +22,7 @@ expression: "resolve_derive(r#\"\n from employees\n derive target_id: 6 ty: Infer ty: - SetExpr: + TypeExpr: Primitive: Column alias: gross_salary - id: 16 @@ -44,7 +44,7 @@ expression: "resolve_derive(r#\"\n from employees\n derive target_id: 6 ty: Infer ty: - SetExpr: + TypeExpr: Primitive: Column alias: gross_cost diff --git a/prql-compiler/src/semantic/transforms.rs b/prql-compiler/src/semantic/transforms.rs index 00fb130d4ca4..41cc14b76180 100644 --- a/prql-compiler/src/semantic/transforms.rs +++ b/prql-compiler/src/semantic/transforms.rs @@ -1132,7 +1132,7 @@ mod tests { target_id: 6 ty: Infer ty: - SetExpr: + TypeExpr: Primitive: Column partition: - id: 12 diff --git a/prql-compiler/src/semantic/type_resolver.rs b/prql-compiler/src/semantic/type_resolver.rs index 229faacfb131..8f695e77879e 100644 --- a/prql-compiler/src/semantic/type_resolver.rs +++ b/prql-compiler/src/semantic/type_resolver.rs @@ -7,27 +7,27 @@ use crate::error::{Error, Reason, WithErrorInfo}; use super::Context; -/// Takes a resolved [Expr] and evaluates it a set expression that can be used to construct a type. -pub fn coerce_to_set(expr: Expr, context: &Context) -> Result { +/// Takes a resolved [Expr] and evaluates it a type expression that can be used to construct a type. +pub fn coerce_to_set(expr: Expr, context: &Context) -> Result { coerce_to_named_set(expr, context).map(|(_, s)| s) } -fn coerce_to_named_set(expr: Expr, context: &Context) -> Result<(Option, SetExpr), Error> { +fn coerce_to_named_set(expr: Expr, context: &Context) -> Result<(Option, TypeExpr), Error> { let name = expr.alias; let expr = coerce_kind_to_set(expr.kind, context).map_err(|e| e.with_span(expr.span))?; Ok((name, expr)) } -fn coerce_kind_to_set(expr: ExprKind, context: &Context) -> Result { +fn coerce_kind_to_set(expr: ExprKind, context: &Context) -> Result { // primitives - if let ExprKind::Set(set_expr) = expr { + if let ExprKind::Type(set_expr) = expr { return Ok(set_expr); } // singletons if let ExprKind::Literal(lit) = expr { - return Ok(SetExpr::Singleton(lit)); + return Ok(TypeExpr::Singleton(lit)); } // tuples @@ -35,12 +35,12 @@ fn coerce_kind_to_set(expr: ExprKind, context: &Context) -> Result Result Result { Ok(match &node.kind { ExprKind::Literal(ref literal) => match literal { Literal::Null => Ty::Infer, - Literal::Integer(_) => Ty::SetExpr(SetExpr::Primitive(TyLit::Int)), - Literal::Float(_) => Ty::SetExpr(SetExpr::Primitive(TyLit::Float)), - Literal::Boolean(_) => Ty::SetExpr(SetExpr::Primitive(TyLit::Bool)), - Literal::String(_) => Ty::SetExpr(SetExpr::Primitive(TyLit::Text)), - Literal::Date(_) => Ty::SetExpr(SetExpr::Primitive(TyLit::Date)), - Literal::Time(_) => Ty::SetExpr(SetExpr::Primitive(TyLit::Time)), - Literal::Timestamp(_) => Ty::SetExpr(SetExpr::Primitive(TyLit::Timestamp)), + Literal::Integer(_) => Ty::TypeExpr(TypeExpr::Primitive(TyLit::Int)), + Literal::Float(_) => Ty::TypeExpr(TypeExpr::Primitive(TyLit::Float)), + Literal::Boolean(_) => Ty::TypeExpr(TypeExpr::Primitive(TyLit::Bool)), + Literal::String(_) => Ty::TypeExpr(TypeExpr::Primitive(TyLit::Text)), + Literal::Date(_) => Ty::TypeExpr(TypeExpr::Primitive(TyLit::Date)), + Literal::Time(_) => Ty::TypeExpr(TypeExpr::Primitive(TyLit::Time)), + Literal::Timestamp(_) => Ty::TypeExpr(TypeExpr::Primitive(TyLit::Timestamp)), Literal::ValueAndUnit(_) => Ty::Infer, // TODO Literal::Relation(_) => unreachable!(), }, @@ -97,11 +97,11 @@ pub fn infer_type(node: &Expr, context: &Context) -> Result { ExprKind::Ident(_) | ExprKind::Pipeline(_) | ExprKind::FuncCall(_) => Ty::Infer, ExprKind::SString(_) => Ty::Infer, - ExprKind::FString(_) => Ty::SetExpr(SetExpr::Primitive(TyLit::Text)), + ExprKind::FString(_) => Ty::TypeExpr(TypeExpr::Primitive(TyLit::Text)), ExprKind::Range(_) => Ty::Infer, // TODO ExprKind::TransformCall(call) => Ty::Table(call.infer_type(context)?), - ExprKind::List(_) => Ty::SetExpr(SetExpr::Primitive(TyLit::List)), + ExprKind::List(_) => Ty::TypeExpr(TypeExpr::Primitive(TyLit::List)), _ => Ty::Infer, }) From cc1f3b97e8637b862c2d8254af2e56b6bbb209a5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 30 Mar 2023 02:04:25 +0000 Subject: [PATCH 05/17] chore: bump clap from 4.1.10 to 4.2.0 (#2345) * chore: bump clap from 4.1.10 to 4.2.0 Bumps [clap](https://github.com/clap-rs/clap) from 4.1.10 to 4.2.0. - [Release notes](https://github.com/clap-rs/clap/releases) - [Changelog](https://github.com/clap-rs/clap/blob/master/CHANGELOG.md) - [Commits](https://github.com/clap-rs/clap/compare/v4.1.10...clap_complete-v4.2.0) --- updated-dependencies: - dependency-name: clap dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Maximilian Roos --- Cargo.lock | 118 ++++++++++++++++++++------------- prql-compiler/prqlc/Cargo.toml | 2 +- web/book/Cargo.toml | 2 +- 3 files changed, 75 insertions(+), 47 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1f80b8df6e23..1b114603d793 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -65,6 +65,46 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" +[[package]] +name = "anstream" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "342258dd14006105c2b75ab1bd7543a03bdf0cfc94383303ac212a04939dff6f" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-wincon", + "concolor-override", + "concolor-query", + "is-terminal", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23ea9e81bd02e310c216d080f6223c179012256e5151c41db88d12c88a1684d2" + +[[package]] +name = "anstyle-parse" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7d1bb534e9efed14f3e5f44e7dd1a4f709384023a4165199a4241e18dff0116" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-wincon" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3127af6145b149f3287bb9a0d10ad9c5692dba8c53ad48285e5bec4063834fa" +dependencies = [ + "anstyle", + "windows-sys 0.45.0", +] + [[package]] name = "anyhow" version = "1.0.70" @@ -505,17 +545,27 @@ dependencies = [ [[package]] name = "clap" -version = "4.1.10" +version = "4.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce38afc168d8665cfc75c7b1dd9672e50716a137f433f070991619744a67342a" +checksum = "6efb5f0a41b5ef5b50c5da28c07609c20091df0c1fc33d418fa2a7e693c2b624" dependencies = [ - "bitflags 1.3.2", + "clap_builder", "clap_derive", - "clap_lex 0.3.3", - "is-terminal", + "once_cell", +] + +[[package]] +name = "clap_builder" +version = "4.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "671fcaa5debda4b9a84aa7fde49c907c8986c0e6ab927e04217c9cb74e7c8bc9" +dependencies = [ + "anstream", + "anstyle", + "bitflags 1.3.2", + "clap_lex 0.4.1", "once_cell", "strsim", - "termcolor", "terminal_size", ] @@ -525,20 +575,19 @@ version = "4.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37686beaba5ac9f3ab01ee3172f792fc6ffdd685bfb9e63cfef02c0571a4e8e1" dependencies = [ - "clap 4.1.10", + "clap 4.2.0", ] [[package]] name = "clap_derive" -version = "4.1.9" +version = "4.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fddf67631444a3a3e3e5ac51c36a5e01335302de677bd78759eaa90ab1f46644" +checksum = "3f9644cd56d6b87dbe899ef8b053e331c0637664e9e21a33dfcdc36093f5c5c4" dependencies = [ "heck", - "proc-macro-error", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.8", ] [[package]] @@ -552,12 +601,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.3.3" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "033f6b7a4acb1f358c742aaca805c939ee73b4c6209ae4318ec7aca81c42e646" -dependencies = [ - "os_str_bytes", -] +checksum = "8a2dd5a6fe8c6e3502f568a6353e5273bbb15193ad9a89e457b9970798efbea1" [[package]] name = "clio" @@ -565,7 +611,7 @@ version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94b38784f05c5e908bb8751e9b9f29fbcd470f636c0d0a76a0f90c0c823f3b68" dependencies = [ - "clap 4.1.10", + "clap 4.2.0", "libc", "windows-sys 0.42.0", ] @@ -653,7 +699,7 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "435ff0007a3bb04099fe1beedc6b76e7dd5340c90b168008ac0d7e87441de1bf" dependencies = [ - "clap 4.1.10", + "clap 4.2.0", "concolor", ] @@ -1606,7 +1652,7 @@ checksum = "764dcbfc2e5f868bc1b566eb179dff1a06458fd0cff846aae2579392dd3f01a0" dependencies = [ "anyhow", "chrono", - "clap 4.1.10", + "clap 4.2.0", "clap_complete", "env_logger", "handlebars", @@ -1629,7 +1675,7 @@ name = "mdbook-prql" version = "0.6.1" dependencies = [ "anyhow", - "clap 4.1.10", + "clap 4.2.0", "globset", "insta", "itertools", @@ -2070,30 +2116,6 @@ dependencies = [ "toml", ] -[[package]] -name = "proc-macro-error" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" -dependencies = [ - "proc-macro-error-attr", - "proc-macro2", - "quote", - "syn 1.0.109", - "version_check", -] - -[[package]] -name = "proc-macro-error-attr" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" -dependencies = [ - "proc-macro2", - "quote", - "version_check", -] - [[package]] name = "proc-macro-hack" version = "0.5.20+deprecated" @@ -2201,7 +2223,7 @@ dependencies = [ "anyhow", "ariadne", "atty", - "clap 4.1.10", + "clap 4.2.0", "clio", "color-eyre", "concolor", @@ -3120,6 +3142,12 @@ version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ad2024452afd3874bf539695e04af6732ba06517424dbf958fdb16a01f3bef6c" +[[package]] +name = "utf8parse" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" + [[package]] name = "valuable" version = "0.1.0" diff --git a/prql-compiler/prqlc/Cargo.toml b/prql-compiler/prqlc/Cargo.toml index 9545799c8c4e..75fe64717f40 100644 --- a/prql-compiler/prqlc/Cargo.toml +++ b/prql-compiler/prqlc/Cargo.toml @@ -12,7 +12,7 @@ version.workspace = true anyhow = {version = "1.0.57"} ariadne = "0.2.0" atty = "0.2.14" -clap = {version = "4.1.1", features = ["derive"]} +clap = {version = "4.2.0", features = ["derive"]} clio = {version = "0.2.4", features = ['clap-parse']} color-eyre = "0.6.1" concolor = "0.1.0" diff --git a/web/book/Cargo.toml b/web/book/Cargo.toml index 349fdd567309..5e80f7f5cb80 100644 --- a/web/book/Cargo.toml +++ b/web/book/Cargo.toml @@ -31,7 +31,7 @@ walkdir = "2.3.2" # Possibly we should just put all dependencies as excluded from wasm? Currently # we only put the ones that break the build. [target.'cfg(not(target_family="wasm"))'.dependencies] -clap = {version = "4.1.1", default-features = false} +clap = {version = "4.2.0", default-features = false} mdbook = {version = "0.4.21", default-features = false} [dev-dependencies] From 03c335d3308fbbfc1bcc4c7ec75e791f9c471e8f Mon Sep 17 00:00:00 2001 From: Jelenkee <59470612+Jelenkee@users.noreply.github.com> Date: Thu, 30 Mar 2023 07:01:16 +0200 Subject: [PATCH 06/17] test: more RDBMS integration tests in CI (#2286) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * -added docker compose -added test skeleton * -added dependencies * -added real test cases * -added mssql * refactoring * -added go-task * -more testcases * -added github workflow * -fixed formatting * -fixed linting * -fixed workflow yaml * -fixed workflow yaml again * -added more testcases * Apply suggestions from code review Co-authored-by: eitsupi <50911393+eitsupi@users.noreply.github.com> * -refactored test workflow * Fix order of args to `assert_display_snapshot` Sorry for the incorrect suggestion prior! * Specify platform for mysql image * -added csv import * -fixed or skipped all prql files * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * -fixed sqlite * added snaps§ * Update prql-compiler/prqlc/src/cli.rs * -removed test on macOS * empty * -skip integration test on windows * -fixed taskfile * -added double quotes * Update .github/workflows/test-rust.yaml --------- Co-authored-by: eitsupi <50911393+eitsupi@users.noreply.github.com> Co-authored-by: Maximilian Roos Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Maximilian Roos <5635139+max-sixty@users.noreply.github.com> --- .github/workflows/test-rust.yaml | 13 + .github/workflows/test-taskfile.yaml | 3 + Cargo.lock | 941 +++++++++++++++++- Taskfile.yml | 4 +- prql-compiler/Cargo.toml | 6 + .../tests/integration-rdbms/conf/my.cnf | 2 + .../tests/integration-rdbms/connection.rs | 316 ++++++ .../integration-rdbms/docker-compose.yml | 47 + prql-compiler/tests/integration-rdbms/main.rs | 182 ++++ ...ion_rdbms__tests__rdbms@distinct.prql.snap | 6 + ...rdbms__tests__rdbms@genre_counts.prql.snap | 6 + ...on_rdbms__tests__rdbms@group_all.prql.snap | 6 + ...bms__tests__rdbms@invoice_totals.prql.snap | 6 + ...gration_rdbms__tests__rdbms@loop.prql.snap | 6 + ...on_rdbms__tests__rdbms@pipelines.prql.snap | 6 + ...bms__tests__rdbms@set_ops_remove.prql.snap | 6 + ...ation_rdbms__tests__rdbms@switch.prql.snap | 6 + .../integration/data/chinook/employees.csv | 2 +- .../tests/integration/data/chinook/schema.sql | 32 +- .../tests/integration/queries/group_all.prql | 5 +- .../integration/queries/invoice_totals.prql | 2 + .../tests/integration/queries/loop.prql | 1 + .../tests/integration/queries/pipelines.prql | 1 + .../integration/queries/set_ops_remove.prql | 1 + .../tests/integration/queries/switch.prql | 2 + ...tegration__tests__test@group_all.prql.snap | 14 +- 26 files changed, 1576 insertions(+), 46 deletions(-) create mode 100644 prql-compiler/tests/integration-rdbms/conf/my.cnf create mode 100644 prql-compiler/tests/integration-rdbms/connection.rs create mode 100644 prql-compiler/tests/integration-rdbms/docker-compose.yml create mode 100644 prql-compiler/tests/integration-rdbms/main.rs create mode 100644 prql-compiler/tests/integration-rdbms/snapshots/integration_rdbms__tests__rdbms@distinct.prql.snap create mode 100644 prql-compiler/tests/integration-rdbms/snapshots/integration_rdbms__tests__rdbms@genre_counts.prql.snap create mode 100644 prql-compiler/tests/integration-rdbms/snapshots/integration_rdbms__tests__rdbms@group_all.prql.snap create mode 100644 prql-compiler/tests/integration-rdbms/snapshots/integration_rdbms__tests__rdbms@invoice_totals.prql.snap create mode 100644 prql-compiler/tests/integration-rdbms/snapshots/integration_rdbms__tests__rdbms@loop.prql.snap create mode 100644 prql-compiler/tests/integration-rdbms/snapshots/integration_rdbms__tests__rdbms@pipelines.prql.snap create mode 100644 prql-compiler/tests/integration-rdbms/snapshots/integration_rdbms__tests__rdbms@set_ops_remove.prql.snap create mode 100644 prql-compiler/tests/integration-rdbms/snapshots/integration_rdbms__tests__rdbms@switch.prql.snap diff --git a/.github/workflows/test-rust.yaml b/.github/workflows/test-rust.yaml index 2fdd27850a57..b3c67090ed78 100644 --- a/.github/workflows/test-rust.yaml +++ b/.github/workflows/test-rust.yaml @@ -54,6 +54,19 @@ jobs: with: command: test args: ${{ inputs.target_option }} --no-run --locked + - name: Run docker compose + run: docker compose up -d + working-directory: ./prql-compiler/tests/integration-rdbms + if: ${{ inputs.os == 'ubuntu-latest' }} + - name: Wait for database + uses: ifaxity/wait-on-action@v1 + with: + resource: "tcp:5432 tcp:3306 tcp:1433" + timeout: 60000 + if: ${{ inputs.os == 'ubuntu-latest' }} + - name: skip integration if not linux + run: echo "SKIP_INTEGRATION=true" >> "$GITHUB_ENV" + if: ${{ inputs.os != 'ubuntu-latest' }} # Only check unreferenced snapshots on the default target tests on ubuntu # # (Maybe there's a nicer approach where we can parameterize one step diff --git a/.github/workflows/test-taskfile.yaml b/.github/workflows/test-taskfile.yaml index 365c689ae889..48e3eba6b4e5 100644 --- a/.github/workflows/test-taskfile.yaml +++ b/.github/workflows/test-taskfile.yaml @@ -12,6 +12,9 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }}-taskfile cancel-in-progress: true +env: + SKIP_INTEGRATION: true + jobs: test-taskfile: runs-on: macos-latest diff --git a/Cargo.lock b/Cargo.lock index 1b114603d793..4ce528521382 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -23,7 +23,7 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47" dependencies = [ - "getrandom", + "getrandom 0.2.8", "once_cell", "version_check", ] @@ -36,7 +36,7 @@ checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f" dependencies = [ "cfg-if", "const-random", - "getrandom", + "getrandom 0.2.8", "once_cell", "version_check", ] @@ -285,6 +285,18 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "async-native-tls" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d57d4cec3c647232e1094dc013546c0b33ce785d8aeb251e1f20dfaf8a9a13fe" +dependencies = [ + "futures-util", + "native-tls", + "thiserror", + "url", +] + [[package]] name = "async-trait" version = "0.1.67" @@ -296,6 +308,19 @@ dependencies = [ "syn 2.0.8", ] +[[package]] +name = "asynchronous-codec" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06a0daa378f5fd10634e44b0a29b2a87b890657658e072a30d6f26e57ddee182" +dependencies = [ + "bytes", + "futures-sink", + "futures-util", + "memchr", + "pin-project-lite", +] + [[package]] name = "atty" version = "0.2.14" @@ -334,6 +359,36 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" +[[package]] +name = "bigdecimal" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6aaf33151a6429fe9211d1b276eafdf70cdff28b071e76c0b0e1503221ea3744" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + +[[package]] +name = "bindgen" +version = "0.59.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bd2a9a458e8f4304c52c43ebb0cfbd520289f8379a52e329a38afda99bf8eb8" +dependencies = [ + "bitflags 1.3.2", + "cexpr", + "clang-sys", + "lazy_static", + "lazycell", + "peeking_take_while", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -346,6 +401,18 @@ version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "487f1e0fcbe47deb8b0574e646def1c903389d95241dd1bbcc6ce4a715dfc0c1" +[[package]] +name = "bitvec" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + [[package]] name = "block-buffer" version = "0.10.4" @@ -412,6 +479,12 @@ dependencies = [ "serde", ] +[[package]] +name = "bufstream" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40e38929add23cdf8a366df9b0e088953150724bcbe5fc330b0d8eb3b328eec8" + [[package]] name = "bumpalo" version = "3.12.0" @@ -473,6 +546,15 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + [[package]] name = "cfg-if" version = "1.0.0" @@ -489,7 +571,7 @@ dependencies = [ "js-sys", "num-integer", "num-traits", - "time", + "time 0.1.45", "wasm-bindgen", "winapi", ] @@ -531,6 +613,17 @@ dependencies = [ "half 1.8.2", ] +[[package]] +name = "clang-sys" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c688fc74432808e3eb684cae8830a86be1d66a2bd58e1f248ed0960a590baf6f" +dependencies = [ + "glob", + "libc", + "libloading", +] + [[package]] name = "clap" version = "3.2.23" @@ -616,6 +709,15 @@ dependencies = [ "windows-sys 0.42.0", ] +[[package]] +name = "cmake" +version = "0.1.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31c789563b815f77f4250caee12365734369f942439b7defd71e18a48197130" +dependencies = [ + "cc", +] + [[package]] name = "codespan-reporting" version = "0.11.1" @@ -718,6 +820,15 @@ dependencies = [ "windows-sys 0.45.0", ] +[[package]] +name = "connection-string" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97faeec45f49581c458f8bf81992c5e3ec17d82cda99f59d3cea14eff62698d" +dependencies = [ + "wasm-bindgen", +] + [[package]] name = "console" version = "0.15.5" @@ -756,12 +867,22 @@ version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d7d6ab3c3a2282db210df5f02c4dab6e0a7057af0fb7ebd4070f30fe05c0ddb" dependencies = [ - "getrandom", + "getrandom 0.2.8", "once_cell", "proc-macro-hack", "tiny-keccak", ] +[[package]] +name = "core-foundation" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation-sys" version = "0.8.3" @@ -777,6 +898,15 @@ dependencies = [ "libc", ] +[[package]] +name = "crc32fast" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +dependencies = [ + "cfg-if", +] + [[package]] name = "criterion" version = "0.4.0" @@ -813,6 +943,20 @@ dependencies = [ "itertools", ] +[[package]] +name = "crossbeam" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2801af0d36612ae591caa9568261fddce32ce6e08a7275ea334a06a4ad021a2c" +dependencies = [ + "cfg-if", + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-epoch", + "crossbeam-queue", + "crossbeam-utils", +] + [[package]] name = "crossbeam-channel" version = "0.5.7" @@ -847,6 +991,16 @@ dependencies = [ "scopeguard", ] +[[package]] +name = "crossbeam-queue" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1cfb3ea8a53f37c40dea2c7bedcbd88bdfae54f5e2175d6ecaff1c988353add" +dependencies = [ + "cfg-if", + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.15" @@ -947,6 +1101,17 @@ dependencies = [ "syn 2.0.8", ] +[[package]] +name = "derive_utils" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dff8f6a793f528719e1ad4425a52a213ac1214ac7158c5fb97a7f50a64bfc96d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.8", +] + [[package]] name = "diff" version = "0.1.13" @@ -995,6 +1160,70 @@ version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" +[[package]] +name = "encoding" +version = "0.2.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b0d943856b990d12d3b55b359144ff341533e516d94098b1d3fc1ac666d36ec" +dependencies = [ + "encoding-index-japanese", + "encoding-index-korean", + "encoding-index-simpchinese", + "encoding-index-singlebyte", + "encoding-index-tradchinese", +] + +[[package]] +name = "encoding-index-japanese" +version = "1.20141219.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04e8b2ff42e9a05335dbf8b5c6f7567e5591d0d916ccef4e0b1710d32a0d0c91" +dependencies = [ + "encoding_index_tests", +] + +[[package]] +name = "encoding-index-korean" +version = "1.20141219.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dc33fb8e6bcba213fe2f14275f0963fd16f0a02c878e3095ecfdf5bee529d81" +dependencies = [ + "encoding_index_tests", +] + +[[package]] +name = "encoding-index-simpchinese" +version = "1.20141219.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d87a7194909b9118fc707194baa434a4e3b0fb6a5a757c73c3adb07aa25031f7" +dependencies = [ + "encoding_index_tests", +] + +[[package]] +name = "encoding-index-singlebyte" +version = "1.20141219.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3351d5acffb224af9ca265f435b859c7c01537c0849754d3db3fdf2bfe2ae84a" +dependencies = [ + "encoding_index_tests", +] + +[[package]] +name = "encoding-index-tradchinese" +version = "1.20141219.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd0e20d5688ce3cab59eb3ef3a2083a5c77bf496cb798dc6fcdb75f323890c18" +dependencies = [ + "encoding_index_tests", +] + +[[package]] +name = "encoding_index_tests" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a246d82be1c9d791c5dfde9a2bd045fc3cbba3fa2b11ad558f27d01712f00569" + [[package]] name = "enum-as-inner" version = "0.5.1" @@ -1007,6 +1236,26 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "enumflags2" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e75d4cd21b95383444831539909fbb14b9dc3fdceb2a6f5d36577329a1f55ccb" +dependencies = [ + "enumflags2_derive", +] + +[[package]] +name = "enumflags2_derive" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f58dc3c5e468259f19f2d46304a6b28f1c3d034442e14b322d2b850e36f6d5ae" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "env_logger" version = "0.10.0" @@ -1084,12 +1333,111 @@ dependencies = [ "windows-sys 0.45.0", ] +[[package]] +name = "flate2" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8a2db397cb1c8772f31494cb8917e48cd1e64f0fa7efac59fbd741a0a8ce841" +dependencies = [ + "crc32fast", + "libz-sys", + "miniz_oxide", +] + [[package]] name = "fnv" version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9c384f161156f5260c24a097c56119f9be8c798586aecc13afbcbe7b7e26bf8" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "frunk" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89c703bf50009f383a0873845357cc400a95fc535f836feddfe015d7df6e1e0" +dependencies = [ + "frunk_core", + "frunk_derives", + "frunk_proc_macros", +] + +[[package]] +name = "frunk_core" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a446d01a558301dca28ef43222864a9fa2bd9a2e71370f769d5d5d5ec9f3537" + +[[package]] +name = "frunk_derives" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b83164912bb4c97cfe0772913c7af7387ee2e00cb6d4636fb65a35b3d0c8f173" +dependencies = [ + "frunk_proc_macro_helpers", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "frunk_proc_macro_helpers" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "015425591bbeb0f5b8a75593340f1789af428e9f887a4f1e36c0c471f067ef50" +dependencies = [ + "frunk_core", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "frunk_proc_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea01524f285deab48affffb342b97f186e657b119c3f1821ac531780e0fbfae0" +dependencies = [ + "frunk_core", + "frunk_proc_macros_impl", + "proc-macro-hack", +] + +[[package]] +name = "frunk_proc_macros_impl" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a802d974cc18ee7fe1a7868fc9ce31086294fd96ba62f8da64ecb44e92a2653" +dependencies = [ + "frunk_core", + "frunk_proc_macro_helpers", + "proc-macro-hack", + "quote", + "syn 1.0.109", +] + [[package]] name = "fsevent-sys" version = "4.1.0" @@ -1099,6 +1447,12 @@ dependencies = [ "libc", ] +[[package]] +name = "funty" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" + [[package]] name = "futures-channel" version = "0.3.27" @@ -1115,6 +1469,12 @@ version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "86d7a0c1aa76363dac491de0ee99faf6941128376f1cf96f07db7603b7de69dd" +[[package]] +name = "futures-io" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89d422fa3cbe3b40dca574ab087abb5bc98258ea57eea3fd6f1fa7162c778b91" + [[package]] name = "futures-macro" version = "0.3.27" @@ -1145,9 +1505,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ef6b17e481503ec85211fed8f39d1970f128935ca1f814cd32ac4a6842e84ab" dependencies = [ "futures-core", + "futures-io", "futures-macro", "futures-sink", "futures-task", + "memchr", "pin-project-lite", "pin-utils", "slab", @@ -1172,6 +1534,17 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "getrandom" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.9.0+wasi-snapshot-preview1", +] + [[package]] name = "getrandom" version = "0.2.8" @@ -1189,6 +1562,12 @@ version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ad0a93d233ebf96623465aad4046a8d3aa4da22d4f4beba5388838c8a434bbb4" +[[package]] +name = "glob" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" + [[package]] name = "globset" version = "0.4.10" @@ -1328,6 +1707,16 @@ dependencies = [ "cxx-build", ] +[[package]] +name = "idna" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + [[package]] name = "indenter" version = "0.3.3" @@ -1395,6 +1784,16 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "io-enum" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01c662c349c9c9f542e7bfd9134143beb27da4b20dfbc3b3ef5b2a5b507dafbd" +dependencies = [ + "derive_utils", + "syn 2.0.8", +] + [[package]] name = "io-lifetimes" version = "1.0.8" @@ -1497,6 +1896,21 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + +[[package]] +name = "lexical" +version = "6.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7aefb36fd43fef7003334742cbf77b243fcd36418a1d1bdd480d613a67968f6" +dependencies = [ + "lexical-core", +] + [[package]] name = "lexical-core" version = "0.8.5" @@ -1578,6 +1992,16 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "libloading" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67380fd3b2fbe7527a606e18729d21c6f3951633d0500574c4dc22d2d638b9f" +dependencies = [ + "cfg-if", + "winapi", +] + [[package]] name = "libm" version = "0.2.6" @@ -1595,6 +2019,17 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "libz-sys" +version = "1.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9702761c3935f8cc2f101793272e202c72b99da8f4224a19ddcf1279a6450bbf" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + [[package]] name = "link-cplusplus" version = "1.0.8" @@ -1635,6 +2070,15 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "lru" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6e8aaa3f231bb4bd57b84b2d5dc3ae7f350265df8aa96492e0bc394a1571909" +dependencies = [ + "hashbrown 0.12.3", +] + [[package]] name = "md-5" version = "0.10.5" @@ -1644,6 +2088,12 @@ dependencies = [ "digest", ] +[[package]] +name = "md5" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e6bcd6433cff03a4bfc3d9834d504467db1f1cf6d0ea765d37d330249ed629d" + [[package]] name = "mdbook" version = "0.4.28" @@ -1716,30 +2166,120 @@ dependencies = [ ] [[package]] -name = "minimal-lexical" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" - -[[package]] -name = "miniz_oxide" -version = "0.6.2" +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "miniz_oxide" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b275950c28b37e794e8c55d88aeb5e139d0ce23fdbbeda68f8d7174abdf9e8fa" +dependencies = [ + "adler", +] + +[[package]] +name = "mio" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b9d9a46eff5b4ff64b45a9e316a6d1e0bc719ef429cbec4dc630684212bfdf9" +dependencies = [ + "libc", + "log", + "wasi 0.11.0+wasi-snapshot-preview1", + "windows-sys 0.45.0", +] + +[[package]] +name = "mysql" +version = "23.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f11339ca5c251941805d51362a07823605a80586ced92914ab7de84fba813f" +dependencies = [ + "bufstream", + "bytes", + "crossbeam", + "flate2", + "io-enum", + "libc", + "lru", + "mysql_common", + "named_pipe", + "native-tls", + "once_cell", + "pem", + "percent-encoding", + "serde", + "serde_json", + "socket2", + "twox-hash", + "url", +] + +[[package]] +name = "mysql_common" +version = "0.29.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9006c95034ccf7b903d955f210469119f6c3477fc9c9e7a7845ce38a3e665c2a" +dependencies = [ + "base64", + "bigdecimal", + "bindgen", + "bitflags 1.3.2", + "bitvec", + "byteorder", + "bytes", + "cc", + "cmake", + "crc32fast", + "flate2", + "frunk", + "lazy_static", + "lexical", + "num-bigint", + "num-traits", + "rand 0.8.5", + "regex", + "rust_decimal", + "saturating", + "serde", + "serde_json", + "sha1", + "sha2", + "smallvec", + "subprocess", + "thiserror", + "time 0.3.20", + "uuid", +] + +[[package]] +name = "named_pipe" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b275950c28b37e794e8c55d88aeb5e139d0ce23fdbbeda68f8d7174abdf9e8fa" +checksum = "ad9c443cce91fc3e12f017290db75dde490d685cdaaf508d7159d7cf41f0eb2b" dependencies = [ - "adler", + "winapi", ] [[package]] -name = "mio" -version = "0.8.6" +name = "native-tls" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b9d9a46eff5b4ff64b45a9e316a6d1e0bc719ef429cbec4dc630684212bfdf9" +checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" dependencies = [ + "lazy_static", "libc", "log", - "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys 0.45.0", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", ] [[package]] @@ -1888,6 +2428,51 @@ dependencies = [ "winapi", ] +[[package]] +name = "openssl" +version = "0.10.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "518915b97df115dd36109bfa429a48b8f737bd05508cf9588977b599648926d2" +dependencies = [ + "bitflags 1.3.2", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + +[[package]] +name = "openssl-sys" +version = "0.9.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "666416d899cf077260dac8698d60a60b435a46d57e82acb1be3d0dad87284e5b" +dependencies = [ + "autocfg", + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "os_str_bytes" version = "6.4.1" @@ -1932,6 +2517,21 @@ dependencies = [ "windows-sys 0.45.0", ] +[[package]] +name = "peeking_take_while" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" + +[[package]] +name = "pem" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8835c273a76a90455d7344889b0964598e3316e2a79ede8e36f16bdcf2228b8" +dependencies = [ + "base64", +] + [[package]] name = "percent-encoding" version = "2.2.0" @@ -1982,6 +2582,19 @@ dependencies = [ "sha2", ] +[[package]] +name = "pg_bigdecimal" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9855a94c74528af62c0ea236577af5e601263c1c404a6ac939b07c97c8e0216" +dependencies = [ + "bigdecimal", + "byteorder", + "bytes", + "num", + "postgres", +] + [[package]] name = "phf" version = "0.11.1" @@ -2073,7 +2686,7 @@ dependencies = [ "hmac", "md-5", "memchr", - "rand", + "rand 0.8.5", "sha2", "stringprep", ] @@ -2095,6 +2708,12 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" +[[package]] +name = "pretty-hex" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6fa0831dd7cc608c38a5e323422a0077678fa5744aa2be4ad91c4ece8eec8d5" + [[package]] name = "pretty_assertions" version = "1.3.0" @@ -2156,7 +2775,9 @@ dependencies = [ "itertools", "lazy_static", "log", + "mysql", "once_cell", + "pg_bigdecimal", "postgres", "pretty_assertions", "regex", @@ -2169,6 +2790,9 @@ dependencies = [ "sqlparser", "strum", "strum_macros", + "tiberius", + "tokio", + "tokio-util", ] [[package]] @@ -2360,6 +2984,25 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "radium" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" + +[[package]] +name = "rand" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" +dependencies = [ + "getrandom 0.1.16", + "libc", + "rand_chacha 0.2.2", + "rand_core 0.5.1", + "rand_hc", +] + [[package]] name = "rand" version = "0.8.5" @@ -2367,8 +3010,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", - "rand_chacha", - "rand_core", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" +dependencies = [ + "ppv-lite86", + "rand_core 0.5.1", ] [[package]] @@ -2378,7 +3031,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" +dependencies = [ + "getrandom 0.1.16", ] [[package]] @@ -2387,7 +3049,16 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom", + "getrandom 0.2.8", +] + +[[package]] +name = "rand_hc" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" +dependencies = [ + "rand_core 0.5.1", ] [[package]] @@ -2505,7 +3176,7 @@ dependencies = [ "byteorder", "bytes", "num-traits", - "rand", + "rand 0.8.5", "rkyv", "serde", "serde_json", @@ -2517,6 +3188,12 @@ version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ef03e0a2b150c7a90d01faf6254c9c48a41e95fb2a8c2ac1c6f0d2b9aefc342" +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + [[package]] name = "rustix" version = "0.36.10" @@ -2585,6 +3262,21 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "saturating" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ece8e78b2f38ec51c51f5d475df0a7187ba5111b2a28bdc761ee05b075d40a71" + +[[package]] +name = "schannel" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "713cfb06c7059f3588fb8044c0fad1d09e3c01d225e25b9220dbfdcf16dbb1b3" +dependencies = [ + "windows-sys 0.42.0", +] + [[package]] name = "scoped-tls" version = "1.0.1" @@ -2609,6 +3301,29 @@ version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" +[[package]] +name = "security-framework" +version = "2.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a332be01508d814fed64bf28f798a146d73792121129962fdf335bb3c49a4254" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31c9bb296072e961fcbd8853511dd39c2d8be2deb1e17c6860b1d30732b323b4" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "semver" version = "1.0.17" @@ -2662,6 +3377,17 @@ dependencies = [ "unsafe-libyaml", ] +[[package]] +name = "sha1" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "sha2" version = "0.10.6" @@ -2688,6 +3414,15 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "43b2853a4d09f215c24cc5489c992ce46052d359b5109343cbafbf26bc62f8a3" +[[package]] +name = "signal-hook-registry" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" +dependencies = [ + "libc", +] + [[package]] name = "simdutf8" version = "0.1.4" @@ -2809,6 +3544,16 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "subprocess" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c2e86926081dda636c546d8c5e641661049d7562a68f5488be4a1f7f66f6086" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "subtle" version = "2.4.1" @@ -2837,6 +3582,12 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + [[package]] name = "target-lexicon" version = "0.12.6" @@ -2911,6 +3662,35 @@ dependencies = [ "once_cell", ] +[[package]] +name = "tiberius" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf8ecd8428f31011260ede6027930f452f45d4e3c4e9e7bf9b0231398772e9ce" +dependencies = [ + "async-native-tls", + "async-trait", + "asynchronous-codec", + "bigdecimal", + "byteorder", + "bytes", + "connection-string", + "encoding", + "enumflags2", + "futures-util", + "num-traits", + "once_cell", + "pin-project-lite", + "pretty-hex", + "thiserror", + "time 0.3.20", + "tokio", + "tokio-util", + "tracing", + "uuid", + "winauth", +] + [[package]] name = "time" version = "0.1.45" @@ -2922,6 +3702,32 @@ dependencies = [ "winapi", ] +[[package]] +name = "time" +version = "0.3.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd0cbfecb4d19b5ea75bb31ad904eb5b9fa13f21079c3b92017ebdf4999a5890" +dependencies = [ + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd" + +[[package]] +name = "time-macros" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd80a657e71da814b8e5d60d3374fc6d35045062245d80224748ae522dd76f36" +dependencies = [ + "time-core", +] + [[package]] name = "tiny-keccak" version = "2.0.2" @@ -2967,11 +3773,26 @@ dependencies = [ "libc", "memchr", "mio", + "num_cpus", + "parking_lot", "pin-project-lite", + "signal-hook-registry", "socket2", + "tokio-macros", "windows-sys 0.45.0", ] +[[package]] +name = "tokio-macros" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d266c00fde287f55d3f1c3e96c500c362a2b8c695076ec180f27918820bc6df8" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "tokio-postgres" version = "0.7.7" @@ -3004,6 +3825,7 @@ checksum = "5427d89453009325de0d8f342c9490009f76e999cb7672d77e46267448f7e6b2" dependencies = [ "bytes", "futures-core", + "futures-io", "futures-sink", "pin-project-lite", "tokio", @@ -3032,10 +3854,23 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" dependencies = [ "cfg-if", + "log", "pin-project-lite", + "tracing-attributes", "tracing-core", ] +[[package]] +name = "tracing-attributes" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "tracing-core" version = "0.1.30" @@ -3067,6 +3902,17 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "twox-hash" +version = "1.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" +dependencies = [ + "cfg-if", + "rand 0.8.5", + "static_assertions", +] + [[package]] name = "typenum" version = "1.16.0" @@ -3142,12 +3988,29 @@ version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ad2024452afd3874bf539695e04af6732ba06517424dbf958fdb16a01f3bef6c" +[[package]] +name = "url" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d68c799ae75762b8c3fe375feb6600ef5602c883c5d21eb51c09f22b83c4643" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + [[package]] name = "utf8parse" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" +[[package]] +name = "uuid" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1674845326ee10d37ca60470760d4288a6f80f304007d92e5c53bab78c9cfd79" + [[package]] name = "valuable" version = "0.1.0" @@ -3182,6 +4045,12 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "wasi" +version = "0.9.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" + [[package]] name = "wasi" version = "0.10.0+wasi-snapshot-preview1" @@ -3325,6 +4194,19 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "winauth" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f820cd208ce9c6b050812dc2d724ba98c6c1e9db5ce9b3f58d925ae5723a5e6" +dependencies = [ + "bitflags 1.3.2", + "byteorder", + "md5", + "rand 0.7.3", + "winapi", +] + [[package]] name = "windows-sys" version = "0.42.0" @@ -3406,6 +4288,15 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" +[[package]] +name = "wyz" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" +dependencies = [ + "tap", +] + [[package]] name = "yaml-rust" version = "0.4.5" diff --git a/Taskfile.yml b/Taskfile.yml index 4347eb5028d9..9a5e052e3b63 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -195,7 +195,9 @@ tasks: # excluded under wasm. Note that this will also over-delete on Windows. # Note that we need to pass the target explicitly to manage # https://github.com/rust-lang/cargo/issues/8899 - - cargo insta test --accept --unreferenced=auto + - cargo insta test --accept + # see #2286 + #- cargo insta test --accept --unreferenced=auto - cargo insta test --accept --target=wasm32-unknown-unknown # We build the book too, because that acts as a test - cd web/book && mdbook build diff --git a/prql-compiler/Cargo.toml b/prql-compiler/Cargo.toml index 9f99e9aeccd8..32d540646d8b 100644 --- a/prql-compiler/Cargo.toml +++ b/prql-compiler/Cargo.toml @@ -48,8 +48,14 @@ insta = {version = "1.29", features = ["colors", "glob", "yaml"]} chrono = {version = "0.4", features = [], default-features = false} criterion = "0.4.0" postgres = "0.19.3" +mysql = "23" +tiberius = {version = "0.12", features = ["sql-browser-tokio", "bigdecimal", "time"]} +tokio = {version = "1", features = ["full"]} +tokio-util = { version = "0.7", features = ["compat"] } pretty_assertions = "1.3.0" rusqlite = {version = "0.29.0", features = ["bundled", "csvtab"]} +pg_bigdecimal = "0.1" +csv = "1.2" # Re-enable on windows when duckdb supports it # https://github.com/wangfenjin/duckdb-rs/issues/62 diff --git a/prql-compiler/tests/integration-rdbms/conf/my.cnf b/prql-compiler/tests/integration-rdbms/conf/my.cnf new file mode 100644 index 000000000000..010d65225a43 --- /dev/null +++ b/prql-compiler/tests/integration-rdbms/conf/my.cnf @@ -0,0 +1,2 @@ +[mysqld] +secure-file-priv= /tmp/chinook/ diff --git a/prql-compiler/tests/integration-rdbms/connection.rs b/prql-compiler/tests/integration-rdbms/connection.rs new file mode 100644 index 000000000000..c9c50ab3b940 --- /dev/null +++ b/prql-compiler/tests/integration-rdbms/connection.rs @@ -0,0 +1,316 @@ +use std::env::current_dir; +use std::time::SystemTime; + +use chrono::{DateTime, Utc}; +use itertools::Itertools; +use mysql::prelude::Queryable; +use mysql::Value; +use pg_bigdecimal::PgNumeric; +use postgres::types::Type; +use tiberius::numeric::BigDecimal; +use tiberius::time::time::PrimitiveDateTime; +use tiberius::*; +use tokio::net::TcpStream; +use tokio::runtime::Runtime; +use tokio_util::compat::Compat; + +use prql_compiler::sql::Dialect; + +pub type Row = Vec; + +pub struct DuckDBConnection(pub duckdb::Connection); + +pub struct SQLiteConnection(pub rusqlite::Connection); + +pub struct PostgresConnection(pub postgres::Client); + +pub struct MysqlConnection(pub mysql::Pool); + +pub struct MssqlConnection(pub tiberius::Client>); + +pub trait DBConnection { + fn run_query(&mut self, sql: &str, runtime: &Runtime) -> Vec; + + fn import_csv(&mut self, csv_name: &str, runtime: &Runtime); + + fn get_dialect(&self) -> Dialect; +} + +impl DBConnection for DuckDBConnection { + fn run_query(&mut self, sql: &str, _runtime: &Runtime) -> Vec { + let mut statement = self.0.prepare(sql).unwrap(); + let mut rows = statement.query([]).unwrap(); + let mut vec = vec![]; + while let Ok(Some(row)) = rows.next() { + let mut columns = vec![]; + // row.len() always gives 1. hence this workaround + for i in 0.. { + let v_ref = match row.get_ref(i) { + Ok(v) => v, + Err(_) => { + break; + } + }; + let value = match v_ref { + duckdb::types::ValueRef::Null => "".to_string(), + duckdb::types::ValueRef::Boolean(v) => v.to_string(), + duckdb::types::ValueRef::TinyInt(v) => v.to_string(), + duckdb::types::ValueRef::SmallInt(v) => v.to_string(), + duckdb::types::ValueRef::Int(v) => v.to_string(), + duckdb::types::ValueRef::BigInt(v) => v.to_string(), + duckdb::types::ValueRef::HugeInt(v) => v.to_string(), + duckdb::types::ValueRef::UTinyInt(v) => v.to_string(), + duckdb::types::ValueRef::USmallInt(v) => v.to_string(), + duckdb::types::ValueRef::UInt(v) => v.to_string(), + duckdb::types::ValueRef::UBigInt(v) => v.to_string(), + duckdb::types::ValueRef::Float(v) => v.to_string(), + duckdb::types::ValueRef::Double(v) => v.to_string(), + duckdb::types::ValueRef::Decimal(v) => v.to_string(), + duckdb::types::ValueRef::Timestamp(u, v) => format!("{} {:?}", v, u), + duckdb::types::ValueRef::Text(v) => String::from_utf8(v.to_vec()).unwrap(), + duckdb::types::ValueRef::Blob(_) => "BLOB".to_string(), + duckdb::types::ValueRef::Date32(v) => v.to_string(), + duckdb::types::ValueRef::Time64(u, v) => format!("{} {:?}", v, u), + }; + columns.push(value); + } + vec.push(columns) + } + vec + } + + fn import_csv(&mut self, csv_name: &str, runtime: &Runtime) { + let mut path = current_dir().unwrap(); + for p in [ + "tests", + "integration", + "data", + "chinook", + format!("{csv_name}.csv").as_str(), + ] { + path.push(p); + } + let path = path.display().to_string().replace('"', ""); + self.run_query( + &format!("COPY {csv_name} FROM '{path}' (AUTO_DETECT TRUE);"), + runtime, + ); + } + + fn get_dialect(&self) -> Dialect { + Dialect::DuckDb + } +} + +impl DBConnection for SQLiteConnection { + fn run_query(&mut self, sql: &str, _runtime: &Runtime) -> Vec { + let mut statement = self.0.prepare(sql).unwrap(); + let mut rows = statement.query([]).unwrap(); + let mut vec = vec![]; + while let Ok(Some(row)) = rows.next() { + let mut columns = vec![]; + // row.len() always gives 1. hence this workaround + for i in 0.. { + let v_ref = match row.get_ref(i) { + Ok(v) => v, + Err(_) => { + break; + } + }; + let value = match v_ref { + rusqlite::types::ValueRef::Null => "".to_string(), + rusqlite::types::ValueRef::Integer(v) => v.to_string(), + rusqlite::types::ValueRef::Real(v) => v.to_string(), + rusqlite::types::ValueRef::Text(v) => String::from_utf8(v.to_vec()).unwrap(), + rusqlite::types::ValueRef::Blob(_) => "BLOB".to_string(), + }; + columns.push(value); + } + vec.push(columns); + } + vec + } + + fn import_csv(&mut self, csv_name: &str, runtime: &Runtime) { + let mut path = current_dir().unwrap(); + for p in [ + "tests", + "integration", + "data", + "chinook", + format!("{csv_name}.csv").as_str(), + ] { + path.push(p); + } + let mut reader = csv::ReaderBuilder::new() + .has_headers(true) + .from_path(path) + .unwrap(); + let headers = reader + .headers() + .unwrap() + .iter() + .map(|s| s.to_string()) + .collect::>(); + for result in reader.records() { + let r = result.unwrap(); + let q = format!( + "INSERT INTO {csv_name} ({}) VALUES ({})", + headers.iter().join(","), + r.iter() + .map(|s| if s.is_empty() { + "null".to_string() + } else { + format!("\"{}\"", s.replace('"', "\"\"")) + }) + .join(",") + ); + self.run_query(q.as_str(), runtime); + } + } + + fn get_dialect(&self) -> Dialect { + Dialect::SQLite + } +} + +impl DBConnection for PostgresConnection { + fn run_query(&mut self, sql: &str, _runtime: &Runtime) -> Vec { + let rows = self.0.query(sql, &[]).unwrap(); + let mut vec = vec![]; + for row in rows.into_iter() { + let mut columns = vec![]; + for i in 0..row.len() { + let col = &(*row.columns())[i]; + let value = match col.type_() { + &Type::BOOL => (row.get::(i)).to_string(), + &Type::INT4 => match row.try_get::(i) { + Ok(v) => v.to_string(), + Err(_) => "".to_string(), + }, + &Type::INT8 => (row.get::(i)).to_string(), + &Type::TEXT | &Type::VARCHAR | &Type::JSON | &Type::JSONB => { + match row.try_get::(i) { + Ok(v) => v, + // handle null + Err(_) => "".to_string(), + } + } + &Type::FLOAT4 => (row.get::(i)).to_string(), + &Type::FLOAT8 => (row.get::(i)).to_string(), + &Type::NUMERIC => row.get::(i).n.unwrap().to_string(), + &Type::TIMESTAMPTZ | &Type::TIMESTAMP => { + let time = row.get::(i); + let date_time: DateTime = time.into(); + date_time.to_rfc3339() + } + typ => unimplemented!("postgres type {:?}", typ), + }; + columns.push(value); + } + vec.push(columns); + } + vec + } + + fn import_csv(&mut self, csv_name: &str, runtime: &Runtime) { + self.run_query( + &format!( + "COPY {csv_name} FROM '/tmp/chinook/{csv_name}.csv' DELIMITER ',' CSV HEADER;" + ), + runtime, + ); + } + + fn get_dialect(&self) -> Dialect { + Dialect::PostgreSql + } +} + +impl DBConnection for MysqlConnection { + fn run_query(&mut self, sql: &str, _runtime: &Runtime) -> Vec { + let mut conn = self.0.get_conn().unwrap(); + let rows: Vec = conn.query(sql).unwrap(); + let mut vec = vec![]; + for row in rows.into_iter() { + let mut columns = vec![]; + for v in row.unwrap() { + let value = match v { + Value::NULL => "".to_string(), + Value::Bytes(v) => String::from_utf8(v).unwrap_or_else(|_| "BLOB".to_string()), + Value::Int(v) => v.to_string(), + Value::UInt(v) => v.to_string(), + Value::Float(v) => v.to_string(), + Value::Double(v) => v.to_string(), + typ => unimplemented!("mysql type {:?}", typ), + }; + columns.push(value); + } + vec.push(columns); + } + vec + } + + fn import_csv(&mut self, csv_name: &str, runtime: &Runtime) { + self.run_query(&format!("LOAD DATA INFILE '/tmp/chinook/{csv_name}.csv' INTO TABLE {csv_name} FIELDS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '\"' LINES TERMINATED BY '\n' IGNORE 1 ROWS;"), runtime); + } + + fn get_dialect(&self) -> Dialect { + Dialect::MySql + } +} + +impl DBConnection for MssqlConnection { + fn run_query(&mut self, sql: &str, runtime: &Runtime) -> Vec { + runtime.block_on(self.query(sql)) + } + + fn import_csv(&mut self, csv_name: &str, runtime: &Runtime) { + self.run_query(&format!("BULK INSERT {csv_name} FROM '/tmp/chinook/{csv_name}.csv' WITH (FIRSTROW = 2, FIELDTERMINATOR = ',', ROWTERMINATOR = '\n', TABLOCK, FORMAT = 'CSV', CODEPAGE = 'RAW');"), runtime); + } + + fn get_dialect(&self) -> Dialect { + Dialect::MsSql + } +} + +impl MssqlConnection { + async fn query(&mut self, sql: &str) -> Vec { + let mut stream = self.0.query(sql, &[]).await.unwrap(); + let mut vec = vec![]; + let cols_option = stream.columns().await.unwrap(); + if cols_option.is_none() { + return vec![]; + } + let cols = cols_option.unwrap().to_vec(); + for row in stream.into_first_result().await.unwrap() { + let mut columns = vec![]; + for (i, col) in cols.iter().enumerate() { + let value = match col.column_type() { + ColumnType::Null => "".to_string(), + ColumnType::Bit => String::from(row.get::<&str, usize>(i).unwrap()), + ColumnType::Intn | ColumnType::Int4 => row + .get::(i) + .map(|i| i.to_string()) + .unwrap_or_else(|| "".to_string()), + ColumnType::Floatn => row + .get::(i) + .map(|i| i.to_string()) + .unwrap_or_else(|| "".to_string()), + ColumnType::Numericn => row.get::(i).unwrap().to_string(), + ColumnType::BigVarChar | ColumnType::NVarchar => { + String::from(row.get::<&str, usize>(i).unwrap_or("")) + } + ColumnType::Datetimen => { + row.get::(i).unwrap().to_string() + } + typ => unimplemented!("mssql type {:?}", typ), + }; + columns.push(value); + } + vec.push(columns); + } + + vec + } +} diff --git a/prql-compiler/tests/integration-rdbms/docker-compose.yml b/prql-compiler/tests/integration-rdbms/docker-compose.yml new file mode 100644 index 000000000000..600a26f410a7 --- /dev/null +++ b/prql-compiler/tests/integration-rdbms/docker-compose.yml @@ -0,0 +1,47 @@ +services: + postgres: + image: "postgres:alpine" + ports: + - "5432:5432" + environment: + POSTGRES_DB: dummy + POSTGRES_USER: root + POSTGRES_PASSWORD: root + volumes: &vol + - ../integration/data/chinook:/tmp/chinook + mysql: + # No arm64 image available; remove when one does become available + platform: linux/amd64 + image: "mysql:debian" + ports: + - "3306:3306" + environment: + MYSQL_DATABASE: dummy + MYSQL_ROOT_PASSWORD: root + volumes: + - ./conf:/etc/mysql/conf.d + - ../integration/data/chinook:/tmp/chinook + # db2: + # image: 'icr.io/db2_community/db2' + # ports: + # - '50000:50000' + # environment: + # LICENSE: accept + # DBNAME: dummy + # DB2INSTANCE: db2 + # DB2INST1_PASSWORD: root + # BLU: false + # TO_CREATE_SAMPLEDB: false + # REPODB: false + # IS_OSXFS: false + mssql: + image: "mcr.microsoft.com/mssql/server:latest" + ports: + - "1433:1433" + environment: + ACCEPT_EULA: Y + MSSQL_PID: Developer + MSSQL_SA_PASSWORD: Wordpass123## + LC_ALL: en_US.UTF-8 + MSSQL_COLLATION: Latin1_General_100_CS_AI_SC_UTF8 + volumes: *vol diff --git a/prql-compiler/tests/integration-rdbms/main.rs b/prql-compiler/tests/integration-rdbms/main.rs new file mode 100644 index 000000000000..2f3cafd63dc9 --- /dev/null +++ b/prql-compiler/tests/integration-rdbms/main.rs @@ -0,0 +1,182 @@ +#![cfg(not(any(target_family = "windows", target_family = "wasm")))] +// TODO enable it for all OS +#![cfg(target_os = "linux")] + +mod connection; + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + use std::{env, fs}; + + use insta::{assert_snapshot, glob}; + use postgres::NoTls; + use tiberius::{AuthMethod, Client, Config}; + use tokio::net::TcpStream; + use tokio::runtime::Runtime; + use tokio_util::compat::{Compat, TokioAsyncWriteCompatExt}; + + use prql_compiler::sql::Dialect; + use prql_compiler::Options; + use prql_compiler::Target::Sql; + + use crate::connection::*; + + #[test] + fn test_rdbms() { + if env::var("SKIP_INTEGRATION").is_ok() { + return; + } + for port in [5432u16, 3306, 1433 /*, 50000*/] { + // test is skipped locally when DB is not listening + // in CI it fails + if !is_port_open(port) { + match env::var("CI") { + Ok(v) if &v == "true" => { + // CI + panic!("No database is listening on port {}", port); + } + _ => { + // locally + return; + } + } + } + } + let runtime = Runtime::new().unwrap(); + let mut duck = DuckDBConnection(duckdb::Connection::open_in_memory().unwrap()); + let mut sqlite = SQLiteConnection(rusqlite::Connection::open_in_memory().unwrap()); + let mut pg = PostgresConnection( + postgres::Client::connect("host=localhost user=root password=root dbname=dummy", NoTls) + .unwrap(), + ); + let mut my = + MysqlConnection(mysql::Pool::new("mysql://root:root@localhost:3306/dummy").unwrap()); + let mut ms = { + let mut config = Config::new(); + config.host("127.0.0.1"); + config.port(1433); + config.trust_cert(); + config.authentication(AuthMethod::sql_server("sa", "Wordpass123##")); + + let client = runtime.block_on(get_client(config.clone())); + + async fn get_client(config: Config) -> Client> { + let tcp = TcpStream::connect(config.get_addr()).await.unwrap(); + tcp.set_nodelay(true).unwrap(); + Client::connect(config, tcp.compat_write()).await.unwrap() + } + MssqlConnection(client) + }; + + let mut connections: Vec<&mut dyn DBConnection> = + vec![&mut duck, &mut sqlite, &mut pg, &mut my, &mut ms]; + + for con in &mut connections { + setup_connection(*con, &runtime); + } + + // for each of the queries + glob!("..", "integration/queries/**/*.prql", |path| { + let test_name = path + .file_name() + .and_then(|s| s.to_str()) + .unwrap_or_default(); + + // read + let prql = fs::read_to_string(path).unwrap(); + + if prql.contains("skip_test") { + return; + } + + let mut results = BTreeMap::new(); + for con in &mut connections { + let vendor = con.get_dialect().to_string().to_lowercase(); + if prql.contains(format!("skip_{}", vendor).as_str()) { + continue; + } + results.insert(vendor, run_query(*con, prql.as_str(), &runtime)); + } + + let first_result = match results.iter().next() { + Some(v) => v, + None => return, + }; + for (k, v) in results.iter().skip(1) { + pretty_assertions::assert_eq!( + *first_result.1, + *v, + "{} == {}: {test_name}", + first_result.0, + k + ); + } + + assert_snapshot!(format!("{:?}", first_result.1)); + }); + } + + fn setup_connection(con: &mut dyn DBConnection, runtime: &Runtime) { + let setup = include_str!("../integration/data/chinook/schema.sql"); + setup + .split(';') + .map(|s| s.trim()) + .filter(|s| !s.is_empty()) + .for_each(|s| { + let sql = match con.get_dialect() { + Dialect::MsSql => s.replace("TIMESTAMP", "DATETIME"), + Dialect::MySql => s.replace('"', "`").replace("TIMESTAMP", "DATETIME"), + _ => s.to_string(), + }; + con.run_query(sql.as_str(), runtime); + }); + let tables = [ + "invoices", + "customers", + "employees", + "tracks", + "albums", + "genres", + "playlist_track", + "playlists", + "media_types", + "artists", + "invoice_items", + ]; + for table in tables { + con.import_csv(table, runtime); + } + } + + fn run_query(con: &mut dyn DBConnection, prql: &str, runtime: &Runtime) -> Vec { + let options = Options::default().with_target(Sql(Some(con.get_dialect()))); + let sql = prql_compiler::compile(prql, &options).unwrap(); + let mut actual_rows = con.run_query(sql.as_str(), runtime); + replace_booleans(&mut actual_rows); + actual_rows + } + + // some sql dialects use 1 and 0 instead of true and false + fn replace_booleans(rows: &mut Vec) { + for row in rows { + for col in row { + if col == "true" { + *col = "1".to_string(); + } else if col == "false" { + *col = "0".to_string(); + } + } + } + } + + fn is_port_open(port: u16) -> bool { + match std::net::TcpStream::connect(("127.0.0.1", port)) { + Ok(stream) => { + stream.shutdown(std::net::Shutdown::Both).unwrap_or(()); + true + } + Err(_) => false, + } + } +} diff --git a/prql-compiler/tests/integration-rdbms/snapshots/integration_rdbms__tests__rdbms@distinct.prql.snap b/prql-compiler/tests/integration-rdbms/snapshots/integration_rdbms__tests__rdbms@distinct.prql.snap new file mode 100644 index 000000000000..0ae946546223 --- /dev/null +++ b/prql-compiler/tests/integration-rdbms/snapshots/integration_rdbms__tests__rdbms@distinct.prql.snap @@ -0,0 +1,6 @@ +--- +source: prql-compiler/tests/integration-rdbms/main.rs +expression: "format!(\"{:?}\", first_result.1)" +input_file: prql-compiler/tests/integration/queries/distinct.prql +--- +[["1", "1"], ["2", "1"], ["3", "1"], ["4", "1"], ["5", "1"], ["6", "1"], ["7", "1"], ["8", "2"], ["9", "3"], ["10", "1"], ["11", "4"], ["12", "5"], ["13", "2"], ["14", "3"], ["15", "3"], ["16", "3"], ["17", "3"], ["18", "4"], ["19", "3"], ["20", "6"], ["21", "7"], ["22", "7"], ["23", "7"], ["24", "7"], ["25", "7"], ["26", "8"], ["27", "8"], ["28", "7"], ["29", "9"], ["30", "1"], ["31", "1"], ["32", "10"], ["33", "7"], ["34", "7"], ["35", "3"], ["36", "1"], ["37", "1"], ["38", "2"], ["39", "4"], ["40", "1"], ["41", "7"], ["42", "4"], ["43", "1"], ["44", "1"], ["45", "7"], ["46", "1"], ["47", "7"], ["48", "2"], ["49", "2"], ["50", "1"], ["51", "2"], ["52", "11"], ["53", "7"], ["54", "1"], ["55", "1"], ["56", "7"], ["57", "7"], ["58", "1"], ["59", "1"], ["60", "1"], ["61", "1"], ["62", "1"], ["63", "1"], ["64", "1"], ["65", "1"], ["66", "1"], ["67", "1"], ["68", "2"], ["69", "7"], ["70", "7"], ["71", "7"], ["72", "6"], ["73", "6"], ["73", "7"], ["74", "4"], ["75", "4"], ["76", "1"], ["77", "4"], ["78", "7"], ["79", "1"], ["80", "1"], ["81", "4"], ["82", "1"], ["83", "12"], ["84", "7"], ["85", "10"], ["86", "7"], ["87", "2"], ["88", "3"], ["89", "4"], ["90", "1"], ["91", "1"], ["92", "3"], ["93", "2"], ["94", "1"], ["95", "3"], ["96", "3"], ["97", "1"], ["98", "13"], ["99", "1"], ["100", "6"], ["101", "13"], ["102", "3"], ["102", "13"], ["103", "1"], ["104", "1"], ["105", "3"], ["106", "3"], ["107", "3"], ["108", "3"], ["109", "1"], ["109", "3"], ["110", "3"], ["111", "3"], ["112", "1"], ["112", "3"], ["113", "1"], ["114", "1"], ["115", "14"], ["116", "1"], ["117", "14"], ["118", "15"], ["119", "4"], ["120", "1"], ["121", "1"], ["122", "7"], ["123", "7"], ["124", "16"], ["125", "3"], ["126", "1"], ["127", "1"], ["128", "1"], ["129", "1"], ["130", "1"], ["131", "1"], ["132", "1"], ["133", "1"], ["134", "1"], ["135", "1"], ["136", "1"], ["137", "1"], ["138", "1"], ["139", "7"], ["140", "7"], ["141", "1"], ["141", "3"], ["141", "8"], ["142", "7"], ["143", "7"], ["144", "1"], ["145", "7"], ["146", "14"], ["147", "1"], ["148", "3"], ["149", "3"], ["150", "3"], ["151", "3"], ["152", "3"], ["153", "3"], ["154", "3"], ["155", "3"], ["156", "3"], ["157", "2"], ["158", "7"], ["159", "7"], ["160", "3"], ["161", "16"], ["162", "3"], ["163", "1"], ["164", "1"], ["165", "1"], ["166", "7"], ["167", "7"], ["168", "7"], ["169", "7"], ["170", "1"], ["171", "1"], ["172", "1"], ["173", "1"], ["174", "3"], ["175", "1"], ["176", "10"], ["177", "1"], ["178", "1"], ["179", "4"], ["180", "1"], ["181", "1"], ["182", "1"], ["183", "1"], ["184", "17"], ["185", "1"], ["186", "1"], ["187", "4"], ["188", "4"], ["189", "1"], ["190", "4"], ["191", "4"], ["192", "1"], ["193", "4"], ["194", "1"], ["195", "1"], ["196", "1"], ["197", "1"], ["198", "1"], ["199", "1"], ["200", "1"], ["201", "4"], ["202", "4"], ["203", "1"], ["204", "2"], ["205", "6"], ["206", "1"], ["207", "3"], ["208", "1"], ["209", "6"], ["210", "6"], ["211", "4"], ["212", "1"], ["213", "1"], ["214", "1"], ["215", "1"], ["216", "1"], ["217", "1"], ["218", "1"], ["219", "4"], ["220", "4"], ["221", "1"], ["222", "7"], ["223", "7"], ["224", "4"], ["225", "4"], ["226", "18"], ["227", "18"], ["227", "19"], ["227", "20"], ["228", "19"], ["228", "21"], ["229", "19"], ["229", "21"], ["230", "19"], ["231", "19"], ["231", "21"], ["232", "1"], ["233", "1"], ["234", "1"], ["235", "1"], ["236", "1"], ["237", "1"], ["238", "1"], ["239", "1"], ["240", "1"], ["241", "8"], ["242", "1"], ["243", "1"], ["244", "1"], ["245", "1"], ["246", "1"], ["247", "7"], ["248", "7"], ["249", "19"], ["250", "19"], ["251", "19"], ["251", "22"], ["252", "1"], ["253", "20"], ["254", "19"], ["255", "9"], ["256", "1"], ["257", "1"], ["258", "17"], ["259", "15"], ["260", "23"], ["261", "19"], ["261", "21"], ["262", "2"], ["263", "16"], ["264", "15"], ["265", "1"], ["266", "7"], ["267", "2"], ["268", "24"], ["269", "23"], ["270", "23"], ["271", "23"], ["272", "24"], ["273", "24"], ["274", "24"], ["275", "24"], ["276", "24"], ["277", "24"], ["278", "24"], ["279", "24"], ["280", "24"], ["281", "24"], ["282", "24"], ["283", "24"], ["284", "24"], ["285", "24"], ["286", "24"], ["287", "24"], ["288", "24"], ["289", "24"], ["290", "24"], ["291", "24"], ["292", "24"], ["293", "24"], ["294", "24"], ["295", "24"], ["296", "24"], ["297", "24"], ["298", "24"], ["299", "24"], ["300", "24"], ["301", "24"], ["302", "24"], ["303", "24"], ["304", "24"], ["305", "24"], ["306", "24"], ["307", "24"], ["308", "24"], ["309", "24"], ["310", "24"], ["311", "24"], ["312", "24"], ["313", "24"], ["314", "24"], ["315", "24"], ["316", "24"], ["317", "25"], ["318", "24"], ["319", "24"], ["320", "24"], ["321", "14"], ["322", "9"], ["323", "23"], ["324", "24"], ["325", "24"], ["326", "24"], ["327", "24"], ["328", "24"], ["329", "24"], ["330", "24"], ["331", "24"], ["332", "24"], ["333", "24"], ["334", "24"], ["335", "24"], ["336", "24"], ["337", "24"], ["338", "24"], ["339", "24"], ["340", "24"], ["341", "24"], ["342", "24"], ["343", "24"], ["344", "24"], ["345", "24"], ["346", "24"], ["347", "10"]] diff --git a/prql-compiler/tests/integration-rdbms/snapshots/integration_rdbms__tests__rdbms@genre_counts.prql.snap b/prql-compiler/tests/integration-rdbms/snapshots/integration_rdbms__tests__rdbms@genre_counts.prql.snap new file mode 100644 index 000000000000..2751f9165721 --- /dev/null +++ b/prql-compiler/tests/integration-rdbms/snapshots/integration_rdbms__tests__rdbms@genre_counts.prql.snap @@ -0,0 +1,6 @@ +--- +source: prql-compiler/tests/integration-rdbms/main.rs +expression: "format!(\"{:?}\", first_result.1)" +input_file: prql-compiler/tests/integration/queries/genre_counts.prql +--- +[["-25"]] diff --git a/prql-compiler/tests/integration-rdbms/snapshots/integration_rdbms__tests__rdbms@group_all.prql.snap b/prql-compiler/tests/integration-rdbms/snapshots/integration_rdbms__tests__rdbms@group_all.prql.snap new file mode 100644 index 000000000000..adcfc49c15ac --- /dev/null +++ b/prql-compiler/tests/integration-rdbms/snapshots/integration_rdbms__tests__rdbms@group_all.prql.snap @@ -0,0 +1,6 @@ +--- +source: prql-compiler/tests/integration-rdbms/main.rs +expression: "format!(\"{:?}\", first_result.1)" +input_file: prql-compiler/tests/integration/queries/group_all.prql +--- +[["1", "For Those About To Rock We Salute You", "9.9"], ["2", "Balls to the Wall", "0.99"], ["3", "Restless and Wild", "2.97"], ["4", "Let There Be Rock", "7.92"], ["5", "Big Ones", "14.85"], ["6", "Jagged Little Pill", "12.87"], ["7", "Facelift", "11.88"], ["8", "Warner 25 Anos", "13.86"], ["9", "Plays Metallica By Four Cellos", "7.92"], ["10", "Audioslave", "13.86"]] diff --git a/prql-compiler/tests/integration-rdbms/snapshots/integration_rdbms__tests__rdbms@invoice_totals.prql.snap b/prql-compiler/tests/integration-rdbms/snapshots/integration_rdbms__tests__rdbms@invoice_totals.prql.snap new file mode 100644 index 000000000000..f9bfc736e21c --- /dev/null +++ b/prql-compiler/tests/integration-rdbms/snapshots/integration_rdbms__tests__rdbms@invoice_totals.prql.snap @@ -0,0 +1,6 @@ +--- +source: prql-compiler/tests/integration-rdbms/main.rs +expression: "format!(\"{:?}\", first_result.1)" +input_file: prql-compiler/tests/integration/queries/invoice_totals.prql +--- +[["2009-01", "2009-01-01", "1", "2", "1.98", "2", ""], ["2009-01", "2009-01-02", "1", "4", "3.96", "6", ""], ["2009-01", "2009-01-03", "1", "6", "5.94", "12", ""], ["2009-01", "2009-01-06", "1", "9", "8.91", "21", ""], ["2009-01", "2009-01-11", "1", "14", "13.86", "35", ""], ["2009-01", "2009-01-19", "1", "1", "0.99", "36", ""], ["2009-02", "2009-02-01", "2", "4", "3.96", "4", ""], ["2009-02", "2009-02-02", "1", "4", "3.96", "8", "2"], ["2009-02", "2009-02-03", "1", "6", "5.94", "14", "4"], ["2009-02", "2009-02-06", "1", "9", "8.91", "23", "6"], ["2009-02", "2009-02-11", "1", "14", "13.86", "37", "9"], ["2009-02", "2009-02-19", "1", "1", "0.99", "38", "14"], ["2009-03", "2009-03-04", "2", "4", "3.96", "4", "1"], ["2009-03", "2009-03-05", "1", "4", "3.96", "8", "4"], ["2009-03", "2009-03-06", "1", "6", "5.94", "14", "4"], ["2009-03", "2009-03-09", "1", "9", "8.91", "23", "6"], ["2009-03", "2009-03-14", "1", "14", "13.86", "37", "9"], ["2009-03", "2009-03-22", "1", "1", "0.99", "38", "14"], ["2009-04", "2009-04-04", "2", "4", "3.96", "4", "1"], ["2009-04", "2009-04-05", "1", "4", "3.96", "8", "4"], ["2009-04", "2009-04-06", "1", "6", "5.94", "14", "4"], ["2009-04", "2009-04-09", "1", "9", "8.91", "23", "6"], ["2009-04", "2009-04-14", "1", "14", "13.86", "37", "9"], ["2009-04", "2009-04-22", "1", "1", "0.99", "38", "14"], ["2009-05", "2009-05-05", "2", "4", "3.96", "4", "1"], ["2009-05", "2009-05-06", "1", "4", "3.96", "8", "4"], ["2009-05", "2009-05-07", "1", "6", "5.94", "14", "4"], ["2009-05", "2009-05-10", "1", "9", "8.91", "23", "6"], ["2009-05", "2009-05-15", "1", "14", "13.86", "37", "9"], ["2009-05", "2009-05-23", "1", "1", "0.99", "38", "14"], ["2009-06", "2009-06-05", "2", "4", "3.96", "4", "1"], ["2009-06", "2009-06-06", "1", "4", "3.96", "8", "4"], ["2009-06", "2009-06-07", "1", "6", "5.94", "14", "4"], ["2009-06", "2009-06-10", "1", "9", "8.91", "23", "6"], ["2009-06", "2009-06-15", "1", "14", "13.86", "37", "9"], ["2009-06", "2009-06-23", "1", "1", "0.99", "38", "14"], ["2009-07", "2009-07-06", "2", "4", "3.96", "4", "1"], ["2009-07", "2009-07-07", "1", "4", "3.96", "8", "4"], ["2009-07", "2009-07-08", "1", "6", "5.94", "14", "4"], ["2009-07", "2009-07-11", "1", "9", "8.91", "23", "6"], ["2009-07", "2009-07-16", "1", "14", "13.86", "37", "9"], ["2009-07", "2009-07-24", "1", "1", "0.99", "38", "14"], ["2009-08", "2009-08-06", "2", "4", "3.96", "4", "1"], ["2009-08", "2009-08-07", "1", "4", "3.96", "8", "4"], ["2009-08", "2009-08-08", "1", "6", "5.94", "14", "4"], ["2009-08", "2009-08-11", "1", "9", "8.91", "23", "6"], ["2009-08", "2009-08-16", "1", "14", "13.86", "37", "9"], ["2009-08", "2009-08-24", "1", "1", "0.99", "38", "14"], ["2009-09", "2009-09-06", "2", "4", "3.96", "4", "1"], ["2009-09", "2009-09-07", "1", "4", "3.96", "8", "4"], ["2009-09", "2009-09-08", "1", "6", "5.94", "14", "4"], ["2009-09", "2009-09-11", "1", "9", "8.91", "23", "6"], ["2009-09", "2009-09-16", "1", "14", "13.86", "37", "9"], ["2009-09", "2009-09-24", "1", "1", "0.99", "38", "14"], ["2009-10", "2009-10-07", "2", "4", "3.96", "4", "1"], ["2009-10", "2009-10-08", "1", "4", "3.96", "8", "4"], ["2009-10", "2009-10-09", "1", "6", "5.94", "14", "4"], ["2009-10", "2009-10-12", "1", "9", "8.91", "23", "6"], ["2009-10", "2009-10-17", "1", "14", "13.86", "37", "9"], ["2009-10", "2009-10-25", "1", "1", "0.99", "38", "14"], ["2009-11", "2009-11-07", "2", "4", "3.96", "4", "1"], ["2009-11", "2009-11-08", "1", "4", "3.96", "8", "4"], ["2009-11", "2009-11-09", "1", "6", "5.94", "14", "4"], ["2009-11", "2009-11-12", "1", "9", "8.91", "23", "6"], ["2009-11", "2009-11-17", "1", "14", "13.86", "37", "9"], ["2009-11", "2009-11-25", "1", "1", "0.99", "38", "14"], ["2009-12", "2009-12-08", "2", "4", "3.96", "4", "1"], ["2009-12", "2009-12-09", "1", "4", "3.96", "8", "4"], ["2009-12", "2009-12-10", "1", "6", "5.94", "14", "4"], ["2009-12", "2009-12-13", "1", "9", "8.91", "23", "6"], ["2009-12", "2009-12-18", "1", "14", "13.86", "37", "9"], ["2009-12", "2009-12-26", "1", "1", "0.99", "38", "14"], ["2010-01", "2010-01-08", "2", "4", "3.96", "4", "1"], ["2010-01", "2010-01-09", "1", "4", "3.96", "8", "4"], ["2010-01", "2010-01-10", "1", "6", "6.94", "14", "4"], ["2010-01", "2010-01-13", "1", "9", "17.91", "23", "6"], ["2010-01", "2010-01-18", "1", "14", "18.86", "37", "9"], ["2010-01", "2010-01-26", "1", "1", "0.99", "38", "14"], ["2010-02", "2010-02-08", "2", "4", "3.96", "4", "1"], ["2010-02", "2010-02-09", "1", "4", "3.96", "8", "4"], ["2010-02", "2010-02-10", "1", "6", "5.94", "14", "4"], ["2010-02", "2010-02-13", "1", "9", "8.91", "23", "6"], ["2010-02", "2010-02-18", "1", "14", "21.86", "37", "9"], ["2010-02", "2010-02-26", "1", "1", "1.99", "38", "14"], ["2010-03", "2010-03-11", "2", "4", "7.96", "4", "1"], ["2010-03", "2010-03-12", "1", "4", "3.96", "8", "4"], ["2010-03", "2010-03-13", "1", "6", "5.94", "14", "4"], ["2010-03", "2010-03-16", "1", "9", "9.91", "23", "6"], ["2010-03", "2010-03-21", "1", "14", "15.86", "37", "9"], ["2010-03", "2010-03-29", "1", "1", "0.99", "38", "14"], ["2010-04", "2010-04-11", "2", "4", "3.96", "4", "1"], ["2010-04", "2010-04-12", "1", "4", "3.96", "8", "4"], ["2010-04", "2010-04-13", "1", "6", "5.94", "14", "4"], ["2010-04", "2010-04-16", "1", "9", "8.91", "23", "6"], ["2010-04", "2010-04-21", "1", "14", "13.86", "37", "9"], ["2010-04", "2010-04-29", "1", "1", "0.99", "38", "14"], ["2010-05", "2010-05-12", "2", "4", "3.96", "4", "1"], ["2010-05", "2010-05-13", "1", "4", "3.96", "8", "4"], ["2010-05", "2010-05-14", "1", "6", "5.94", "14", "4"], ["2010-05", "2010-05-17", "1", "9", "8.91", "23", "6"], ["2010-05", "2010-05-22", "1", "14", "13.86", "37", "9"], ["2010-05", "2010-05-30", "1", "1", "0.99", "38", "14"], ["2010-06", "2010-06-12", "2", "4", "3.96", "4", "1"], ["2010-06", "2010-06-13", "1", "4", "3.96", "8", "4"], ["2010-06", "2010-06-14", "1", "6", "5.94", "14", "4"], ["2010-06", "2010-06-17", "1", "9", "8.91", "23", "6"], ["2010-06", "2010-06-22", "1", "14", "13.86", "37", "9"], ["2010-06", "2010-06-30", "1", "1", "0.99", "38", "14"], ["2010-07", "2010-07-13", "2", "4", "3.96", "4", "1"], ["2010-07", "2010-07-14", "1", "4", "3.96", "8", "4"], ["2010-07", "2010-07-15", "1", "6", "5.94", "14", "4"], ["2010-07", "2010-07-18", "1", "9", "8.91", "23", "6"], ["2010-07", "2010-07-23", "1", "14", "13.86", "37", "9"], ["2010-07", "2010-07-31", "1", "1", "0.99", "38", "14"], ["2010-08", "2010-08-13", "2", "4", "3.96", "4", "1"], ["2010-08", "2010-08-14", "1", "4", "3.96", "8", "4"], ["2010-08", "2010-08-15", "1", "6", "5.94", "14", "4"], ["2010-08", "2010-08-18", "1", "9", "8.91", "23", "6"], ["2010-08", "2010-08-23", "1", "14", "13.86", "37", "9"], ["2010-08", "2010-08-31", "1", "1", "0.99", "38", "14"], ["2010-09", "2010-09-13", "2", "4", "3.96", "4", "1"], ["2010-09", "2010-09-14", "1", "4", "3.96", "8", "4"], ["2010-09", "2010-09-15", "1", "6", "5.94", "14", "4"], ["2010-09", "2010-09-18", "1", "9", "8.91", "23", "6"], ["2010-09", "2010-09-23", "1", "14", "13.86", "37", "9"], ["2010-10", "2010-10-01", "1", "1", "0.99", "1", "14"], ["2010-10", "2010-10-14", "2", "4", "3.96", "5", "1"], ["2010-10", "2010-10-15", "1", "4", "3.96", "9", "4"], ["2010-10", "2010-10-16", "1", "6", "5.94", "15", "4"], ["2010-10", "2010-10-19", "1", "9", "8.91", "24", "6"], ["2010-10", "2010-10-24", "1", "14", "13.86", "38", "9"], ["2010-11", "2010-11-01", "1", "1", "0.99", "1", "14"], ["2010-11", "2010-11-14", "2", "4", "3.96", "5", "1"], ["2010-11", "2010-11-15", "1", "4", "3.96", "9", "4"], ["2010-11", "2010-11-16", "1", "6", "5.94", "15", "4"], ["2010-11", "2010-11-19", "1", "9", "8.91", "24", "6"], ["2010-11", "2010-11-24", "1", "14", "13.86", "38", "9"], ["2010-12", "2010-12-02", "1", "1", "0.99", "1", "14"], ["2010-12", "2010-12-15", "2", "4", "3.96", "5", "1"], ["2010-12", "2010-12-16", "1", "4", "3.96", "9", "4"], ["2010-12", "2010-12-17", "1", "6", "5.94", "15", "4"], ["2010-12", "2010-12-20", "1", "9", "8.91", "24", "6"], ["2010-12", "2010-12-25", "1", "14", "13.86", "38", "9"], ["2011-01", "2011-01-02", "1", "1", "0.99", "1", "14"], ["2011-01", "2011-01-15", "2", "4", "3.96", "5", "1"], ["2011-01", "2011-01-16", "1", "4", "3.96", "9", "4"], ["2011-01", "2011-01-17", "1", "6", "5.94", "15", "4"], ["2011-01", "2011-01-20", "1", "9", "8.91", "24", "6"], ["2011-01", "2011-01-25", "1", "14", "13.86", "38", "9"], ["2011-02", "2011-02-02", "1", "1", "0.99", "1", "14"], ["2011-02", "2011-02-15", "2", "4", "3.96", "5", "1"], ["2011-02", "2011-02-16", "1", "4", "3.96", "9", "4"], ["2011-02", "2011-02-17", "1", "6", "5.94", "15", "4"], ["2011-02", "2011-02-20", "1", "9", "8.91", "24", "6"], ["2011-02", "2011-02-25", "1", "14", "13.86", "38", "9"], ["2011-03", "2011-03-05", "1", "1", "0.99", "1", "14"], ["2011-03", "2011-03-18", "2", "4", "3.96", "5", "1"], ["2011-03", "2011-03-19", "1", "4", "3.96", "9", "4"], ["2011-03", "2011-03-20", "1", "6", "5.94", "15", "4"], ["2011-03", "2011-03-23", "1", "9", "8.91", "24", "6"], ["2011-03", "2011-03-28", "1", "14", "13.86", "38", "9"], ["2011-04", "2011-04-05", "1", "1", "0.99", "1", "14"], ["2011-04", "2011-04-18", "2", "4", "3.96", "5", "1"], ["2011-04", "2011-04-19", "1", "4", "3.96", "9", "4"], ["2011-04", "2011-04-20", "1", "6", "5.94", "15", "4"], ["2011-04", "2011-04-23", "1", "9", "14.91", "24", "6"], ["2011-04", "2011-04-28", "1", "14", "21.86", "38", "9"], ["2011-05", "2011-05-06", "1", "1", "0.99", "1", "14"], ["2011-05", "2011-05-19", "2", "4", "3.96", "5", "1"], ["2011-05", "2011-05-20", "1", "4", "3.96", "9", "4"], ["2011-05", "2011-05-21", "1", "6", "5.94", "15", "4"], ["2011-05", "2011-05-24", "1", "9", "8.91", "24", "6"], ["2011-05", "2011-05-29", "1", "14", "18.86", "38", "9"], ["2011-06", "2011-06-06", "1", "1", "1.99", "1", "14"], ["2011-06", "2011-06-19", "2", "4", "6.96", "5", "1"], ["2011-06", "2011-06-20", "1", "4", "7.96", "9", "4"], ["2011-06", "2011-06-21", "1", "6", "8.94", "15", "4"], ["2011-06", "2011-06-24", "1", "9", "8.91", "24", "6"], ["2011-06", "2011-06-29", "1", "14", "15.86", "38", "9"], ["2011-07", "2011-07-07", "1", "1", "0.99", "1", "14"], ["2011-07", "2011-07-20", "2", "4", "3.96", "5", "1"], ["2011-07", "2011-07-21", "1", "4", "3.96", "9", "4"], ["2011-07", "2011-07-22", "1", "6", "5.94", "15", "4"], ["2011-07", "2011-07-25", "1", "9", "8.91", "24", "6"], ["2011-07", "2011-07-30", "1", "14", "13.86", "38", "9"], ["2011-08", "2011-08-07", "1", "1", "0.99", "1", "14"], ["2011-08", "2011-08-20", "2", "4", "3.96", "5", "1"], ["2011-08", "2011-08-21", "1", "4", "3.96", "9", "4"], ["2011-08", "2011-08-22", "1", "6", "5.94", "15", "4"], ["2011-08", "2011-08-25", "1", "9", "8.91", "24", "6"], ["2011-08", "2011-08-30", "1", "14", "13.86", "38", "9"], ["2011-09", "2011-09-07", "1", "1", "0.99", "1", "14"], ["2011-09", "2011-09-20", "2", "4", "3.96", "5", "1"], ["2011-09", "2011-09-21", "1", "4", "3.96", "9", "4"], ["2011-09", "2011-09-22", "1", "6", "5.94", "15", "4"], ["2011-09", "2011-09-25", "1", "9", "8.91", "24", "6"], ["2011-09", "2011-09-30", "1", "14", "13.86", "38", "9"], ["2011-10", "2011-10-08", "1", "1", "0.99", "1", "14"], ["2011-10", "2011-10-21", "2", "4", "3.96", "5", "1"], ["2011-10", "2011-10-22", "1", "4", "3.96", "9", "4"], ["2011-10", "2011-10-23", "1", "6", "5.94", "15", "4"], ["2011-10", "2011-10-26", "1", "9", "8.91", "24", "6"], ["2011-10", "2011-10-31", "1", "14", "13.86", "38", "9"], ["2011-11", "2011-11-08", "1", "1", "0.99", "1", "14"], ["2011-11", "2011-11-21", "2", "4", "3.96", "5", "1"], ["2011-11", "2011-11-22", "1", "4", "3.96", "9", "4"], ["2011-11", "2011-11-23", "1", "6", "5.94", "15", "4"], ["2011-11", "2011-11-26", "1", "9", "8.91", "24", "6"], ["2011-12", "2011-12-01", "1", "14", "13.86", "14", "9"], ["2011-12", "2011-12-09", "1", "1", "0.99", "15", "14"], ["2011-12", "2011-12-22", "2", "4", "3.96", "19", "1"], ["2011-12", "2011-12-23", "1", "4", "3.96", "23", "4"], ["2011-12", "2011-12-24", "1", "6", "5.94", "29", "4"], ["2011-12", "2011-12-27", "1", "9", "8.91", "38", "6"], ["2012-01", "2012-01-01", "1", "14", "13.86", "14", "9"], ["2012-01", "2012-01-09", "1", "1", "0.99", "15", "14"], ["2012-01", "2012-01-22", "2", "4", "3.96", "19", "1"], ["2012-01", "2012-01-23", "1", "4", "3.96", "23", "4"], ["2012-01", "2012-01-24", "1", "6", "5.94", "29", "4"], ["2012-01", "2012-01-27", "1", "9", "8.91", "38", "6"], ["2012-02", "2012-02-01", "1", "14", "13.86", "14", "9"], ["2012-02", "2012-02-09", "1", "1", "0.99", "15", "14"], ["2012-02", "2012-02-22", "2", "4", "3.96", "19", "1"], ["2012-02", "2012-02-23", "1", "4", "3.96", "23", "4"], ["2012-02", "2012-02-24", "1", "6", "5.94", "29", "4"], ["2012-02", "2012-02-27", "1", "9", "8.91", "38", "6"], ["2012-03", "2012-03-03", "1", "14", "13.86", "14", "9"], ["2012-03", "2012-03-11", "1", "1", "0.99", "15", "14"], ["2012-03", "2012-03-24", "2", "4", "3.96", "19", "1"], ["2012-03", "2012-03-25", "1", "4", "3.96", "23", "4"], ["2012-03", "2012-03-26", "1", "6", "5.94", "29", "4"], ["2012-03", "2012-03-29", "1", "9", "8.91", "38", "6"], ["2012-04", "2012-04-03", "1", "14", "13.86", "14", "9"], ["2012-04", "2012-04-11", "1", "1", "0.99", "15", "14"], ["2012-04", "2012-04-24", "2", "4", "3.96", "19", "1"], ["2012-04", "2012-04-25", "1", "4", "3.96", "23", "4"], ["2012-04", "2012-04-26", "1", "6", "5.94", "29", "4"], ["2012-04", "2012-04-29", "1", "9", "8.91", "38", "6"], ["2012-05", "2012-05-04", "1", "14", "13.86", "14", "9"], ["2012-05", "2012-05-12", "1", "1", "0.99", "15", "14"], ["2012-05", "2012-05-25", "2", "4", "3.96", "19", "1"], ["2012-05", "2012-05-26", "1", "4", "3.96", "23", "4"], ["2012-05", "2012-05-27", "1", "6", "5.94", "29", "4"], ["2012-05", "2012-05-30", "1", "9", "8.91", "38", "6"], ["2012-06", "2012-06-04", "1", "14", "13.86", "14", "9"], ["2012-06", "2012-06-12", "1", "1", "0.99", "15", "14"], ["2012-06", "2012-06-25", "2", "4", "3.96", "19", "1"], ["2012-06", "2012-06-26", "1", "4", "3.96", "23", "4"], ["2012-06", "2012-06-27", "1", "6", "5.94", "29", "4"], ["2012-06", "2012-06-30", "1", "9", "8.91", "38", "6"], ["2012-07", "2012-07-05", "1", "14", "13.86", "14", "9"], ["2012-07", "2012-07-13", "1", "1", "0.99", "15", "14"], ["2012-07", "2012-07-26", "2", "4", "3.96", "19", "1"], ["2012-07", "2012-07-27", "1", "4", "3.96", "23", "4"], ["2012-07", "2012-07-28", "1", "6", "5.94", "29", "4"], ["2012-07", "2012-07-31", "1", "9", "10.91", "38", "6"], ["2012-08", "2012-08-05", "1", "14", "23.86", "14", "9"], ["2012-08", "2012-08-13", "1", "1", "0.99", "15", "14"], ["2012-08", "2012-08-26", "2", "4", "3.96", "19", "1"], ["2012-08", "2012-08-27", "1", "4", "3.96", "23", "4"], ["2012-08", "2012-08-28", "1", "6", "5.94", "29", "4"], ["2012-08", "2012-08-31", "1", "9", "8.91", "38", "6"], ["2012-09", "2012-09-05", "1", "14", "16.86", "14", "9"], ["2012-09", "2012-09-13", "1", "1", "1.99", "15", "14"], ["2012-09", "2012-09-26", "2", "4", "7.96", "19", "1"], ["2012-09", "2012-09-27", "1", "4", "7.96", "23", "4"], ["2012-09", "2012-09-28", "1", "6", "11.94", "29", "4"], ["2012-10", "2012-10-01", "1", "9", "10.91", "9", "6"], ["2012-10", "2012-10-06", "1", "14", "16.86", "23", "9"], ["2012-10", "2012-10-14", "1", "1", "0.99", "24", "14"], ["2012-10", "2012-10-27", "2", "4", "3.96", "28", "1"], ["2012-10", "2012-10-28", "1", "4", "3.96", "32", "4"], ["2012-10", "2012-10-29", "1", "6", "5.94", "38", "4"], ["2012-11", "2012-11-01", "1", "9", "8.91", "9", "6"], ["2012-11", "2012-11-06", "1", "14", "13.86", "23", "9"], ["2012-11", "2012-11-14", "1", "1", "0.99", "24", "14"], ["2012-11", "2012-11-27", "2", "4", "3.96", "28", "1"], ["2012-11", "2012-11-28", "1", "4", "3.96", "32", "4"], ["2012-11", "2012-11-29", "1", "6", "5.94", "38", "4"], ["2012-12", "2012-12-02", "1", "9", "8.91", "9", "6"], ["2012-12", "2012-12-07", "1", "14", "13.86", "23", "9"], ["2012-12", "2012-12-15", "1", "1", "0.99", "24", "14"], ["2012-12", "2012-12-28", "2", "4", "3.96", "28", "1"], ["2012-12", "2012-12-29", "1", "4", "3.96", "32", "4"], ["2012-12", "2012-12-30", "1", "6", "5.94", "38", "4"], ["2013-01", "2013-01-02", "1", "9", "8.91", "9", "6"], ["2013-01", "2013-01-07", "1", "14", "13.86", "23", "9"], ["2013-01", "2013-01-15", "1", "1", "0.99", "24", "14"], ["2013-01", "2013-01-28", "2", "4", "3.96", "28", "1"], ["2013-01", "2013-01-29", "1", "4", "3.96", "32", "4"], ["2013-01", "2013-01-30", "1", "6", "5.94", "38", "4"], ["2013-02", "2013-02-02", "1", "9", "8.91", "9", "6"], ["2013-02", "2013-02-07", "1", "14", "13.86", "23", "9"], ["2013-02", "2013-02-15", "1", "1", "0.99", "24", "14"], ["2013-02", "2013-02-28", "2", "4", "3.96", "28", "1"], ["2013-03", "2013-03-01", "1", "4", "3.96", "4", "4"], ["2013-03", "2013-03-02", "1", "6", "5.94", "10", "4"], ["2013-03", "2013-03-05", "1", "9", "8.91", "19", "6"], ["2013-03", "2013-03-10", "1", "14", "13.86", "33", "9"], ["2013-03", "2013-03-18", "1", "1", "0.99", "34", "14"], ["2013-03", "2013-03-31", "2", "4", "3.96", "38", "1"], ["2013-04", "2013-04-01", "1", "4", "3.96", "4", "4"], ["2013-04", "2013-04-02", "1", "6", "5.94", "10", "4"], ["2013-04", "2013-04-05", "1", "9", "8.91", "19", "6"], ["2013-04", "2013-04-10", "1", "14", "13.86", "33", "9"], ["2013-04", "2013-04-18", "1", "1", "0.99", "34", "14"], ["2013-05", "2013-05-01", "2", "4", "3.96", "4", "1"], ["2013-05", "2013-05-02", "1", "4", "3.96", "8", "4"], ["2013-05", "2013-05-03", "1", "6", "5.94", "14", "4"], ["2013-05", "2013-05-06", "1", "9", "8.91", "23", "6"], ["2013-05", "2013-05-11", "1", "14", "13.86", "37", "9"], ["2013-05", "2013-05-19", "1", "1", "0.99", "38", "14"], ["2013-06", "2013-06-01", "2", "4", "3.96", "4", "1"], ["2013-06", "2013-06-02", "1", "4", "3.96", "8", "4"], ["2013-06", "2013-06-03", "1", "6", "5.94", "14", "4"], ["2013-06", "2013-06-06", "1", "9", "8.91", "23", "6"], ["2013-06", "2013-06-11", "1", "14", "13.86", "37", "9"], ["2013-06", "2013-06-19", "1", "1", "0.99", "38", "14"], ["2013-07", "2013-07-02", "2", "4", "3.96", "4", "1"], ["2013-07", "2013-07-03", "1", "4", "3.96", "8", "4"], ["2013-07", "2013-07-04", "1", "6", "5.94", "14", "4"], ["2013-07", "2013-07-07", "1", "9", "8.91", "23", "6"], ["2013-07", "2013-07-12", "1", "14", "13.86", "37", "9"], ["2013-07", "2013-07-20", "1", "1", "0.99", "38", "14"], ["2013-08", "2013-08-02", "2", "4", "3.96", "4", "1"], ["2013-08", "2013-08-03", "1", "4", "3.96", "8", "4"], ["2013-08", "2013-08-04", "1", "6", "5.94", "14", "4"], ["2013-08", "2013-08-07", "1", "9", "8.91", "23", "6"], ["2013-08", "2013-08-12", "1", "14", "13.86", "37", "9"], ["2013-08", "2013-08-20", "1", "1", "0.99", "38", "14"], ["2013-09", "2013-09-02", "2", "4", "3.96", "4", "1"], ["2013-09", "2013-09-03", "1", "4", "3.96", "8", "4"], ["2013-09", "2013-09-04", "1", "6", "5.94", "14", "4"], ["2013-09", "2013-09-07", "1", "9", "8.91", "23", "6"], ["2013-09", "2013-09-12", "1", "14", "13.86", "37", "9"], ["2013-09", "2013-09-20", "1", "1", "0.99", "38", "14"], ["2013-10", "2013-10-03", "2", "4", "3.96", "4", "1"], ["2013-10", "2013-10-04", "1", "4", "3.96", "8", "4"], ["2013-10", "2013-10-05", "1", "6", "5.94", "14", "4"], ["2013-10", "2013-10-08", "1", "9", "8.91", "23", "6"], ["2013-10", "2013-10-13", "1", "14", "13.86", "37", "9"], ["2013-10", "2013-10-21", "1", "1", "0.99", "38", "14"], ["2013-11", "2013-11-03", "2", "4", "3.96", "4", "1"], ["2013-11", "2013-11-04", "1", "4", "3.96", "8", "4"], ["2013-11", "2013-11-05", "1", "6", "5.94", "14", "4"], ["2013-11", "2013-11-08", "1", "9", "8.91", "23", "6"], ["2013-11", "2013-11-13", "1", "14", "25.86", "37", "9"], ["2013-11", "2013-11-21", "1", "1", "0.99", "38", "14"], ["2013-12", "2013-12-04", "2", "4", "3.96", "4", "1"], ["2013-12", "2013-12-05", "1", "4", "3.96", "8", "4"], ["2013-12", "2013-12-06", "1", "6", "5.94", "14", "4"], ["2013-12", "2013-12-09", "1", "9", "8.91", "23", "6"], ["2013-12", "2013-12-14", "1", "14", "13.86", "37", "9"], ["2013-12", "2013-12-22", "1", "1", "1.99", "38", "14"]] diff --git a/prql-compiler/tests/integration-rdbms/snapshots/integration_rdbms__tests__rdbms@loop.prql.snap b/prql-compiler/tests/integration-rdbms/snapshots/integration_rdbms__tests__rdbms@loop.prql.snap new file mode 100644 index 000000000000..58ff5dde5c95 --- /dev/null +++ b/prql-compiler/tests/integration-rdbms/snapshots/integration_rdbms__tests__rdbms@loop.prql.snap @@ -0,0 +1,6 @@ +--- +source: prql-compiler/tests/integration-rdbms/main.rs +expression: "format!(\"{:?}\", first_result.1)" +input_file: prql-compiler/tests/integration/queries/loop.prql +--- +[["-2"], ["0"], ["2"], ["4"], ["6"], ["8"]] diff --git a/prql-compiler/tests/integration-rdbms/snapshots/integration_rdbms__tests__rdbms@pipelines.prql.snap b/prql-compiler/tests/integration-rdbms/snapshots/integration_rdbms__tests__rdbms@pipelines.prql.snap new file mode 100644 index 000000000000..1399a70119ed --- /dev/null +++ b/prql-compiler/tests/integration-rdbms/snapshots/integration_rdbms__tests__rdbms@pipelines.prql.snap @@ -0,0 +1,6 @@ +--- +source: prql-compiler/tests/integration-rdbms/main.rs +expression: "format!(\"{:?}\", first_result.1)" +input_file: prql-compiler/tests/integration/queries/pipelines.prql +--- +[["0", "20"], ["0", "21"], ["0", "22"], ["1", "23"], ["1", "24"], ["1", "25"]] diff --git a/prql-compiler/tests/integration-rdbms/snapshots/integration_rdbms__tests__rdbms@set_ops_remove.prql.snap b/prql-compiler/tests/integration-rdbms/snapshots/integration_rdbms__tests__rdbms@set_ops_remove.prql.snap new file mode 100644 index 000000000000..59dd2b6211f1 --- /dev/null +++ b/prql-compiler/tests/integration-rdbms/snapshots/integration_rdbms__tests__rdbms@set_ops_remove.prql.snap @@ -0,0 +1,6 @@ +--- +source: prql-compiler/tests/integration-rdbms/main.rs +expression: "format!(\"{:?}\", first_result.1)" +input_file: prql-compiler/tests/integration/queries/set_ops_remove.prql +--- +[["3"]] diff --git a/prql-compiler/tests/integration-rdbms/snapshots/integration_rdbms__tests__rdbms@switch.prql.snap b/prql-compiler/tests/integration-rdbms/snapshots/integration_rdbms__tests__rdbms@switch.prql.snap new file mode 100644 index 000000000000..d3a93beee26a --- /dev/null +++ b/prql-compiler/tests/integration-rdbms/snapshots/integration_rdbms__tests__rdbms@switch.prql.snap @@ -0,0 +1,6 @@ +--- +source: prql-compiler/tests/integration-rdbms/main.rs +expression: "format!(\"{:?}\", first_result.1)" +input_file: prql-compiler/tests/integration/queries/switch.prql +--- +[["Samuel Rosa", "1071"], ["no composer", "4884"], ["no composer", "6373"], ["no composer", "6635"], ["L. Muggerud", "7941"], ["no composer", "11650"], ["L. Muggerud", "21211"], ["unknown composer", "29048"], ["Gilberto Gil", "32287"], ["Chico Science", "33149"]] diff --git a/prql-compiler/tests/integration/data/chinook/employees.csv b/prql-compiler/tests/integration/data/chinook/employees.csv index cb61d44bbcb8..70842000f38a 100644 --- a/prql-compiler/tests/integration/data/chinook/employees.csv +++ b/prql-compiler/tests/integration/data/chinook/employees.csv @@ -1,5 +1,5 @@ employee_id,last_name,first_name,title,reports_to,birth_date,hire_date,address,city,state,country,postal_code,phone,fax,email -1,Adams,Andrew,General Manager,,1962-02-18T00:00:00.000000000,2002-08-14T00:00:00.000000000,11120 Jasper Ave NW,Edmonton,AB,Canada,T5K 2N1,+1 (780) 428-9482,+1 (780) 428-3457,andrew@chinookcorp.com +1,Adams,Andrew,General Manager,6,1962-02-18T00:00:00.000000000,2002-08-14T00:00:00.000000000,11120 Jasper Ave NW,Edmonton,AB,Canada,T5K 2N1,+1 (780) 428-9482,+1 (780) 428-3457,andrew@chinookcorp.com 2,Edwards,Nancy,Sales Manager,1,1958-12-08T00:00:00.000000000,2002-05-01T00:00:00.000000000,825 8 Ave SW,Calgary,AB,Canada,T2P 2T3,+1 (403) 262-3443,+1 (403) 262-3322,nancy@chinookcorp.com 3,Peacock,Jane,Sales Support Agent,2,1973-08-29T00:00:00.000000000,2002-04-01T00:00:00.000000000,1111 6 Ave SW,Calgary,AB,Canada,T2P 5M5,+1 (403) 262-3443,+1 (403) 262-6712,jane@chinookcorp.com 4,Park,Margaret,Sales Support Agent,2,1947-09-19T00:00:00.000000000,2003-05-03T00:00:00.000000000,683 10 Street SW,Calgary,AB,Canada,T2P 5G3,+1 (403) 263-4423,+1 (403) 263-4289,margaret@chinookcorp.com diff --git a/prql-compiler/tests/integration/data/chinook/schema.sql b/prql-compiler/tests/integration/data/chinook/schema.sql index 9539b3ccde12..65a7dc219d94 100644 --- a/prql-compiler/tests/integration/data/chinook/schema.sql +++ b/prql-compiler/tests/integration/data/chinook/schema.sql @@ -1,11 +1,23 @@ -CREATE TABLE invoices(invoice_id INTEGER, customer_id INTEGER, invoice_date TIMESTAMP, billing_address VARCHAR, billing_city VARCHAR, billing_state VARCHAR, billing_country VARCHAR, billing_postal_code VARCHAR, total DOUBLE); -CREATE TABLE customers(customer_id INTEGER, first_name VARCHAR, last_name VARCHAR, company VARCHAR, address VARCHAR, city VARCHAR, state VARCHAR, country VARCHAR, postal_code VARCHAR, phone VARCHAR, fax VARCHAR, email VARCHAR, support_rep_id INTEGER); -CREATE TABLE employees(employee_id INTEGER, last_name VARCHAR, first_name VARCHAR, title VARCHAR, reports_to INTEGER, birth_date TIMESTAMP, hire_date TIMESTAMP, address VARCHAR, city VARCHAR, state VARCHAR, country VARCHAR, postal_code VARCHAR, phone VARCHAR, fax VARCHAR, email VARCHAR); -CREATE TABLE tracks(track_id INTEGER, "name" VARCHAR, album_id INTEGER, media_type_id INTEGER, genre_id INTEGER, composer VARCHAR, "milliseconds" INTEGER, bytes INTEGER, unit_price DOUBLE); -CREATE TABLE albums(album_id INTEGER, title VARCHAR, artist_id INTEGER); -CREATE TABLE genres(genre_id INTEGER, "name" VARCHAR); +DROP TABLE IF EXISTS invoices; +DROP TABLE IF EXISTS customers; +DROP TABLE IF EXISTS employees; +DROP TABLE IF EXISTS tracks; +DROP TABLE IF EXISTS albums; +DROP TABLE IF EXISTS genres; +DROP TABLE IF EXISTS playlist_track; +DROP TABLE IF EXISTS playlists; +DROP TABLE IF EXISTS media_types; +DROP TABLE IF EXISTS artists; +DROP TABLE IF EXISTS invoice_items; + +CREATE TABLE invoices(invoice_id INTEGER, customer_id INTEGER, invoice_date TIMESTAMP, billing_address VARCHAR(255), billing_city VARCHAR(255), billing_state VARCHAR(255), billing_country VARCHAR(255), billing_postal_code VARCHAR(255), total REAL); +CREATE TABLE customers(customer_id INTEGER, first_name VARCHAR(255), last_name VARCHAR(255), company VARCHAR(255), address VARCHAR(255), city VARCHAR(255), state VARCHAR(255), country VARCHAR(255), postal_code VARCHAR(255), phone VARCHAR(255), fax VARCHAR(255), email VARCHAR(255), support_rep_id INTEGER); +CREATE TABLE employees(employee_id INTEGER, last_name VARCHAR(255), first_name VARCHAR(255), title VARCHAR(255), reports_to INTEGER, birth_date TIMESTAMP, hire_date TIMESTAMP, address VARCHAR(255), city VARCHAR(255), state VARCHAR(255), country VARCHAR(255), postal_code VARCHAR(255), phone VARCHAR(255), fax VARCHAR(255), email VARCHAR(255)); +CREATE TABLE tracks(track_id INTEGER, "name" VARCHAR(255), album_id INTEGER, media_type_id INTEGER, genre_id INTEGER, composer VARCHAR(255), "milliseconds" INTEGER, bytes INTEGER, unit_price REAL); +CREATE TABLE albums(album_id INTEGER, title VARCHAR(255), artist_id INTEGER); +CREATE TABLE genres(genre_id INTEGER, "name" VARCHAR(255)); CREATE TABLE playlist_track(playlist_id INTEGER, track_id INTEGER); -CREATE TABLE playlists(playlist_id INTEGER, "name" VARCHAR); -CREATE TABLE media_types(media_type_id INTEGER, "name" VARCHAR); -CREATE TABLE artists(artist_id INTEGER, "name" VARCHAR); -CREATE TABLE invoice_items(invoice_line_id INTEGER, invoice_id INTEGER, track_id INTEGER, unit_price DOUBLE, quantity INTEGER); +CREATE TABLE playlists(playlist_id INTEGER, "name" VARCHAR(255)); +CREATE TABLE media_types(media_type_id INTEGER, "name" VARCHAR(255)); +CREATE TABLE artists(artist_id INTEGER, "name" VARCHAR(255)); +CREATE TABLE invoice_items(invoice_line_id INTEGER, invoice_id INTEGER, track_id INTEGER, unit_price REAL, quantity INTEGER); diff --git a/prql-compiler/tests/integration/queries/group_all.prql b/prql-compiler/tests/integration/queries/group_all.prql index a5a73e1c98ae..58d10065569f 100644 --- a/prql-compiler/tests/integration/queries/group_all.prql +++ b/prql-compiler/tests/integration/queries/group_all.prql @@ -1,5 +1,6 @@ +# skip_postgresql (real cannot be rounded) from a=albums -sort album_id take 10 join tracks [==album_id] -group [a.album_id, a.title] (aggregate price = (sum tracks.unit_price)) +group [a.album_id, a.title] (aggregate price = ((sum tracks.unit_price) | round 2)) +sort album_id diff --git a/prql-compiler/tests/integration/queries/invoice_totals.prql b/prql-compiler/tests/integration/queries/invoice_totals.prql index 45a1d4b80281..1206a1b1a2a3 100644 --- a/prql-compiler/tests/integration/queries/invoice_totals.prql +++ b/prql-compiler/tests/integration/queries/invoice_totals.prql @@ -1,3 +1,5 @@ +# skip_mysql +# skip_mssql from i=invoices join ii=invoice_items [==invoice_id] derive [ diff --git a/prql-compiler/tests/integration/queries/loop.prql b/prql-compiler/tests/integration/queries/loop.prql index e83d02e37057..0d6b320a41aa 100644 --- a/prql-compiler/tests/integration/queries/loop.prql +++ b/prql-compiler/tests/integration/queries/loop.prql @@ -1,3 +1,4 @@ +# skip_mssql from_text format:json '[{"n": 1 }]' select n = n - 2 loop ( diff --git a/prql-compiler/tests/integration/queries/pipelines.prql b/prql-compiler/tests/integration/queries/pipelines.prql index 2b3c6c6cf64a..739de3ca513e 100644 --- a/prql-compiler/tests/integration/queries/pipelines.prql +++ b/prql-compiler/tests/integration/queries/pipelines.prql @@ -1,3 +1,4 @@ +# skip_mssql from tracks sort track_id take 20..25 diff --git a/prql-compiler/tests/integration/queries/set_ops_remove.prql b/prql-compiler/tests/integration/queries/set_ops_remove.prql index 9f88bb59e0ef..1138681b4f3d 100644 --- a/prql-compiler/tests/integration/queries/set_ops_remove.prql +++ b/prql-compiler/tests/integration/queries/set_ops_remove.prql @@ -1,3 +1,4 @@ +# skip_mssql func distinct rel -> (from t = _param.rel | group [t.*] (take 1)) from_text format:json '{ "columns": ["a"], "data": [[1], [2], [2], [3]] }' diff --git a/prql-compiler/tests/integration/queries/switch.prql b/prql-compiler/tests/integration/queries/switch.prql index 7214c48fc708..8c7930f9004f 100644 --- a/prql-compiler/tests/integration/queries/switch.prql +++ b/prql-compiler/tests/integration/queries/switch.prql @@ -1,3 +1,5 @@ +# skip_mssql +# skip_mysql (mysql imports empty strings in csv as empty strings instead of null) from tracks sort milliseconds select display = case [ diff --git a/prql-compiler/tests/integration/snapshots/integration__tests__test@group_all.prql.snap b/prql-compiler/tests/integration/snapshots/integration__tests__test@group_all.prql.snap index 1b5c0359f1bb..8d8930d5d4a1 100644 --- a/prql-compiler/tests/integration/snapshots/integration__tests__test@group_all.prql.snap +++ b/prql-compiler/tests/integration/snapshots/integration__tests__test@group_all.prql.snap @@ -6,11 +6,11 @@ input_file: prql-compiler/tests/integration/queries/group_all.prql album_id,title,price 1,For Those About To Rock We Salute You,9.9 2,Balls to the Wall,0.99 -3,Restless and Wild,2.9699999999999998 -4,Let There Be Rock,7.920000000000001 -5,Big Ones,14.850000000000001 -6,Jagged Little Pill,12.870000000000001 +3,Restless and Wild,2.97 +4,Let There Be Rock,7.92 +5,Big Ones,14.85 +6,Jagged Little Pill,12.87 7,Facelift,11.88 -8,Warner 25 Anos,13.860000000000001 -9,Plays Metallica By Four Cellos,7.920000000000001 -10,Audioslave,13.860000000000001 +8,Warner 25 Anos,13.86 +9,Plays Metallica By Four Cellos,7.92 +10,Audioslave,13.86 From 4e0b57655953861afc3666b8fba30adffcefdc02 Mon Sep 17 00:00:00 2001 From: eitsupi <50911393+eitsupi@users.noreply.github.com> Date: Thu, 30 Mar 2023 23:17:29 +0900 Subject: [PATCH 07/17] chore(test): bind mount read only in compose file for testing (#2352) * test: bind mount read only in compose file for testing * chore: should not chown /tmp/chinook --- prql-compiler/tests/integration-rdbms/conf/my.cnf | 2 +- prql-compiler/tests/integration-rdbms/docker-compose.yml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/prql-compiler/tests/integration-rdbms/conf/my.cnf b/prql-compiler/tests/integration-rdbms/conf/my.cnf index 010d65225a43..34a97baaa0bb 100644 --- a/prql-compiler/tests/integration-rdbms/conf/my.cnf +++ b/prql-compiler/tests/integration-rdbms/conf/my.cnf @@ -1,2 +1,2 @@ [mysqld] -secure-file-priv= /tmp/chinook/ +secure-file-priv = "" diff --git a/prql-compiler/tests/integration-rdbms/docker-compose.yml b/prql-compiler/tests/integration-rdbms/docker-compose.yml index 600a26f410a7..60d855759c01 100644 --- a/prql-compiler/tests/integration-rdbms/docker-compose.yml +++ b/prql-compiler/tests/integration-rdbms/docker-compose.yml @@ -8,7 +8,7 @@ services: POSTGRES_USER: root POSTGRES_PASSWORD: root volumes: &vol - - ../integration/data/chinook:/tmp/chinook + - ../integration/data/chinook:/tmp/chinook:ro mysql: # No arm64 image available; remove when one does become available platform: linux/amd64 @@ -19,8 +19,8 @@ services: MYSQL_DATABASE: dummy MYSQL_ROOT_PASSWORD: root volumes: - - ./conf:/etc/mysql/conf.d - - ../integration/data/chinook:/tmp/chinook + - ./conf/my.cnf:/etc/mysql/conf.d/my.cnf:ro + - ../integration/data/chinook:/tmp/chinook:ro # db2: # image: 'icr.io/db2_community/db2' # ports: From afb6e5ee7485e756127a87d5c8b5da8cc6411c3d Mon Sep 17 00:00:00 2001 From: eitsupi <50911393+eitsupi@users.noreply.github.com> Date: Fri, 31 Mar 2023 02:50:00 +0900 Subject: [PATCH 08/17] build: tweak megalinter config for sql (#2349) * build: disable tsqllint in megalinter * build: add sqlfluff config file * format: autoformatting by sqlfluff * build: ignore `references.keywords` rule of sqlfluff --- .mega-linter.yaml | 2 +- .sqlfluff | 3 + .../tests/integration/data/chinook/schema.sql | 78 ++++++++++++++++--- 3 files changed, 71 insertions(+), 12 deletions(-) create mode 100644 .sqlfluff diff --git a/.mega-linter.yaml b/.mega-linter.yaml index 3d600e1027b2..93a433b7a2ba 100644 --- a/.mega-linter.yaml +++ b/.mega-linter.yaml @@ -11,6 +11,7 @@ DISABLE_LINTERS: - CSS_STYLELINT - PHP_PSALM - PHP_PHPSTAN # Disabled for now as we couldn't figure out how to prevent false positives. #2069 + - SQL_TSQLLINT DISABLE_ERRORS_LINTERS: - COPYPASTE_JSCPD - REPOSITORY_TRIVY @@ -28,7 +29,6 @@ DISABLE_ERRORS_LINTERS: - MAKEFILE_CHECKMAKE - MARKDOWN_MARKDOWN_LINK_CHECK - SPELL_MISSPELL - - SQL_TSQLLINT - YAML_V8R PHP_PHPCS_ARGUMENTS: - --standard=PSR12 diff --git a/.sqlfluff b/.sqlfluff new file mode 100644 index 000000000000..2a6e1b1c97fd --- /dev/null +++ b/.sqlfluff @@ -0,0 +1,3 @@ +[sqlfluff] +dialect = ansi +exclude_rules = references.keywords diff --git a/prql-compiler/tests/integration/data/chinook/schema.sql b/prql-compiler/tests/integration/data/chinook/schema.sql index 65a7dc219d94..fa6151e9eb2d 100644 --- a/prql-compiler/tests/integration/data/chinook/schema.sql +++ b/prql-compiler/tests/integration/data/chinook/schema.sql @@ -10,14 +10,70 @@ DROP TABLE IF EXISTS media_types; DROP TABLE IF EXISTS artists; DROP TABLE IF EXISTS invoice_items; -CREATE TABLE invoices(invoice_id INTEGER, customer_id INTEGER, invoice_date TIMESTAMP, billing_address VARCHAR(255), billing_city VARCHAR(255), billing_state VARCHAR(255), billing_country VARCHAR(255), billing_postal_code VARCHAR(255), total REAL); -CREATE TABLE customers(customer_id INTEGER, first_name VARCHAR(255), last_name VARCHAR(255), company VARCHAR(255), address VARCHAR(255), city VARCHAR(255), state VARCHAR(255), country VARCHAR(255), postal_code VARCHAR(255), phone VARCHAR(255), fax VARCHAR(255), email VARCHAR(255), support_rep_id INTEGER); -CREATE TABLE employees(employee_id INTEGER, last_name VARCHAR(255), first_name VARCHAR(255), title VARCHAR(255), reports_to INTEGER, birth_date TIMESTAMP, hire_date TIMESTAMP, address VARCHAR(255), city VARCHAR(255), state VARCHAR(255), country VARCHAR(255), postal_code VARCHAR(255), phone VARCHAR(255), fax VARCHAR(255), email VARCHAR(255)); -CREATE TABLE tracks(track_id INTEGER, "name" VARCHAR(255), album_id INTEGER, media_type_id INTEGER, genre_id INTEGER, composer VARCHAR(255), "milliseconds" INTEGER, bytes INTEGER, unit_price REAL); -CREATE TABLE albums(album_id INTEGER, title VARCHAR(255), artist_id INTEGER); -CREATE TABLE genres(genre_id INTEGER, "name" VARCHAR(255)); -CREATE TABLE playlist_track(playlist_id INTEGER, track_id INTEGER); -CREATE TABLE playlists(playlist_id INTEGER, "name" VARCHAR(255)); -CREATE TABLE media_types(media_type_id INTEGER, "name" VARCHAR(255)); -CREATE TABLE artists(artist_id INTEGER, "name" VARCHAR(255)); -CREATE TABLE invoice_items(invoice_line_id INTEGER, invoice_id INTEGER, track_id INTEGER, unit_price REAL, quantity INTEGER); +CREATE TABLE invoices ( + invoice_id INTEGER, + customer_id INTEGER, + invoice_date TIMESTAMP, + billing_address VARCHAR(255), + billing_city VARCHAR(255), + billing_state VARCHAR(255), + billing_country VARCHAR(255), + billing_postal_code VARCHAR(255), + total REAL +); +CREATE TABLE customers ( + customer_id INTEGER, + first_name VARCHAR(255), + last_name VARCHAR(255), + company VARCHAR(255), + address VARCHAR(255), + city VARCHAR(255), + state VARCHAR(255), + country VARCHAR(255), + postal_code VARCHAR(255), + phone VARCHAR(255), + fax VARCHAR(255), + email VARCHAR(255), + support_rep_id INTEGER +); +CREATE TABLE employees ( + employee_id INTEGER, + last_name VARCHAR(255), + first_name VARCHAR(255), + title VARCHAR(255), + reports_to INTEGER, + birth_date TIMESTAMP, + hire_date TIMESTAMP, + address VARCHAR(255), + city VARCHAR(255), + state VARCHAR(255), + country VARCHAR(255), + postal_code VARCHAR(255), + phone VARCHAR(255), + fax VARCHAR(255), + email VARCHAR(255) +); +CREATE TABLE tracks ( + track_id INTEGER, + name VARCHAR(255), + album_id INTEGER, + media_type_id INTEGER, + genre_id INTEGER, + composer VARCHAR(255), + milliseconds INTEGER, + bytes INTEGER, + unit_price REAL +); +CREATE TABLE albums (album_id INTEGER, title VARCHAR(255), artist_id INTEGER); +CREATE TABLE genres (genre_id INTEGER, name VARCHAR(255)); +CREATE TABLE playlist_track (playlist_id INTEGER, track_id INTEGER); +CREATE TABLE playlists (playlist_id INTEGER, name VARCHAR(255)); +CREATE TABLE media_types (media_type_id INTEGER, name VARCHAR(255)); +CREATE TABLE artists (artist_id INTEGER, name VARCHAR(255)); +CREATE TABLE invoice_items ( + invoice_line_id INTEGER, + invoice_id INTEGER, + track_id INTEGER, + unit_price REAL, + quantity INTEGER +); From 3aa52a88609995ff8dc2220d49f03d2a8329b2a3 Mon Sep 17 00:00:00 2001 From: Maximilian Roos <5635139+max-sixty@users.noreply.github.com> Date: Thu, 30 Mar 2023 10:54:45 -0700 Subject: [PATCH 09/17] refactor: Use tags for the language code blocks (#2350) * refactor: Use tags for the language code blocks Idea from https://github.com/PRQL/prql/pull/2348 * change to spaces --- web/book/highlight-prql.js | 6 +-- web/book/src/examples/cte.md | 2 +- web/book/src/examples/employees.md | 2 +- web/book/src/language-features/case.md | 4 +- web/book/src/language-features/s-strings.md | 2 +- web/book/src/language-features/strings.md | 2 +- web/book/src/language-features/target.md | 4 +- web/book/src/lib.rs | 55 ++++++++++++--------- web/book/src/queries/functions.md | 2 +- web/book/src/syntax.md | 4 +- web/book/src/transforms/select.md | 6 +-- web/book/tests/snapshot.rs | 46 +++++++++-------- 12 files changed, 74 insertions(+), 61 deletions(-) diff --git a/web/book/highlight-prql.js b/web/book/highlight-prql.js index bc78c7c7d511..e7626be1024f 100644 --- a/web/book/highlight-prql.js +++ b/web/book/highlight-prql.js @@ -205,8 +205,8 @@ formatting = function (hljs) { hljs.registerLanguage("prql", formatting); hljs.registerLanguage("prql_no_test", formatting); -hljs.registerLanguage("prql_error", formatting); -hljs.registerLanguage("prql_no_fmt", formatting); +hljs.registerLanguage("prql,error", formatting); +hljs.registerLanguage("prql,no-fmt", formatting); hljs.registerLanguage("elm", formatting); // These lines should only exists in the book, not the website. @@ -217,7 +217,7 @@ Array.from(document.querySelectorAll("code.language-prql")).forEach( (a) => console.log(a) || hljs.highlightBlock(a) ); -Array.from(document.querySelectorAll("code.language-prql_error")).forEach( +Array.from(document.querySelectorAll("code.language-prql,error")).forEach( (a) => console.log(a) || hljs.highlightBlock(a) ); diff --git a/web/book/src/examples/cte.md b/web/book/src/examples/cte.md index eab8c1592c7a..f0b119d65ada 100644 --- a/web/book/src/examples/cte.md +++ b/web/book/src/examples/cte.md @@ -1,4 +1,4 @@ -```prql_no_fmt +```prql no-fmt let newest_employees = ( from employees sort tenure diff --git a/web/book/src/examples/employees.md b/web/book/src/examples/employees.md index 48467dc87ecb..6ea1a138aba5 100644 --- a/web/book/src/examples/employees.md +++ b/web/book/src/examples/employees.md @@ -99,7 +99,7 @@ select [mng_name, managers.gender, salary_avg, salary_sd] > Find distributions of titles, salaries and genders for each department. -```prql_no_fmt +```prql no-fmt from de=dept_emp join s=salaries side:left [ (s.emp_no == de.emp_no), diff --git a/web/book/src/language-features/case.md b/web/book/src/language-features/case.md index f91768d21952..4fe611bb91c2 100644 --- a/web/book/src/language-features/case.md +++ b/web/book/src/language-features/case.md @@ -6,7 +6,7 @@ PRQL uses `case` for both SQL's `CASE` and `IF` statements. Here's an example: -```prql_no_fmt +```prql no-fmt from employees derive distance = case [ city == "Calgary" => 0, @@ -17,7 +17,7 @@ derive distance = case [ If no condition is met, the value takes a `null` value. To set a default, use a `true` condition: -```prql_no_fmt +```prql no-fmt from employees derive distance = case [ city == "Calgary" => 0, diff --git a/web/book/src/language-features/s-strings.md b/web/book/src/language-features/s-strings.md index 8cde2da4560d..6201d39594e1 100644 --- a/web/book/src/language-features/s-strings.md +++ b/web/book/src/language-features/s-strings.md @@ -61,7 +61,7 @@ should implement it in PRQL or PRQL's stdlib. To output braces from an s-string, use double braces: -```prql_no_fmt +```prql no-fmt from employees derive [ has_valid_title = s"regexp_contains(title, '([a-z0-9]*-){{2,}}')" diff --git a/web/book/src/language-features/strings.md b/web/book/src/language-features/strings.md index 23acb935ce77..7dc7ee8f392b 100644 --- a/web/book/src/language-features/strings.md +++ b/web/book/src/language-features/strings.md @@ -33,7 +33,7 @@ select x = """""I said """hello world"""!""""" Strings can also contain any escape defined by [JSON standard](https://www.ecma-international.org/publications-and-standards/standards/ecma-404/). -```prql_no_fmt +```prql no-fmt from my_table select x = "\t\tline ends here\n \\ " ``` diff --git a/web/book/src/language-features/target.md b/web/book/src/language-features/target.md index 9a7a87391457..f4496f7637b8 100644 --- a/web/book/src/language-features/target.md +++ b/web/book/src/language-features/target.md @@ -15,7 +15,7 @@ sort age take 10 ``` -```prql_no_fmt +```prql no-fmt prql target:sql.mssql from employees @@ -47,7 +47,7 @@ very welcome. PRQL allows specifying a version of the language in the PRQL header, like: -```prql_no_fmt +```prql no-fmt prql version:"0.6.1" from employees diff --git a/web/book/src/lib.rs b/web/book/src/lib.rs index 6f8a71c6ca7b..85f593d437a3 100644 --- a/web/book/src/lib.rs +++ b/web/book/src/lib.rs @@ -96,9 +96,11 @@ impl Preprocessor for ComparisonPreprocessor { } } -pub fn code_block_lang<'a>(event: &'a Event) -> Option<&'a str> { +/// Returns the language of a code block, divided by commas +/// For example: ```prql no-test +pub fn code_block_lang_tags(event: &Event) -> Option> { if let Event::Start(Tag::CodeBlock(CodeBlockKind::Fenced(lang))) = event { - Some(lang.as_ref()) + Some(lang.to_string().split(' ').map(|x| x.to_string()).collect()) } else { None } @@ -110,15 +112,22 @@ fn replace_examples(text: &str) -> Result { while let Some(event) = parser.next() { // If it's not PRQL, just push it and continue - let Some(lang) = code_block_lang(&event) else { + let Some(lang_tags) = code_block_lang_tags(&event) else { cmark_acc.push(event.to_owned()); continue; }; - if !lang.starts_with("prql") { + if !lang_tags.contains(&"prql".to_string()) { cmark_acc.push(event.to_owned()); continue; } + let allowed_tags = ["prql", "no-test", "no-fmt", "error"]; + for tag in lang_tags.iter() { + if !allowed_tags.contains(&tag.as_str()) { + bail!("Unknown code block language: {}", tag) + } + } + let Some(Event::Text(text)) = parser.next() else { bail!("Expected text within code block") }; @@ -127,17 +136,10 @@ fn replace_examples(text: &str) -> Result { let options = prql_compiler::Options::default().no_signature(); let result = compile(&prql, &options); - match lang { - "prql" | "prql_no_fmt" => cmark_acc.push(Event::Html( - table_of_comparison( - &prql, - result - .map_err(|_| anyhow::anyhow!("Query raised an error:\n\n {prql}\n\n"))? - .as_str(), - ) - .into(), - )), - "prql_error" => cmark_acc.push(Event::Html( + if lang_tags.contains(&"no-test".to_string()) { + cmark_acc.push(Event::Html(table_of_prql_only(&prql).into())); + } else if lang_tags.contains(&"error".to_string()) { + cmark_acc.push(Event::Html( table_of_error( &prql, result @@ -151,14 +153,19 @@ fn replace_examples(text: &str) -> Result { .as_str(), ) .into(), - )), - "prql_no_test" => { - cmark_acc.push(Event::Html(table_of_prql_only(&prql).into())); - } - _ => { - bail!("Unknown code block language: {}", lang) - } - }; + )) + } else { + // Either a bare `prql` or with `no-fmt` + cmark_acc.push(Event::Html( + table_of_comparison( + &prql, + result + .map_err(|_| anyhow::anyhow!("Query raised an error:\n\n {prql}\n\n"))? + .as_str(), + ) + .into(), + )) + } // Skip ending tag parser.next(); } @@ -270,7 +277,7 @@ from x import sys ``` -```prql_error +```prql error this is an error ``` "###; diff --git a/web/book/src/queries/functions.md b/web/book/src/queries/functions.md index 5f31e4de8903..e363c3578aee 100644 --- a/web/book/src/queries/functions.md +++ b/web/book/src/queries/functions.md @@ -16,7 +16,7 @@ Functions have two types of parameters: So this function is named `fahrenheit_to_celsius` and has one parameter `temp`: -```prql_no_fmt +```prql no-fmt func fahrenheit_to_celsius temp -> (temp - 32) / 1.8 from cities diff --git a/web/book/src/syntax.md b/web/book/src/syntax.md index 44cd64cbc4e0..9c43d9f6c40e 100644 --- a/web/book/src/syntax.md +++ b/web/book/src/syntax.md @@ -137,7 +137,7 @@ function call, like `foo + bar`. Here's a full rundown of times this applier: -```prql_no_fmt +```prql no-fmt from employees # Requires parentheses, because it's contains a pipe derive is_proximate = (distance | in 0..20) @@ -169,7 +169,7 @@ sort [-distance] This doesn't work, for example (though it should provide a much better error message): -```prql_error +```prql error from employees derive total_distance = sum distance ``` diff --git a/web/book/src/transforms/select.md b/web/book/src/transforms/select.md index b6f9694ef6d9..81f67de6c7fd 100644 --- a/web/book/src/transforms/select.md +++ b/web/book/src/transforms/select.md @@ -50,19 +50,19 @@ We can use `!` to exclude a list of columns. This can operate in two ways: Some examples: -```prql_no_fmt +```prql no-fmt prql target:sql.bigquery from tracks select ![milliseconds,bytes] ``` -```prql_no_fmt +```prql no-fmt from tracks select [track_id, title, composer, bytes] select ![title, composer] ``` -```prql_no_fmt +```prql no-fmt from artists derive nick = name select ![artists.*] diff --git a/web/book/tests/snapshot.rs b/web/book/tests/snapshot.rs index 6149f32287a4..1563b55f15e4 100644 --- a/web/book/tests/snapshot.rs +++ b/web/book/tests/snapshot.rs @@ -2,6 +2,7 @@ use anyhow::{bail, Result}; use globset::Glob; use insta::assert_snapshot; +use mdbook_prql::code_block_lang_tags; use prql_compiler::*; use std::path::{Path, PathBuf}; use std::{collections::HashMap, fs}; @@ -32,6 +33,10 @@ const ROOT_EXAMPLES_PATH: &str = "tests/prql"; /// Collect all the PRQL examples in the book, as a map of . fn collect_book_examples() -> Result> { + // TODO: instead of returning Strings with embedded tags (e.g. `# error`), + // we could instead return a struct with a `prql` field and a struct of its + // metadata. That would make `test_display` work by matching on the metadata + // rather than re-parsing the string. use pulldown_cmark::{Event, Parser}; let glob = Glob::new("**/*.md")?.compile_matcher(); let examples_in_book: HashMap = WalkDir::new(Path::new("./src/")) @@ -46,24 +51,25 @@ fn collect_book_examples() -> Result> { let mut parser = Parser::new(&text); let mut prql_blocks = vec![]; while let Some(event) = parser.next() { - match mdbook_prql::code_block_lang(&event) { - Some(lang) if lang.starts_with("prql") => { - let mut text = String::new(); - while let Some(Event::Text(line)) = parser.next() { - text.push_str(line.to_string().as_str()); - } - if text.is_empty() { - bail!("Expected text after PRQL code block"); - } - if lang == "prql" { - prql_blocks.push(text.to_string()); - } else if lang == "prql_error" { - prql_blocks.push(format!("# Error expected\n\n{text}")); - } else if lang == "prql_no_fmt" { - prql_blocks.push(format!("# Can't yet format & compile\n\n{text}")); - } + let Some(lang_tags) = code_block_lang_tags(&event) else { + continue + }; + + if lang_tags.contains(&"prql".to_string()) { + let mut prql_text = String::new(); + while let Some(Event::Text(line)) = parser.next() { + prql_text.push_str(line.to_string().as_str()); + } + if prql_text.is_empty() { + bail!("Expected text after PRQL code block"); + } + if lang_tags.contains(&"error".to_string()) { + prql_blocks.push(format!("# error\n\n{prql_text}")); + } else if lang_tags.contains(&"no-fmt".to_string()) { + prql_blocks.push(format!("# no-fmt\n\n{prql_text}")); + } else { + prql_blocks.push(prql_text.to_string()); } - _ => {} } } let snapshot_prefix = &dir_entry @@ -104,7 +110,7 @@ fn test_display() -> Result<(), ErrorMessages> { collect_book_examples()? .iter() .try_for_each(|(path, prql)| { - if prql.contains("# Error expected") || prql.contains("# Can't yet format & compile") { + if prql.contains("# error") || prql.contains("# no-fmt") { return Ok(()); } prql_to_pl(prql) @@ -114,7 +120,7 @@ fn test_display() -> Result<(), ErrorMessages> { panic!( " Failed compiling the formatted result of {path:?} -To skip this test for an example, use `prql_no_fmt` as the language label. +To skip this test for an example, use `prql,no-fmt` as the language label. The original PRQL was: @@ -135,7 +141,7 @@ The original PRQL was: #[test] fn test_rq_serialize() -> Result<(), ErrorMessages> { for (_, prql) in collect_book_examples()? { - if prql.contains("# Error expected") { + if prql.contains("# error") { continue; } let rq = prql_to_pl(&prql).map(pl_to_rq)?; From 16e7c72953cb2cdd5c8631c8d024299d2ebd2447 Mon Sep 17 00:00:00 2001 From: Jelenkee <59470612+Jelenkee@users.noreply.github.com> Date: Thu, 30 Mar 2023 20:09:31 +0200 Subject: [PATCH 10/17] docs: Update CHANGELOG.md (#2356) * Update CHANGELOG.md * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Update CHANGELOG.md --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Maximilian Roos <5635139+max-sixty@users.noreply.github.com> --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7509cfbdde7a..748767e7c112 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,8 @@ technically a breaking change to the API. (@max-sixty, #2251) - The `Error` struct now exposes the `MessageKind` enum. (@vanillajonathan, #2307) +- Integration tests run in CI with DuckDB, SQLite, PostgreSQL, MySQL and SQL + Server (@jelenkee, #2286) **New Contributors**: From 113d08cd562ce9a5e45f0ba020e682919ce5cfaa Mon Sep 17 00:00:00 2001 From: Maximilian Roos Date: Sun, 2 Apr 2023 14:16:47 -0700 Subject: [PATCH 11/17] . --- Cargo.lock | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4ce528521382..2577ddedc847 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3764,14 +3764,13 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.26.0" +version = "1.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03201d01c3c27a29c8a5cee5b55a93ddae1ccf6f08f65365c2c918f8c1b76f64" +checksum = "d0de47a4eecbe11f498978a9b29d792f0d2692d1dd003650c24c76510e3bc001" dependencies = [ "autocfg", "bytes", "libc", - "memchr", "mio", "num_cpus", "parking_lot", @@ -3784,13 +3783,13 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "1.8.2" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d266c00fde287f55d3f1c3e96c500c362a2b8c695076ec180f27918820bc6df8" +checksum = "61a573bdc87985e9d6ddeed1b3d864e8a302c847e40d647746df2f1de209d1ce" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.8", ] [[package]] From c3651377cc18e38b0fa202c9eb9f9dae2b1f0ae8 Mon Sep 17 00:00:00 2001 From: Maximilian Roos <5635139+max-sixty@users.noreply.github.com> Date: Fri, 31 Mar 2023 15:04:15 -0700 Subject: [PATCH 12/17] internal: Remove unneeded expression (#2360) We can't hit this since the `*` case is covered above IIUC --- prql-compiler/src/semantic/context.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/prql-compiler/src/semantic/context.rs b/prql-compiler/src/semantic/context.rs index ec75b7ce5d73..01935c4287d5 100644 --- a/prql-compiler/src/semantic/context.rs +++ b/prql-compiler/src/semantic/context.rs @@ -2,7 +2,6 @@ use anyhow::Result; use enum_as_inner::EnumAsInner; use itertools::Itertools; use serde::{Deserialize, Serialize}; -use std::collections::HashSet; use std::{collections::HashMap, fmt::Debug}; use super::module::{Module, NS_DEFAULT_DB, NS_FRAME, NS_FRAME_RIGHT, NS_INFER, NS_SELF, NS_STD}; @@ -198,13 +197,11 @@ impl Context { } // fallback case: this variable can be from a namespace that we don't know all columns of - let decls = if ident.name != "*" { + let decls = { self.root_mod.lookup(&Ident { path: ident.path.clone(), name: NS_INFER.to_string(), }) - } else { - HashSet::new() }; match decls.len() { 0 => Err(format!("Unknown name {ident}")), From 5ae5ac48cbf4abe1cdd329b2b075ad74ae75a54c Mon Sep 17 00:00:00 2001 From: eitsupi <50911393+eitsupi@users.noreply.github.com> Date: Sat, 1 Apr 2023 10:55:58 +0900 Subject: [PATCH 13/17] chore(test): set MySQL's `secure-file-priv=""` option in the command instead of conf file (#2359) * chore(test): remove unused mysql conf file * fix: pass `--secure-file-priv=""` via compose file --- prql-compiler/tests/integration-rdbms/conf/my.cnf | 2 -- prql-compiler/tests/integration-rdbms/docker-compose.yml | 5 ++--- 2 files changed, 2 insertions(+), 5 deletions(-) delete mode 100644 prql-compiler/tests/integration-rdbms/conf/my.cnf diff --git a/prql-compiler/tests/integration-rdbms/conf/my.cnf b/prql-compiler/tests/integration-rdbms/conf/my.cnf deleted file mode 100644 index 34a97baaa0bb..000000000000 --- a/prql-compiler/tests/integration-rdbms/conf/my.cnf +++ /dev/null @@ -1,2 +0,0 @@ -[mysqld] -secure-file-priv = "" diff --git a/prql-compiler/tests/integration-rdbms/docker-compose.yml b/prql-compiler/tests/integration-rdbms/docker-compose.yml index 60d855759c01..dd0e533d916c 100644 --- a/prql-compiler/tests/integration-rdbms/docker-compose.yml +++ b/prql-compiler/tests/integration-rdbms/docker-compose.yml @@ -18,9 +18,8 @@ services: environment: MYSQL_DATABASE: dummy MYSQL_ROOT_PASSWORD: root - volumes: - - ./conf/my.cnf:/etc/mysql/conf.d/my.cnf:ro - - ../integration/data/chinook:/tmp/chinook:ro + command: --secure-file-priv="" + volumes: *vol # db2: # image: 'icr.io/db2_community/db2' # ports: From ee066d429428cc3a51b4130e32aec1878cb64988 Mon Sep 17 00:00:00 2001 From: Maximilian Roos <5635139+max-sixty@users.noreply.github.com> Date: Sat, 1 Apr 2023 17:25:19 -0700 Subject: [PATCH 14/17] chore: Add changelog for 0.7.0 (#2363) --- CHANGELOG.md | 61 ++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 54 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 748767e7c112..5db69352b85b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,23 +1,68 @@ # PRQL Changelog -## 0.6.2 — [unreleased] +## 0.7.0 — 2023-24-01 + +0.7.0 is a fairly small release in terms of new features, with lots of internal +improvements, such as integration tests with a whole range of DBs, a blog post +on Pi day, RFCs for a type system, and more robust language bindings. + +There's a very small breaking change to the rust API, hence the minor version +bump. + +Here's our April 2023 Update, from our +[Readme](https://github.com/PRQL/prql/blob/main/README.md): + +> ### April 2023 update +> +> PRQL is being actively developed by a growing community. It's ready to use by +> the intrepid, either as part of one of our supported extensions, or within +> your own tools, using one of our supported language bindings. +> +> PRQL still has some minor bugs and some missing features, and probably is only +> ready to be rolled out to non-technical teams for fairly simple queries. +> +> Here's our current [Roadmap](https://prql-lang.org/roadmap/) and our +> [Milestones.](https://github.com/PRQL/prql/milestones) +> +> Our immediate focus for the code is on: +> +> - Building out the next few big features, including +> [types](https://github.com/PRQL/prql/pull/1964) and +> [modules](https://github.com/PRQL/prql/pull/2129). +> - Ensuring our supported features feel extremely robust; resolving any +> [priority bugs](https://github.com/PRQL/prql/issues?q=is%3Aissue+is%3Aopen+label%3Abug+label%3Apriority). +> +> We're also spending time thinking about: +> +> - Making it really easy to start using PRQL. We're doing that by building +> integrations with tools that folks already use; for example our VS Code +> extension & Jupyter integration. If there are tools you're familiar with +> that you think would be open to integrating with PRQL, please let us know in +> an issue. +> - Making it easier to contribute to the compiler. We have a wide group of +> contributors to the project, but contributions to the compiler itself are +> quite concentrated. We're keen to expand this; +> [#1840](https://github.com/PRQL/prql/issues/1840) for feedback. + +--- + +The release has 131 commits from 10 contributors. Particular credit goes to to +@eitsupi & @jelenkee, who have made significant contributions, and +@vanillajonathan, whose prolific contribution include our growing language +bindings. + +A small selection of the changes: **Features**: - `prqlc compile` adds `--color` & `--include-signature-comment` options. (@max-sixty, #2267) -**Fixes**: - -**Documentation**: - **Web**: - Added the PRQL snippets from the book to the [Playground](https://prql-lang.org/playground/) (@jelenkee, #2197) -**Integrations**: - **Internal changes**: - _Breaking_: The `compile` function's `Options` now includes a `color` member, @@ -30,6 +75,8 @@ **New Contributors**: +- @k-nut, with #2294 + ## 0.6.1 — 2023-03-12 0.6.1 is a small release containing an internal refactoring and improved From 41f63210df57951b4250912c43eef0b9d09d1b5d Mon Sep 17 00:00:00 2001 From: Maximilian Roos <5635139+max-sixty@users.noreply.github.com> Date: Sat, 1 Apr 2023 17:33:34 -0700 Subject: [PATCH 15/17] chore: Bump versions to 0.7.0 (#2364) --- Cargo.lock | 20 +++++++++---------- Cargo.toml | 2 +- bindings/prql-elixir/native/prql/Cargo.toml | 2 +- bindings/prql-js/package-lock.json | 4 ++-- bindings/prql-js/package.json | 2 +- prql-compiler/prql-compiler-macros/Cargo.toml | 2 +- prql-compiler/prqlc/Cargo.toml | 2 +- web/book/src/language-features/target.md | 2 +- web/playground/package-lock.json | 2 +- 9 files changed, 19 insertions(+), 19 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2577ddedc847..d3b6bcd0b437 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -778,7 +778,7 @@ dependencies = [ [[package]] name = "compile-files" -version = "0.6.1" +version = "0.7.0" dependencies = [ "prql-compiler", ] @@ -2122,7 +2122,7 @@ dependencies = [ [[package]] name = "mdbook-prql" -version = "0.6.1" +version = "0.7.0" dependencies = [ "anyhow", "clap 4.2.0", @@ -2752,7 +2752,7 @@ dependencies = [ [[package]] name = "prql" -version = "0.6.1" +version = "0.7.0" dependencies = [ "prql-compiler", "rustler", @@ -2760,7 +2760,7 @@ dependencies = [ [[package]] name = "prql-compiler" -version = "0.6.1" +version = "0.7.0" dependencies = [ "anyhow", "ariadne", @@ -2797,7 +2797,7 @@ dependencies = [ [[package]] name = "prql-compiler-macros" -version = "0.6.1" +version = "0.7.0" dependencies = [ "prql-compiler", "syn 2.0.8", @@ -2805,7 +2805,7 @@ dependencies = [ [[package]] name = "prql-java" -version = "0.6.1" +version = "0.7.0" dependencies = [ "jni", "prql-compiler", @@ -2813,7 +2813,7 @@ dependencies = [ [[package]] name = "prql-js" -version = "0.6.1" +version = "0.7.0" dependencies = [ "console_error_panic_hook", "prql-compiler", @@ -2823,7 +2823,7 @@ dependencies = [ [[package]] name = "prql-lib" -version = "0.6.1" +version = "0.7.0" dependencies = [ "libc", "prql-compiler", @@ -2832,7 +2832,7 @@ dependencies = [ [[package]] name = "prql-python" -version = "0.6.1" +version = "0.7.0" dependencies = [ "insta", "prql-compiler", @@ -2842,7 +2842,7 @@ dependencies = [ [[package]] name = "prqlc" -version = "0.6.1" +version = "0.7.0" dependencies = [ "anyhow", "ariadne", diff --git a/Cargo.toml b/Cargo.toml index 30d13c22c079..27f6c68bfbc8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,7 +27,7 @@ edition = "2021" license = "Apache-2.0" repository = "https://github.com/PRQL/prql" rust-version = "1.65.0" -version = "0.6.1" +version = "0.7.0" [profile.release.package.prql-js] # Tell `rust-js` to optimize for small code size. diff --git a/bindings/prql-elixir/native/prql/Cargo.toml b/bindings/prql-elixir/native/prql/Cargo.toml index 7194ca4f2b38..cee042bb984c 100644 --- a/bindings/prql-elixir/native/prql/Cargo.toml +++ b/bindings/prql-elixir/native/prql/Cargo.toml @@ -16,7 +16,7 @@ name = "prql" path = "src/lib.rs" [dependencies] -prql-compiler = {path = "../../../../prql-compiler", default-features = false, version = "0.6.1"} +prql-compiler = {path = "../../../../prql-compiler", default-features = false, version = "0.7.0" } # See Readme for details on Mac [target.'cfg(not(any(target_family="wasm", target_os = "macos")))'.dependencies] diff --git a/bindings/prql-js/package-lock.json b/bindings/prql-js/package-lock.json index 025d9190129d..f008473d6013 100644 --- a/bindings/prql-js/package-lock.json +++ b/bindings/prql-js/package-lock.json @@ -1,12 +1,12 @@ { "name": "prql-js", - "version": "0.6.1", + "version": "0.7.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "prql-js", - "version": "0.6.1", + "version": "0.7.0", "license": "Apache-2.0", "devDependencies": { "chai": "^4.3.6", diff --git a/bindings/prql-js/package.json b/bindings/prql-js/package.json index 19ca1964d03c..cfab34a9b60a 100644 --- a/bindings/prql-js/package.json +++ b/bindings/prql-js/package.json @@ -25,5 +25,5 @@ "test": "mocha tests" }, "types": "dist/node/prql_js.d.ts", - "version": "0.6.1" + "version": "0.7.0" } diff --git a/prql-compiler/prql-compiler-macros/Cargo.toml b/prql-compiler/prql-compiler-macros/Cargo.toml index d924bef93050..870a76ea7d8f 100644 --- a/prql-compiler/prql-compiler-macros/Cargo.toml +++ b/prql-compiler/prql-compiler-macros/Cargo.toml @@ -14,7 +14,7 @@ proc_macro = true test = false [dependencies] -prql-compiler = {path = "..", default-features = false, version = "0.6.1"} +prql-compiler = {path = "..", default-features = false, version = "0.7.0" } syn = "2.0.2" [package.metadata.release] diff --git a/prql-compiler/prqlc/Cargo.toml b/prql-compiler/prqlc/Cargo.toml index 75fe64717f40..6ff57ee59946 100644 --- a/prql-compiler/prqlc/Cargo.toml +++ b/prql-compiler/prqlc/Cargo.toml @@ -21,7 +21,7 @@ env_logger = {version = "0.10.0", features = ["color"]} itertools = "0.10.3" minijinja = {version = "0.31.0", features = ["unstable_machinery"]} notify = "^5.1.0" -prql-compiler = {path = '..', version = "0.6.1"} +prql-compiler = {path = '..', version = "0.7.0" } regex = {version = "1.7.1", features = ["std", "unicode"]} serde = "^1" serde_json = "1.0.81" diff --git a/web/book/src/language-features/target.md b/web/book/src/language-features/target.md index f4496f7637b8..4d941d5610ec 100644 --- a/web/book/src/language-features/target.md +++ b/web/book/src/language-features/target.md @@ -48,7 +48,7 @@ very welcome. PRQL allows specifying a version of the language in the PRQL header, like: ```prql no-fmt -prql version:"0.6.1" +prql version:"0.7.0" from employees ``` diff --git a/web/playground/package-lock.json b/web/playground/package-lock.json index e3d63f0058dc..46b00b23f426 100644 --- a/web/playground/package-lock.json +++ b/web/playground/package-lock.json @@ -46,7 +46,7 @@ } }, "../prql-js": { - "version": "0.6.1", + "version": "0.7.0", "extraneous": true, "license": "Apache-2.0", "devDependencies": { From 66c65bfe497988ed89e7b7c868ae8f44112b6582 Mon Sep 17 00:00:00 2001 From: Maximilian Roos <5635139+max-sixty@users.noreply.github.com> Date: Sat, 1 Apr 2023 17:45:50 -0700 Subject: [PATCH 16/17] chore: Add changelog template for 0.7.1 (#2365) --- CHANGELOG.md | 16 ++++++++++++++++ web/book/src/contributing/development.md | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5db69352b85b..167da339b5f8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,21 @@ # PRQL Changelog +## 0.7.1 — [unreleased] + +**Features**: + +**Fixes**: + +**Documentation**: + +**Web**: + +**Integrations**: + +**Internal changes**: + +**New Contributors**: + ## 0.7.0 — 2023-24-01 0.7.0 is a fairly small release in terms of new features, with lots of internal diff --git a/web/book/src/contributing/development.md b/web/book/src/contributing/development.md index 8d55955301c0..cd5d1dc31970 100644 --- a/web/book/src/contributing/development.md +++ b/web/book/src/contributing/development.md @@ -402,7 +402,7 @@ Currently we release in a semi-automated way: 5. Add in the sections for a new Changelog: ```md - ## 0.6.X — [unreleased] + ## 0.7.X — [unreleased] **Features**: From 414fd71c1e9fce77f2cd081631bf876db3043275 Mon Sep 17 00:00:00 2001 From: eitsupi <50911393+eitsupi@users.noreply.github.com> Date: Sun, 2 Apr 2023 09:52:50 +0900 Subject: [PATCH 17/17] docs(book): include PRQL changelog in book (#2348) Close #2344 Since `prql` code blocks in md files included in the Book will evaluated and replace with html, this PR introduce a new Option `no-eval` that modifies the Book preprocessor to not evaluate Changelog's `prql` code blocks. Also add a pre-commit hook to avoid using bare `prql` code blocks on `CHANGELOG.md`. --------- Co-authored-by: Maximilian Roos <5635139+max-sixty@users.noreply.github.com> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .github/workflows/build-web.yaml | 1 + .pre-commit-config.yaml | 11 +++++++++++ CHANGELOG.md | 24 ++++++++++++++---------- web/book/src/SUMMARY.md | 2 ++ web/book/src/changelog.md | 1 + web/book/src/lib.rs | 2 +- 6 files changed, 30 insertions(+), 11 deletions(-) create mode 100644 web/book/src/changelog.md diff --git a/.github/workflows/build-web.yaml b/.github/workflows/build-web.yaml index 7cf0003f8b0d..65aeff6bf2fc 100644 --- a/.github/workflows/build-web.yaml +++ b/.github/workflows/build-web.yaml @@ -5,6 +5,7 @@ on: paths: - "web/**" - ".github/workflows/build-web.yaml" + - "**.md" workflow_call: concurrency: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b78caabe0a13..3ead716e8110 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -53,6 +53,17 @@ repos: types: ["rust"] # A regex searching sql files for either `prj-` or 'tgs-'. entry: "dbg!" + - repo: local + hooks: + - id: prql-codeblock + name: Prevent prql codeblocks evaluating in book + description: + prql code blocks are evaluated and replaced in the book; instead use + `prql no-eval` + language: pygrep + entry: "```prql$" + files: 'CHANGELOG\.md$' + # This is quite strict, and doesn't fix a large enough share of the issues it # finds, so we don't include it. But it's reasonable to run every now & again # manually and take its fixes. diff --git a/CHANGELOG.md b/CHANGELOG.md index 167da339b5f8..862133f1a527 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -298,17 +298,21 @@ This release has 74 commits from 12 contributors. Selected changes: formats. _format-arg_ can be `format:csv` or `format:json`. _string-arg_ can be a string in any format. (@aljazerzen & @snth, #1514) - ```prql + ```prql no-eval from_text format:csv """ a,b,c 1,2,3 4,5,6 """ + ``` + ```prql no-eval from_text format:json ''' [{"a": 1, "b": "x", "c": false }, {"a": 4, "b": "y", "c": null }] ''' + ``` + ```prql no-eval from_text format:json '''{ "columns": ["a", "b", "c"], "data": [ @@ -341,7 +345,7 @@ This release has 74 commits from 12 contributors. Selected changes: - Inferred column names include the relation name (@aljazerzen, #1550): - ```prql + ```prql no-eval from albums select title # name used to be inferred as title only select albums.title # so using albums was not possible here @@ -380,7 +384,7 @@ below in this release). a variable to a value based on one of several expressions (@aljazerzen, #1278). - ```prql + ```prql no-eval derive var = case [ score <= 10 -> "low", score <= 30 -> "medium", @@ -411,7 +415,7 @@ below in this release). - _Experimental:_ Columns can be excluded by name with `select` (@aljazerzen, #1329) - ```prql + ```prql no-eval from albums select ![title, composer] ``` @@ -419,7 +423,7 @@ below in this release). - _Experimental:_ `append` transform, equivalent to `UNION ALL` in SQL. (@aljazerzen, #894) - ```prql + ```prql no-eval from employees append managers ``` @@ -431,7 +435,7 @@ below in this release). - Numbers can contain underscores, which can make reading long numbers easier (@max-sixty, #1467): - ```prql + ```prql no-eval from numbers select [ small = 1.000_000_1, @@ -444,7 +448,7 @@ below in this release). - `dialect` is renamed to `target`, and its values are prefixed with `sql.` (@max-sixty, #1388); for example: - ```prql + ```prql no-eval prql target:sql.bigquery # previously was `dialect:bigquery` from employees @@ -456,7 +460,7 @@ below in this release). - Tables definitions can contain a bare s-string (@max-sixty, #1422), which enables us to include a full CTE of SQL, for example: - ```prql + ```prql no-eval let grouping = s""" SELECT SUM(a) FROM tbl @@ -533,7 +537,7 @@ improvements] - Support for using s-strings for `from` (#1197, @aljazerzen) - ```prql + ```prql no-eval from s"SELECT * FROM employees WHERE foo > 5" ``` @@ -677,7 +681,7 @@ fix rather than a breaking change in semantic versioning. last; for example `round 2 foo_col` / `cast int foo`. This is consistent with other functions, and makes piping possible: - ```prql + ```prql no-eval derive [ gross_salary = (salary + payroll_tax | as int), gross_salary_rounded = (gross_salary | round 0), diff --git a/web/book/src/SUMMARY.md b/web/book/src/SUMMARY.md index 927748e51dc4..c498d5d2ba84 100644 --- a/web/book/src/SUMMARY.md +++ b/web/book/src/SUMMARY.md @@ -84,3 +84,5 @@ - [Name resolving](./internals/name-resolving.md) - [Functions](./internals/functional-lang.md) - [Syntax highlighting](./internals/syntax-highlighting.md) + +- [Changelog](./changelog.md) diff --git a/web/book/src/changelog.md b/web/book/src/changelog.md new file mode 100644 index 000000000000..994b20d1c2b9 --- /dev/null +++ b/web/book/src/changelog.md @@ -0,0 +1 @@ +{{#include ../../../CHANGELOG.md}} diff --git a/web/book/src/lib.rs b/web/book/src/lib.rs index 85f593d437a3..9b379992ae52 100644 --- a/web/book/src/lib.rs +++ b/web/book/src/lib.rs @@ -116,7 +116,7 @@ fn replace_examples(text: &str) -> Result { cmark_acc.push(event.to_owned()); continue; }; - if !lang_tags.contains(&"prql".to_string()) { + if !lang_tags.contains(&"prql".to_string()) || lang_tags.contains(&"no-eval".to_string()) { cmark_acc.push(event.to_owned()); continue; }