Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: show statistics in explain verbose #8113

Merged
merged 7 commits into from
Nov 13, 2023
Merged

feat: show statistics in explain verbose #8113

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
e3e6bc2
feat: show statistics in explain verbose
NGA-TRAN Nov 9, 2023
354d0a2
chore: address review comments
NGA-TRAN Nov 10, 2023
20b04f5
chore: address review comments
NGA-TRAN Nov 13, 2023
1e78ad2
fix: add new enum types in pbjson
NGA-TRAN Nov 13, 2023
99c9f04
fix: add new types into message proto
NGA-TRAN Nov 13, 2023
729535b
Merge remote-tracking branch 'apache/main' into ntran/verbose_stats
alamb Nov 13, 2023
74952a7
Update explain plan
alamb Nov 13, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions datafusion/common/src/display/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,13 +47,17 @@ pub enum PlanType {
FinalLogicalPlan,
/// The initial physical plan, prepared for execution
InitialPhysicalPlan,
/// The initial physical plan with stats, prepared for execution
InitialPhysicalPlanWithStats,
/// The ExecutionPlan which results from applying an optimizer pass
OptimizedPhysicalPlan {
/// The name of the optimizer which produced this plan
optimizer_name: String,
},
/// The final, fully optimized physical which would be executed
FinalPhysicalPlan,
/// The final with stats, fully optimized physical which would be executed
FinalPhysicalPlanWithStats,
}

impl Display for PlanType {
Expand All @@ -69,10 +73,14 @@ impl Display for PlanType {
}
PlanType::FinalLogicalPlan => write!(f, "logical_plan"),
PlanType::InitialPhysicalPlan => write!(f, "initial_physical_plan"),
PlanType::InitialPhysicalPlanWithStats => {
write!(f, "initial_physical_plan_with_stats")
}
PlanType::OptimizedPhysicalPlan { optimizer_name } => {
write!(f, "physical_plan after {optimizer_name}")
}
PlanType::FinalPhysicalPlan => write!(f, "physical_plan"),
PlanType::FinalPhysicalPlanWithStats => write!(f, "physical_plan_with_stats"),
}
}
}
Expand Down
38 changes: 33 additions & 5 deletions datafusion/core/src/physical_planner.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1893,12 +1893,25 @@ impl DefaultPhysicalPlanner {
.await
{
Ok(input) => {
// This plan will includes statistics if show_statistics is on
stringified_plans.push(
displayable(input.as_ref())
.set_show_statistics(config.show_statistics)
.to_stringified(e.verbose, InitialPhysicalPlan),
);

// If the show_statisitcs is off, add another line to show statsitics in the case of explain verbose
if e.verbose && !config.show_statistics {
stringified_plans.push(
displayable(input.as_ref())
.set_show_statistics(true)
.to_stringified(
e.verbose,
InitialPhysicalPlanWithStats,
),
);
}

match self.optimize_internal(
input,
session_state,
Expand All @@ -1912,11 +1925,26 @@ impl DefaultPhysicalPlanner {
);
},
) {
Ok(input) => stringified_plans.push(
displayable(input.as_ref())
.set_show_statistics(config.show_statistics)
.to_stringified(e.verbose, FinalPhysicalPlan),
),
Ok(input) => {
// This plan will includes statistics if show_statistics is on
stringified_plans.push(
displayable(input.as_ref())
.set_show_statistics(config.show_statistics)
.to_stringified(e.verbose, FinalPhysicalPlan),
);

// If the show_statisitcs is off, add another line to show statsitics in the case of explain verbose
if e.verbose && !config.show_statistics {
stringified_plans.push(
displayable(input.as_ref())
.set_show_statistics(true)
.to_stringified(
e.verbose,
FinalPhysicalPlanWithStats,
),
);
}
}
Err(DataFusionError::Context(optimizer_name, e)) => {
let plan_type = OptimizedPhysicalPlan { optimizer_name };
stringified_plans
Expand Down
2 changes: 2 additions & 0 deletions datafusion/proto/proto/datafusion.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1085,8 +1085,10 @@ message PlanType {
OptimizedLogicalPlanType OptimizedLogicalPlan = 2;
EmptyMessage FinalLogicalPlan = 3;
EmptyMessage InitialPhysicalPlan = 4;
EmptyMessage InitialPhysicalPlanWithStats = 9;
OptimizedPhysicalPlanType OptimizedPhysicalPlan = 5;
EmptyMessage FinalPhysicalPlan = 6;
EmptyMessage FinalPhysicalPlanWithStats = 10;
}
}

Expand Down
26 changes: 26 additions & 0 deletions datafusion/proto/src/generated/pbjson.rs
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

6 changes: 5 additions & 1 deletion datafusion/proto/src/generated/prost.rs
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

5 changes: 4 additions & 1 deletion datafusion/proto/src/logical_plan/from_proto.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@ use crate::protobuf::{
self,
plan_type::PlanTypeEnum::{
AnalyzedLogicalPlan, FinalAnalyzedLogicalPlan, FinalLogicalPlan,
FinalPhysicalPlan, InitialLogicalPlan, InitialPhysicalPlan, OptimizedLogicalPlan,
FinalPhysicalPlan, FinalPhysicalPlanWithStats, InitialLogicalPlan,
InitialPhysicalPlan, InitialPhysicalPlanWithStats, OptimizedLogicalPlan,
OptimizedPhysicalPlan,
},
AnalyzedLogicalPlanType, CubeNode, GroupingSetNode, OptimizedLogicalPlanType,
Expand Down Expand Up @@ -406,12 +407,14 @@ impl From<&protobuf::StringifiedPlan> for StringifiedPlan {
}
FinalLogicalPlan(_) => PlanType::FinalLogicalPlan,
InitialPhysicalPlan(_) => PlanType::InitialPhysicalPlan,
InitialPhysicalPlanWithStats(_) => PlanType::InitialPhysicalPlanWithStats,
OptimizedPhysicalPlan(OptimizedPhysicalPlanType { optimizer_name }) => {
PlanType::OptimizedPhysicalPlan {
optimizer_name: optimizer_name.clone(),
}
}
FinalPhysicalPlan(_) => PlanType::FinalPhysicalPlan,
FinalPhysicalPlanWithStats(_) => PlanType::FinalPhysicalPlanWithStats,
},
plan: Arc::new(stringified_plan.plan.clone()),
}
Expand Down
9 changes: 8 additions & 1 deletion datafusion/proto/src/logical_plan/to_proto.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,8 @@ use crate::protobuf::{
arrow_type::ArrowTypeEnum,
plan_type::PlanTypeEnum::{
AnalyzedLogicalPlan, FinalAnalyzedLogicalPlan, FinalLogicalPlan,
FinalPhysicalPlan, InitialLogicalPlan, InitialPhysicalPlan, OptimizedLogicalPlan,
FinalPhysicalPlan, FinalPhysicalPlanWithStats, InitialLogicalPlan,
InitialPhysicalPlan, InitialPhysicalPlanWithStats, OptimizedLogicalPlan,
OptimizedPhysicalPlan,
},
AnalyzedLogicalPlanType, CubeNode, EmptyMessage, GroupingSetNode, LogicalExprList,
Expand Down Expand Up @@ -352,6 +353,12 @@ impl From<&StringifiedPlan> for protobuf::StringifiedPlan {
PlanType::FinalPhysicalPlan => Some(protobuf::PlanType {
plan_type_enum: Some(FinalPhysicalPlan(EmptyMessage {})),
}),
PlanType::InitialPhysicalPlanWithStats => Some(protobuf::PlanType {
plan_type_enum: Some(InitialPhysicalPlanWithStats(EmptyMessage {})),
}),
PlanType::FinalPhysicalPlanWithStats => Some(protobuf::PlanType {
plan_type_enum: Some(FinalPhysicalPlanWithStats(EmptyMessage {})),
}),
},
plan: stringified_plan.plan.to_string(),
}
Expand Down
Loading