From 74c00ed03f25a8d693ed086d7d34465d8dc685ca Mon Sep 17 00:00:00 2001 From: Robin Lee Powell Date: Tue, 4 Feb 2025 17:41:58 -0800 Subject: [PATCH] Debugger tweaks, added tracing - Fixed debugger argument handling - Expanded debugger behaviour a bit (like being able to continue a bunch of times) - Added support for tracing the productions of a grammar during parsing, in both the debugger and normal modes --- README.md | 2 + debugger/Cargo.toml | 1 + debugger/src/lib.rs | 27 +- debugger/src/main.rs | 402 +++++++++++++------------- derive/src/lib.rs | 34 ++- generator/Cargo.toml | 2 +- generator/src/generator.rs | 78 +++-- generator/src/macros.rs | 14 +- generator/src/parse_derive.rs | 98 ++++++- pest/Cargo.toml | 2 + pest/src/iterators/queueable_token.rs | 2 +- pest/src/lib.rs | 36 ++- pest/src/parser_state.rs | 196 ++++++++++++- vm/src/lib.rs | 229 ++++++++++----- 14 files changed, 826 insertions(+), 297 deletions(-) diff --git a/README.md b/README.md index 6d91eafa..10b44171 100644 --- a/README.md +++ b/README.md @@ -161,6 +161,8 @@ mod b { * Input handling * Custom errors * Runs on stable Rust +* Specialized parser debugger +* Optional tracing output ## Projects using pest diff --git a/debugger/Cargo.toml b/debugger/Cargo.toml index 70324579..bfa2c6c6 100644 --- a/debugger/Cargo.toml +++ b/debugger/Cargo.toml @@ -17,6 +17,7 @@ readme = "_README.md" rust-version = "1.61" [dependencies] +clap = { version = "4.0.32", features = ["derive"] } pest = { path = "../pest", version = "2.7.15" } pest_meta = { path = "../meta", version = "2.7.15" } pest_vm = { path = "../vm", version = "2.7.15" } diff --git a/debugger/src/lib.rs b/debugger/src/lib.rs index 09013980..d2fbf645 100644 --- a/debugger/src/lib.rs +++ b/debugger/src/lib.rs @@ -70,7 +70,7 @@ use std::{ thread::{self, JoinHandle}, }; -use pest::{error::Error, Position}; +use pest::{error::Error, Position, TracingConfig, TracingType}; use pest_meta::{ optimizer::OptimizedRule, parse_and_optimize, @@ -133,6 +133,7 @@ pub struct DebuggerContext { grammar: Option>, input: Option, breakpoints: Arc>>, + tracing_config: TracingConfig, } const POISONED_LOCK_PANIC: &str = "poisoned lock"; @@ -211,6 +212,26 @@ impl DebuggerContext { breakpoints.insert(rule); } + /// Sets tracing on and its type + pub fn tracing(&mut self, ttype: TracingType) { + self.tracing_config.ttype = ttype; + } + + /// Sets tracing indent spacing + pub fn tracing_spacing(&mut self, size: usize) { + self.tracing_config.spacing = size; + } + + /// Sets tracing to skip implicit whitespace / comments + pub fn tracing_skip_implicit(&mut self) { + self.tracing_config.skip_implicit = true; + } + + /// Sets tracing to skip silent rules + pub fn tracing_skip_silent(&mut self) { + self.tracing_config.skip_silent = true; + } + /// Removes a rule from breakpoints. pub fn delete_breakpoint(&mut self, rule: &str) { let mut breakpoints = self.breakpoints.lock().expect(POISONED_LOCK_PANIC); @@ -243,6 +264,7 @@ impl DebuggerContext { let breakpoints = Arc::clone(&self.breakpoints); let is_done = Arc::clone(&self.is_done); let is_done_signal = Arc::clone(&self.is_done); + let tracing_config = self.tracing_config; let rsender = sender.clone(); thread::spawn(move || { @@ -269,7 +291,7 @@ impl DebuggerContext { }), ); - match vm.parse(&rule, &input) { + match vm.parse_with_tracing(&rule, &input, tracing_config) { Ok(_) => sender.send(DebuggerEvent::Eof).expect(CHANNEL_CLOSED_PANIC), Err(error) => sender .send(DebuggerEvent::Error(error.to_string())) @@ -364,6 +386,7 @@ impl Default for DebuggerContext { grammar: None, input: None, breakpoints: Arc::new(Mutex::new(HashSet::new())), + tracing_config: Default::default(), } } } diff --git a/debugger/src/main.rs b/debugger/src/main.rs index 7ea83838..ca9da3e2 100644 --- a/debugger/src/main.rs +++ b/debugger/src/main.rs @@ -15,12 +15,14 @@ html_favicon_url = "https://raw.githubusercontent.com/pest-parser/pest/master/pest-logo.svg" )] #![warn(missing_docs, rust_2018_idioms, unused_qualifications)] +use core::panic; use std::path::PathBuf; use std::sync::mpsc::{self, Receiver}; use std::time::Duration; use pest::error::{Error, ErrorVariant}; +use pest::TracingType; use pest_debugger::{DebuggerContext, DebuggerError, DebuggerEvent}; use reqwest::blocking::{Client, ClientBuilder}; use rustyline::completion::{Completer, FilenameCompleter, Pair}; @@ -30,6 +32,101 @@ use rustyline::hint::{Hinter, HistoryHinter}; use rustyline::validate::Validator; use rustyline::{Editor, Helper}; +use clap::{CommandFactory, Parser}; + +#[derive(clap::Parser)] +#[command(version, about, long_about = None)] +struct ClapCliArgs { + /// Select the grammar + #[arg(short, long, alias = "grammar")] + grammar_file: Option, + + /// Select the input file + #[arg(short, long, alias = "input")] + input_file: Option, + + /// Select the session file + #[arg(short, long, alias = "session")] + session_file: Option, + + /// Select the start rule + #[arg(short, long)] + rule: Option, + + /// Select initial breakpoint rules (takes a space separated list) + #[arg(short, long, alias = "breakpoint")] + breakpoints: Vec, + + /// Don't check for updates + #[arg(short, long)] + no_update: bool, + + /// Turn on tracing (string should be PegViz or Indented) + #[arg(short, long)] + tracing: Option, + + /// Set indent depth for Indented tracing + #[arg(long)] + tracing_spacing: Option, + + /// Do not show tracing lines for implicit whitespace and comments + #[arg(long)] + tracing_skip_implicit: bool, + + /// Do not show tracing lines for silent rules + #[arg(long)] + tracing_skip_silent: bool, +} + +#[derive(clap::Parser)] +#[command(version, about, long_about = None)] +struct ClapRLArgs { + #[command(subcommand)] + command: Option, +} + +#[derive(clap::Subcommand)] +#[clap(disable_help_subcommand = true)] +#[clap(disable_help_flag = true)] +#[clap(disable_version_flag = true)] +#[clap(infer_subcommands = true)] +enum Commands { + /// Select the grammar + Grammar { grammar: String }, + /// Select the input file + Input { input: String }, + /// Run starting at the given rule + #[clap(visible_alias("rule"))] + Run { run: String }, + /// Add breakpoints (space separated) + Breakpoints { + #[arg(trailing_var_arg = true, allow_hyphen_values = true)] + breakpoints: Vec, + }, + /// Give an input string + #[clap(visible_alias("text_input"))] + Id { + #[arg(trailing_var_arg = true, allow_hyphen_values = true)] + args: Vec, + }, + /// Set all breakpoints + Ba {}, + /// Delete breakpoints (space separated) + Delete { + #[arg(trailing_var_arg = true, allow_hyphen_values = true)] + delete: Vec, + }, + /// Delete all breakpoints + Da {}, + /// List all breakpoints + List {}, + /// Continue the given number of steps (default 1) + Continue { count: Option }, + // We have to do help ourselves so it doesn't exit :D + /// Show help + Help {}, +} + const VERSION: &str = env!("CARGO_PKG_VERSION"); #[derive(Default)] @@ -51,11 +148,33 @@ impl Cli { self.context.add_breakpoint(rule.to_owned()); } + fn tracing(&mut self, ttype: &str) { + let ttype = match ttype.to_lowercase().as_str() { + "pegviz" => TracingType::PegViz, + "indented" => TracingType::Indented, + "none" => TracingType::Indented, + _ => panic!("Bad tracing type {ttype}!"), + }; + self.context.tracing(ttype); + } + + fn tracing_spacing(&mut self, size: usize) { + self.context.tracing_spacing(size); + } + + fn tracing_skip_implicit(&mut self) { + self.context.tracing_skip_implicit(); + } + + fn tracing_skip_silent(&mut self) { + self.context.tracing_skip_silent(); + } + fn run(&mut self, rule: &str) -> Result<(), DebuggerError> { let (sender, receiver) = mpsc::sync_channel(1); let rec = &receiver; self.context.run(rule, sender)?; - match rec.recv_timeout(Duration::from_secs(5)) { + match rec.recv_timeout(Duration::from_secs(30)) { Ok(DebuggerEvent::Breakpoint(rule, pos)) => { let error: Error<()> = Error::new_from_pos( ErrorVariant::CustomError { @@ -63,6 +182,7 @@ impl Cli { }, self.context.get_position(pos)?, ); + println!("{}", error); } Ok(DebuggerEvent::Eof) => println!("end-of-input reached"), @@ -77,7 +197,7 @@ impl Cli { self.context.cont()?; match self.receiver { - Some(ref rec) => match rec.recv_timeout(Duration::from_secs(5)) { + Some(ref rec) => match rec.recv_timeout(Duration::from_secs(30)) { Ok(DebuggerEvent::Breakpoint(rule, pos)) => { let error: Error<()> = Error::new_from_pos( ErrorVariant::CustomError { @@ -85,6 +205,7 @@ impl Cli { }, self.context.get_position(pos)?, ); + println!("{}", error); } Ok(DebuggerEvent::Eof) => println!("end-of-input reached"), @@ -102,88 +223,60 @@ impl Cli { println!("Breakpoints: {}", breakpoints.join(", ")); } - fn help() { - println!( - "\n\ - Use the following commands:\n\ - g(grammar) - load .pest grammar\n\ - i(input) - load input from a file\n\ - id - load input directly from a single-line input\n\ - ba - add breakpoints at all rules\n\ - b(breakpoint) - add a breakpoint at a rule\n\ - d(delete) - delete a breakpoint at a rule\n\ - da - delete all breakpoints\n\ - r(run) - run a rule\n\ - c(continue) - continue\n\ - l(list) - list breakpoints\n\ - h(help) - help\n\ - " - ); - } - - fn unrecognized(command: &str) { - println!("Unrecognized command: {}; use h for help", command); - } - - fn extract_arg(cmd: &str) -> Option<&str> { - cmd.find(' ').map(|pos| &cmd[pos + 1..]) - } - fn execute_command(&mut self, command: &str) -> Result<(), DebuggerError> { - let verb = command.split(&[' ', '\t']).next().unwrap().trim(); - match verb { - "" => (), - help if "help".starts_with(help) => Cli::help(), - list if "list".starts_with(list) => self.list(), - cont if "continue".starts_with(cont) => self.cont()?, - "ba" => self.context.add_all_rules_breakpoints()?, - "da" => self.context.delete_all_breakpoints(), - grammar if "grammar".starts_with(grammar) => { - let grammar_file = Self::extract_arg(command); - if let Some(grammar_file) = grammar_file { - self.grammar(PathBuf::from(grammar_file))?; - } else { - println!("expected filename, usage: g(grammar) "); - } + // First "argument" needs to be the program name; we don't care so empty string + let mut words = vec![""]; + for word in command.split_whitespace() { + words.push(word); + } + let args = ClapRLArgs::parse_from(&words); + match &args.command { + // We have to do help ourselves so it doesn't exit :D + Some(Commands::Help {}) => { + println!("{}", ClapRLArgs::command().render_long_help()); } - input if "input".starts_with(input) => { - let input_file = Self::extract_arg(command); - if let Some(input_file) = input_file { - self.input(PathBuf::from(input_file))?; + Some(Commands::Grammar { grammar }) => { + self.grammar(PathBuf::from(grammar))?; + } + Some(Commands::Input { input }) => { + self.input(PathBuf::from(input))?; + } + Some(Commands::Run { run }) => { + self.run(run)?; + } + Some(Commands::Ba {}) => self.context.add_all_rules_breakpoints()?, + Some(Commands::Da {}) => self.context.delete_all_breakpoints(), + Some(Commands::Continue { count }) => { + if let Some(x) = count { + for _ in 0..*x { + self.cont()? + } } else { - println!("expected filename, usage: i(input) "); + self.cont()? } } - x if x.starts_with("id ") => { - let input = &command[3..]; - self.context.load_input_direct(input.to_owned()); + // NOTE: It seems possible that there are conditions under which this doesn't work + // because the "arguments" are not ones that Clap likes; if that happens, just pull the + // `id string` handling out of here. Just check that the string starts with "id " and + // keep the rest. + Some(Commands::Id { args }) => { + self.context.load_input_direct(args.join(" ").to_owned()); } - breakpoint if "breakpoint".starts_with(breakpoint) => { - let rule = Self::extract_arg(command); - if let Some(rule) = rule { + Some(Commands::Breakpoints { breakpoints }) => { + for rule in breakpoints { self.breakpoint(rule); - } else { - println!("expected rule, usage: b(breakpoint) "); } } - delete if "delete".starts_with(delete) => { - let rule = Self::extract_arg(command); - if let Some(rule) = rule { + Some(Commands::Delete { delete }) => { + for rule in delete { self.context.delete_breakpoint(rule); - } else { - println!("expected rule, usage: d(delete) "); } } - run if "run".starts_with(run) => { - let rule = Self::extract_arg(command); - if let Some(rule) = rule { - self.run(rule)?; - } else { - println!("expected rule, usage: r(run) "); - } + Some(Commands::List {}) => { + self.list(); } - x => Cli::unrecognized(x), - }; + None => {} + } Ok(()) } } @@ -218,129 +311,10 @@ impl Completer for CliHelper { } } -struct CliArgs { - grammar_file: Option, - input_file: Option, - rule: Option, - breakpoints: Vec, - session_file: Option, - no_update: bool, -} - -impl Default for CliArgs { - fn default() -> Self { - let mut result = Self { - grammar_file: None, - input_file: None, - rule: None, - breakpoints: Vec::new(), - session_file: None, - no_update: false, - }; - let args = std::env::args(); - let mut iter = args.skip(1); - let mut unexpected_arg = false; - while let Some(arg) = iter.next() { - match arg.as_str() { - "-g" | "--grammar" => { - if let Some(grammar_file) = iter.next() { - result.grammar_file = Some(PathBuf::from(grammar_file)); - } else { - eprintln!("Error: missing grammar file"); - std::process::exit(1); - } - } - "-i" | "--input" => { - if let Some(input_file) = iter.next() { - result.input_file = Some(PathBuf::from(input_file)); - } else { - eprintln!("Error: missing input file"); - std::process::exit(1); - } - } - "-r" | "--rule" => { - if let Some(rule) = iter.next() { - result.rule = Some(rule); - } else { - eprintln!("Error: missing rule"); - std::process::exit(1); - } - } - "-b" | "--breakpoint" => { - if let Some(breakpoint) = iter.next() { - result.breakpoints.push(breakpoint); - } else { - eprintln!("Error: missing breakpoint"); - std::process::exit(1); - } - } - "-s" | "--session" => { - if let Some(session_file) = iter.next() { - result.session_file = Some(PathBuf::from(session_file)); - } else { - eprintln!("Error: missing session file"); - std::process::exit(1); - } - } - "--no-update" => { - result.no_update = true; - } - "-h" | "--help" => { - println!( - "\n\ - Usage: pest_debugger [options]\n\ - \n\ - Options:\n\ - -g, --grammar - load .pest grammar\n\ - -i, --input - load input file\n\ - -r, --rule - run rule\n\ - -b, --breakpoint - breakpoint at rule (can be specified multiple times)\n\ - -s, --session - load session history file\n\ - -h, --help - print this help menu\n\ - " - ); - std::process::exit(0); - } - arg => { - eprintln!("Error: unexpected argument `{}`", arg); - unexpected_arg = true; - } - } - } - if unexpected_arg { - std::process::exit(1); - } - result - } -} - -impl CliArgs { - fn init(self, context: &mut Cli) { - if let Some(grammar_file) = self.grammar_file { - if let Err(e) = context.grammar(grammar_file) { - eprintln!("Error: {}", e); - } - } - if let Some(input_file) = self.input_file { - if let Err(e) = context.input(input_file) { - eprintln!("Error: {}", e); - } - } - for breakpoint in self.breakpoints { - context.breakpoint(&breakpoint); - } - if let Some(rule) = self.rule { - if let Err(e) = context.run(&rule) { - eprintln!("Error: {}", e); - } - } - } -} - fn main() -> rustyline::Result<()> { let mut rl = Editor::::new()?; let mut context = Cli::default(); - let cli_args = CliArgs::default(); + let cli_args = ClapCliArgs::parse(); if !cli_args.no_update { let client = ClientBuilder::new() @@ -349,7 +323,7 @@ fn main() -> rustyline::Result<()> { "/", env!("CARGO_PKG_VERSION") )) - .timeout(Some(Duration::from_secs(5))) + .timeout(Some(Duration::from_secs(30))) .build() .ok(); @@ -369,8 +343,11 @@ fn main() -> rustyline::Result<()> { completer: FilenameCompleter::new(), hinter: HistoryHinter {}, }; + rl.set_helper(Some(h)); + println!("pest_debugger v{}\n", VERSION); + let historyfile = if let Some(session_file) = &cli_args.session_file { if let Err(e) = rl.load_history(session_file) { eprintln!("Error loading history file: {}", e); @@ -379,7 +356,46 @@ fn main() -> rustyline::Result<()> { } else { None }; - cli_args.init(&mut context); + + if let Some(grammar_file) = cli_args.grammar_file { + if let Err(e) = context.grammar(grammar_file) { + eprintln!("Error: {}", e); + } + } + if let Some(input_file) = cli_args.input_file { + if let Err(e) = context.input(input_file) { + eprintln!("Error: {}", e); + } + } + // The list is generated by one or more -b arguments, but each -b argument can also be a + // space-separated list + for breakpoint_list in cli_args.breakpoints { + for breakpoint in breakpoint_list.split_whitespace() { + context.breakpoint(breakpoint); + } + } + if let Some(rule) = cli_args.rule { + if let Err(e) = context.run(&rule) { + eprintln!("Error: {}", e); + } + } + + if let Some(tracing) = cli_args.tracing { + context.tracing(&tracing); + } + + if let Some(size) = cli_args.tracing_spacing { + context.tracing_spacing(size); + } + + if cli_args.tracing_skip_implicit { + context.tracing_skip_implicit(); + } + + if cli_args.tracing_skip_silent { + context.tracing_skip_silent(); + } + loop { match rl.readline("> ") { Ok(line) => { diff --git a/derive/src/lib.rs b/derive/src/lib.rs index 923eda72..d43d554a 100644 --- a/derive/src/lib.rs +++ b/derive/src/lib.rs @@ -52,10 +52,40 @@ //! struct MyParser; //! ``` //! -//! ## Inline grammars +//! ### Inline grammars //! //! Grammars can also be inlined by using the `#[grammar_inline = "..."]` attribute. //! +//! ### Tracing +//! +//! Tracing output for your parser, which is very verbose, can be enabled with +//! +//! ```ignore +//! #[tracing(Indented(2))] +//! ``` +//! +//! , where the number is how many spaces to indent each level of trace, or +//! +//! ```ignore +//! #[tracing(PegViz)] +//! ``` +//! +//! , which generates [PegViz compatible](https://github.com/fasterthanlime/pegviz) output. +//! +//! Other tracing options: +//! +//! Skip implicit whitepsace/comment rules in the output: +//! +//! ```ignore +//! #[tracing(SkipImplicit)] +//! ``` +//! +//! Skip silent rules in the output: +//! +//! ```ignore +//! #[tracing(SkipSilent)] +//! ``` +//! //! ## Grammar //! //! A grammar is a series of rules separated by whitespace, possibly containing comments. @@ -319,7 +349,7 @@ use proc_macro::TokenStream; /// The main method that's called by the proc macro /// (a wrapper around `pest_generator::derive_parser`) -#[proc_macro_derive(Parser, attributes(grammar, grammar_inline))] +#[proc_macro_derive(Parser, attributes(grammar, grammar_inline, tracing))] pub fn derive_parser(input: TokenStream) -> TokenStream { pest_generator::derive_parser(input.into(), true).into() } diff --git a/generator/Cargo.toml b/generator/Cargo.toml index c5ae5657..502db4bb 100644 --- a/generator/Cargo.toml +++ b/generator/Cargo.toml @@ -26,4 +26,4 @@ pest = { path = "../pest", version = "2.7.15", default-features = false } pest_meta = { path = "../meta", version = "2.7.15" } proc-macro2 = "1.0" quote = "1.0" -syn = "2.0" +syn = { version = "2.0", features = ["extra-traits"]} diff --git a/generator/src/generator.rs b/generator/src/generator.rs index fd48ec9d..efd5e747 100644 --- a/generator/src/generator.rs +++ b/generator/src/generator.rs @@ -35,6 +35,7 @@ pub fn generate( ) -> TokenStream { let uses_eoi = defaults.iter().any(|name| *name == "EOI"); let name = parsed_derive.name; + let tracing_config = parsed_derive.tracing_config; let builtins = generate_builtin_rules(); let include_fix = if include_grammar { generate_include(&name, paths) @@ -76,6 +77,7 @@ pub fn generate( } pub mod visible { + use ::pest::TracingType; use super::super::Rule; #( #rules )* } @@ -83,7 +85,9 @@ pub fn generate( pub use self::visible::*; } - ::pest::state(input, |state| { + ::pest::state_with_tracing(input, + #tracing_config, + |state| { match rule { #patterns } @@ -300,14 +304,23 @@ fn generate_patterns(rules: &[OptimizedRule], uses_eoi: bool) -> TokenStream { fn generate_rule(rule: OptimizedRule) -> TokenStream { let name = format_ident!("r#{}", rule.name); + let str_name = rule.name.clone(); + + let mut implicit = false; + if rule.name == "WHITESPACE" || rule.name == "COMMENT" { + implicit = true; + } + let expr = if rule.ty == RuleType::Atomic || rule.ty == RuleType::CompoundAtomic { generate_expr_atomic(rule.expr) } else if rule.name == "WHITESPACE" || rule.name == "COMMENT" { let atomic = generate_expr_atomic(rule.expr); quote! { - state.atomic(::pest::Atomicity::Atomic, |state| { - #atomic + state.trace_wrapper(#str_name, #implicit, false, |state| { + state.atomic(::pest::Atomicity::Atomic, |state| { + #atomic + }) }) } } else { @@ -321,8 +334,10 @@ fn generate_rule(rule: OptimizedRule) -> TokenStream { #[inline] #[allow(non_snake_case, unused_variables)] pub fn #name(state: #box_ty<::pest::ParserState<'_, Rule>>) -> ::pest::ParseResult<#box_ty<::pest::ParserState<'_, Rule>>> { - state.rule(Rule::#name, |state| { - #expr + state.trace_wrapper(#str_name, #implicit, false, |state| { + state.rule(Rule::#name, |state| { + #expr + }) }) } }, @@ -330,16 +345,20 @@ fn generate_rule(rule: OptimizedRule) -> TokenStream { #[inline] #[allow(non_snake_case, unused_variables)] pub fn #name(state: #box_ty<::pest::ParserState<'_, Rule>>) -> ::pest::ParseResult<#box_ty<::pest::ParserState<'_, Rule>>> { - #expr + state.trace_wrapper(#str_name, #implicit, true, |state| { + #expr + }) } }, RuleType::Atomic => quote! { #[inline] #[allow(non_snake_case, unused_variables)] pub fn #name(state: #box_ty<::pest::ParserState<'_, Rule>>) -> ::pest::ParseResult<#box_ty<::pest::ParserState<'_, Rule>>> { - state.rule(Rule::#name, |state| { - state.atomic(::pest::Atomicity::Atomic, |state| { - #expr + state.trace_wrapper(#str_name, #implicit, false, |state| { + state.rule(Rule::#name, |state| { + state.atomic(::pest::Atomicity::Atomic, |state| { + #expr + }) }) }) } @@ -348,9 +367,11 @@ fn generate_rule(rule: OptimizedRule) -> TokenStream { #[inline] #[allow(non_snake_case, unused_variables)] pub fn #name(state: #box_ty<::pest::ParserState<'_, Rule>>) -> ::pest::ParseResult<#box_ty<::pest::ParserState<'_, Rule>>> { - state.atomic(::pest::Atomicity::CompoundAtomic, |state| { - state.rule(Rule::#name, |state| { - #expr + state.trace_wrapper(#str_name, #implicit, false, |state| { + state.atomic(::pest::Atomicity::CompoundAtomic, |state| { + state.rule(Rule::#name, |state| { + #expr + }) }) }) } @@ -359,9 +380,11 @@ fn generate_rule(rule: OptimizedRule) -> TokenStream { #[inline] #[allow(non_snake_case, unused_variables)] pub fn #name(state: #box_ty<::pest::ParserState<'_, Rule>>) -> ::pest::ParseResult<#box_ty<::pest::ParserState<'_, Rule>>> { - state.atomic(::pest::Atomicity::NonAtomic, |state| { - state.rule(Rule::#name, |state| { - #expr + state.trace_wrapper(#str_name, #implicit, false, |state| { + state.atomic(::pest::Atomicity::NonAtomic, |state| { + state.rule(Rule::#name, |state| { + #expr + }) }) }) } @@ -374,9 +397,10 @@ fn generate_skip(rules: &[OptimizedRule]) -> TokenStream { let comment = rules.iter().any(|rule| rule.name == "COMMENT"); match (whitespace, comment) { - (false, false) => generate_rule!(skip, Ok(state)), + (false, false) => generate_rule!(skip, true, Ok(state)), (true, false) => generate_rule!( skip, + true, if state.atomicity() == ::pest::Atomicity::NonAtomic { state.repeat(|state| super::visible::WHITESPACE(state)) } else { @@ -385,6 +409,7 @@ fn generate_skip(rules: &[OptimizedRule]) -> TokenStream { ), (false, true) => generate_rule!( skip, + true, if state.atomicity() == ::pest::Atomicity::NonAtomic { state.repeat(|state| super::visible::COMMENT(state)) } else { @@ -393,6 +418,7 @@ fn generate_skip(rules: &[OptimizedRule]) -> TokenStream { ), (true, true) => generate_rule!( skip, + true, if state.atomicity() == ::pest::Atomicity::NonAtomic { state.sequence(|state| { state @@ -1218,6 +1244,7 @@ mod tests { name, generics, non_exhaustive: false, + tracing_config: Default::default(), }; assert_eq!( generate(parsed_derive, vec![PathBuf::from("base.pest"), PathBuf::from("test.pest")], rules, defaults, doc_comment, true).to_string(), @@ -1256,36 +1283,45 @@ mod tests { #[inline] #[allow(dead_code, non_snake_case, unused_variables)] pub fn skip(state: #box_ty<::pest::ParserState<'_, Rule>>) -> ::pest::ParseResult<#box_ty<::pest::ParserState<'_, Rule>>> { - Ok(state) + state.trace_wrapper(stringify!(skip), true, false, |state| { Ok (state) }) } } pub mod visible { + use ::pest::TracingType; use super::super::Rule; #[inline] #[allow(non_snake_case, unused_variables)] pub fn r#a(state: #box_ty<::pest::ParserState<'_, Rule>>) -> ::pest::ParseResult<#box_ty<::pest::ParserState<'_, Rule>>> { - state.match_string("b") + state.trace_wrapper("a", false, true, |state| { state.match_string("b") }) } #[inline] #[allow(non_snake_case, unused_variables)] pub fn r#if(state: #box_ty<::pest::ParserState<'_, Rule>>) -> ::pest::ParseResult<#box_ty<::pest::ParserState<'_, Rule>>> { - self::r#a(state) + state.trace_wrapper("if", false, true, |state| { self::r#a(state) }) } #[inline] #[allow(dead_code, non_snake_case, unused_variables)] pub fn ANY(state: #box_ty<::pest::ParserState<'_, Rule>>) -> ::pest::ParseResult<#box_ty<::pest::ParserState<'_, Rule>>> { - state.skip(1) + state.trace_wrapper(stringify!(ANY), false, false, |state| { state.skip(1) }) } } pub use self::visible::*; } - ::pest::state(input, |state| { + ::pest::state_with_tracing( + input, + ::pest::TracingConfig { + ttype: ::pest::TracingType::None, + spacing: 2usize, + skip_implicit: false, + skip_silent: false, + }, + |state| { match rule { Rule::r#a => rules::r#a(state), Rule::r#if => rules::r#if(state) diff --git a/generator/src/macros.rs b/generator/src/macros.rs index 377f66e6..726e3cb3 100644 --- a/generator/src/macros.rs +++ b/generator/src/macros.rs @@ -9,18 +9,20 @@ macro_rules! insert_builtin { ($builtin: expr, $name: ident, $pattern: expr) => { - $builtin.push((stringify!($name), generate_rule!($name, $pattern))); + $builtin.push((stringify!($name), generate_rule!($name, false, $pattern))); }; } #[cfg(feature = "std")] macro_rules! generate_rule { - ($name: ident, $pattern: expr) => { + ($name: ident, $implicit: ident, $pattern: expr) => { quote! { #[inline] #[allow(dead_code, non_snake_case, unused_variables)] pub fn $name(state: ::std::boxed::Box<::pest::ParserState<'_, Rule>>) -> ::pest::ParseResult<::std::boxed::Box<::pest::ParserState<'_, Rule>>> { - $pattern + state.trace_wrapper(stringify!($name), $implicit, false, |state| { + $pattern + }) } } } @@ -28,12 +30,14 @@ macro_rules! generate_rule { #[cfg(not(feature = "std"))] macro_rules! generate_rule { - ($name: ident, $pattern: expr) => { + ($name: ident, $implicit: ident, $pattern: expr) => { quote! { #[inline] #[allow(dead_code, non_snake_case, unused_variables)] pub fn $name(state: ::alloc::boxed::Box<::pest::ParserState<'_, Rule>>) -> ::pest::ParseResult<::alloc::boxed::Box<::pest::ParserState<'_, Rule>>> { - $pattern + state.trace_wrapper(stringify!($name), $implicit, false, |state| { + $pattern + }) } } } diff --git a/generator/src/parse_derive.rs b/generator/src/parse_derive.rs index 52374c2a..8d8157d1 100644 --- a/generator/src/parse_derive.rs +++ b/generator/src/parse_derive.rs @@ -9,6 +9,7 @@ //! Types and helpers to parse the input of the derive macro. +use pest::{TracingConfig, TracingType}; use syn::{Attribute, DeriveInput, Expr, ExprLit, Generics, Ident, Lit, Meta}; #[derive(Debug, PartialEq)] @@ -25,6 +26,8 @@ pub struct ParsedDerive { pub generics: Generics, /// Indicates whether the 'non_exhaustive' attribute is added to the 'Rule' enum. pub non_exhaustive: bool, + /// Sets tracing settings + pub tracing_config: TracingConfig, } pub(crate) fn parse_derive(ast: DeriveInput) -> (ParsedDerive, Vec) { @@ -46,9 +49,17 @@ pub(crate) fn parse_derive(ast: DeriveInput) -> (ParsedDerive, Vec = ast + .attrs + .iter() + .filter(|attr| attr.meta.path().is_ident("tracing")) + .collect(); + + let tracing_config = attr_to_tracing_config(tracing_items); + let non_exhaustive = ast .attrs .iter() @@ -59,12 +70,13 @@ pub(crate) fn parse_derive(ast: DeriveInput) -> (ParsedDerive, Vec GrammarSource { +fn attr_to_grammar_source(attr: &Attribute) -> GrammarSource { match &attr.meta { Meta::NameValue(name_value) => match &name_value.value { Expr::Lit(ExprLit { @@ -83,6 +95,88 @@ fn get_attribute(attr: &Attribute) -> GrammarSource { } } +fn attr_to_tracing_config(attrs: Vec<&Attribute>) -> TracingConfig { + let mut tracing_config: TracingConfig = Default::default(); + + for attr in attrs { + println!("attr"); + let attr_meta = attr.meta.clone(); + + match attr_meta { + Meta::List(list) => { + list.parse_nested_meta(|meta| { + // #[tracing(Indented(N))] or #[tracing(Indented)] + // + if case_insensitive_ident_check(&meta, "Indented")? { + let content; + if meta.input.peek(syn::token::Paren) { + syn::parenthesized!(content in meta.input); + let lit: syn::LitInt = content.parse()?; + let n: usize = lit.base10_parse()?; + println!("indent: {:#?}", n); + tracing_config.ttype = TracingType::Indented; + tracing_config.spacing = n; + } else { + println!("indent: {:#?}", 2); + tracing_config.ttype = TracingType::Indented; + tracing_config.spacing = 2; + } + return Ok(()); + } + + // #[tracing(PegViz)] + if case_insensitive_ident_check(&meta, "PegViz")? { + tracing_config.ttype = TracingType::PegViz; + return Ok(()); + } + + // #[tracing(SkipImplicit)] + if case_insensitive_ident_check(&meta, "SkipImplicit")? { + tracing_config.skip_implicit = true; + return Ok(()); + } + + // #[tracing(SkipSilent)] + if case_insensitive_ident_check(&meta, "SkipSilent")? { + tracing_config.skip_silent = true; + return Ok(()); + } + + meta.error(format!( + "Token inside `tracing` macro attribute unexpected: {:#?}", + meta.path + )); + + Ok(()) + }) + .expect("Macro attribute `tracing' parse failed."); + } + _ => panic!("wut"), + }; + } + + tracing_config +} + +// This blob is to do case-insensitive ident comparison with error checking. Maybe a bit silly, +// but why force the user to use particular casing? +fn case_insensitive_ident_check( + meta: &syn::meta::ParseNestedMeta<'_>, + str: &str, +) -> Result { + let meta_ident = meta + .path + .get_ident() + .ok_or(meta.error(format!( + "Token should be an ident but isn't: {:#?}", + meta.path + )))? + .to_string() + .to_lowercase(); + + Ok(meta_ident == str.to_lowercase()) +} + #[cfg(test)] mod tests { use super::parse_derive; diff --git a/pest/Cargo.toml b/pest/Cargo.toml index c70db8a5..5ad8fefd 100644 --- a/pest/Cargo.toml +++ b/pest/Cargo.toml @@ -31,6 +31,8 @@ serde_json = { version = "1.0.85", optional = true } thiserror = { version = "2", optional = true } memchr = { version = "2", optional = true } miette = { version = "7.2.0", optional = true, features = ["fancy"] } +quote = "1.0" +proc-macro2 = "1.0" [dev-dependencies] criterion = { version = "0.5.1", features = ["html_reports"] } diff --git a/pest/src/iterators/queueable_token.rs b/pest/src/iterators/queueable_token.rs index 1b520182..cad731cd 100644 --- a/pest/src/iterators/queueable_token.rs +++ b/pest/src/iterators/queueable_token.rs @@ -24,7 +24,7 @@ pub enum QueueableToken<'i, R> { }, End { /// Queue (as a vec) contains both `Start` token and `End` for the same rule. - /// This filed is an index of corresponding `Start` token in vec. + /// This field is an index of corresponding `Start` token in vec. start_token_index: usize, rule: R, tag: Option<&'i str>, diff --git a/pest/src/lib.rs b/pest/src/lib.rs index 0dde153f..b5b38e84 100644 --- a/pest/src/lib.rs +++ b/pest/src/lib.rs @@ -62,13 +62,43 @@ //! //! The syntax of `.pest` files is documented in the [`pest_derive` crate]. //! -//! ## Inline grammars +//! ### Inline grammars //! //! Grammars can also be inlined by using the `#[grammar_inline = "..."]` attribute. //! //! [`Parser`]: trait.Parser.html //! [`pest_derive` crate]: https://docs.rs/pest_derive/ //! +//! ### Tracing +//! +//! Tracing output for your parser, which is very verbose, can be enabled with +//! +//! ```ignore +//! #[tracing(Indented(2))] +//! ``` +//! +//! , where the number is how many spaces to indent each level of trace, or +//! +//! ```ignore +//! #[tracing(PegViz)] +//! ``` +//! +//! , which generates [PegViz compatible](https://github.com/fasterthanlime/pegviz) output. +//! +//! Other tracing options: +//! +//! Skip implicit whitepsace/comment rules in the output: +//! +//! ```ignore +//! #[tracing(SkipImplicit)] +//! ``` +//! +//! Skip silent rules in the output: +//! +//! ```ignore +//! #[tracing(SkipSilent)] +//! ``` +//! //! ## Grammar //! //! A grammar is a series of rules separated by whitespace, possibly containing comments. @@ -336,8 +366,8 @@ extern crate std; pub use crate::parser::Parser; pub use crate::parser_state::{ - set_call_limit, set_error_detail, state, Atomicity, Lookahead, MatchDir, ParseResult, - ParserState, + set_call_limit, set_error_detail, state, state_with_tracing, Atomicity, Lookahead, MatchDir, + ParseResult, ParserState, TracingConfig, TracingType, }; pub use crate::position::Position; pub use crate::span::{merge_spans, Lines, LinesSpan, Span}; diff --git a/pest/src/parser_state.rs b/pest/src/parser_state.rs index 02c57268..2d0014d2 100644 --- a/pest/src/parser_state.rs +++ b/pest/src/parser_state.rs @@ -21,6 +21,9 @@ use core::fmt::{Debug, Display, Formatter}; use core::num::NonZeroUsize; use core::ops::Range; use core::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use proc_macro2::TokenStream; +use quote::quote; +use std::println; use crate::error::{Error, ErrorVariant}; use crate::iterators::pairs::new; @@ -74,6 +77,71 @@ pub enum Atomicity { NonAtomic, } +/// Tracing options +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum TracingType { + /// No tracing output + None, + /// Indented messages for tracing + Indented, + /// Tracing intended to be processed using the PegViz crate + PegViz, +} + +impl quote::ToTokens for TracingType { + fn to_tokens(&self, tokens: &mut TokenStream) { + let path = match self { + TracingType::Indented => quote!(::pest::TracingType::Indented), + TracingType::None => quote!(::pest::TracingType::None), + TracingType::PegViz => quote!(::pest::TracingType::PegViz), + }; + tokens.extend(path); + } +} + +/// Configuration for whether and how to create tracing output during the parser run +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct TracingConfig { + /// Type of tracing: None, Indented, or PegViz + pub ttype: TracingType, + /// How many characters wide to make each indent level + pub spacing: usize, + /// Whether to skip implicit whitespace / comments + pub skip_implicit: bool, + /// Whether to skip silent rules + pub skip_silent: bool, +} + +// Used by `parse_derive` +impl quote::ToTokens for TracingConfig { + fn to_tokens(&self, tokens: &mut TokenStream) { + let ttype = self.ttype; + let spacing = self.spacing; + let si = self.skip_implicit; + let ss = self.skip_silent; + let path = quote! { + ::pest::TracingConfig { + ttype: #ttype, + spacing: #spacing, + skip_implicit: #si, + skip_silent: #ss, + } + }; + tokens.extend(path); + } +} + +impl Default for TracingConfig { + fn default() -> Self { + TracingConfig { + ttype: TracingType::None, + spacing: 2, + skip_implicit: false, + skip_silent: false, + } + } +} + /// Type alias to simplify specifying the return value of chained closures. pub type ParseResult = Result; @@ -442,6 +510,10 @@ pub struct ParserState<'i, R: RuleType> { /// While parsing the query we'll update tracker position to the start of "Bobby", because we'd /// successfully parse "create" + "user" (and not "table"). parse_attempts: ParseAttempts, + /// Configuration for tracing, if any + tracing_config: TracingConfig, + /// Used by the tracing module to track indentation + tracing_depth: usize, } /// Creates a `ParserState` from a `&str`, supplying it to a closure `f`. @@ -458,7 +530,19 @@ pub fn state<'i, R: RuleType, F>(input: &'i str, f: F) -> Result>) -> ParseResult>>, { - let state = ParserState::new(input); + state_with_tracing(input, Default::default(), f) +} + +/// Same as `state` but allows specification of tracing-related attributes for the ParserState +pub fn state_with_tracing<'i, R: RuleType, F>( + input: &'i str, + tracing_config: TracingConfig, + f: F, +) -> Result, Error> +where + F: FnOnce(Box>) -> ParseResult>>, +{ + let state = ParserState::new_with_tracing(input, tracing_config); match f(state) { Ok(state) => { @@ -509,6 +593,10 @@ impl<'i, R: RuleType> ParserState<'i, R> { /// let state: Box> = pest::ParserState::new(input); /// ``` pub fn new(input: &'i str) -> Box { + ParserState::new_with_tracing(input, Default::default()) + } + + pub fn new_with_tracing(input: &'i str, tracing_config: TracingConfig) -> Box { Box::new(ParserState { position: Position::from_start(input), queue: vec![], @@ -520,9 +608,115 @@ impl<'i, R: RuleType> ParserState<'i, R> { stack: Stack::new(), call_tracker: Default::default(), parse_attempts: ParseAttempts::new(), + tracing_config, + tracing_depth: 0, }) } + /// Wrapper for other `ParserState` calls that return `ParseResult`, wraps the calls in tracing + /// output if requested. + /// + /// `implicit` is set to true by callers handling an implicit whitespace / comment rule + /// + /// `silent` is the same for silent rules + /// + /// It's much easier to bail here than in the caller + #[inline] + pub fn trace_wrapper( + mut self: Box, + rule_name: &str, + implicit: bool, + silent: bool, + func: F, + ) -> ParseResult> + where + F: FnOnce(Box) -> ParseResult>, + { + let pos = self.position; + let tracing_type = self.tracing_config.ttype; + + if implicit && self.tracing_config.skip_implicit { + return func(self); + } + + if silent && self.tracing_config.skip_silent { + return func(self); + } + + match tracing_type { + TracingType::None => {} + TracingType::Indented => { + println!( + "{}TRYING `{}` at {}:{}", + " ".repeat(self.tracing_depth * self.tracing_config.spacing), + rule_name, + pos.line_col().0, + pos.line_col().1, + ); + self.tracing_depth += 1; + } + TracingType::PegViz => { + println!( + "[PEG_TRACE] Attempting to match rule `{}` at {}:{}", + rule_name, + pos.line_col().0, + pos.line_col().1, + ); + } + } + + let result = func(self); + + match (tracing_type, result) { + (TracingType::None, Ok(new_state)) => Ok(new_state), + (TracingType::None, Err(new_state)) => Err(new_state), + (TracingType::Indented, Ok(mut new_state)) => { + new_state.tracing_depth -= 1; + println!( + "{}MATCHED `{}` at {}:{}", + " ".repeat(new_state.tracing_depth * new_state.tracing_config.spacing), + rule_name, + pos.line_col().0, + pos.line_col().1, + ); + + Ok(new_state) + } + (TracingType::Indented, Err(mut new_state)) => { + new_state.tracing_depth -= 1; + println!( + "{}FAILED `{}` at {}:{}", + " ".repeat(new_state.tracing_depth * new_state.tracing_config.spacing), + rule_name, + pos.line_col().0, + pos.line_col().1, + ); + + Err(new_state) + } + (TracingType::PegViz, Ok(new_state)) => { + println!( + "[PEG_TRACE] Matched rule `{}` at {}:{}", + rule_name, + pos.line_col().0, + pos.line_col().1, + ); + + Ok(new_state) + } + (TracingType::PegViz, Err(new_state)) => { + println!( + "[PEG_TRACE] Failed to match rule `{}` at {}:{}", + rule_name, + pos.line_col().0, + pos.line_col().1, + ); + + Err(new_state) + } + } + } + /// Get all parse attempts after process of parsing is finished. pub fn get_parse_attempts(&self) -> &ParseAttempts { &self.parse_attempts diff --git a/vm/src/lib.rs b/vm/src/lib.rs index c3e61d77..1efe75e4 100644 --- a/vm/src/lib.rs +++ b/vm/src/lib.rs @@ -18,7 +18,7 @@ use pest::error::Error; use pest::iterators::Pairs; -use pest::{unicode, Position}; +use pest::{unicode, Position, TracingConfig}; use pest::{Atomicity, MatchDir, ParseResult, ParserState}; use pest_meta::ast::RuleType; use pest_meta::optimizer::{OptimizedExpr, OptimizedRule}; @@ -69,7 +69,18 @@ impl Vm { rule: &'a str, input: &'a str, ) -> Result, Error<&str>> { - pest::state(input, |state| self.parse_rule(rule, state)) + self.parse_with_tracing(rule, input, Default::default()) + } + + /// Runs a parser rule on an input with tracing + #[allow(clippy::perf)] + pub fn parse_with_tracing<'a>( + &'a self, + rule: &'a str, + input: &'a str, + tracing_config: TracingConfig, + ) -> Result, Error<&str>> { + pest::state_with_tracing(input, tracing_config, |state| self.parse_rule(rule, state)) } #[allow(clippy::suspicious)] @@ -84,43 +95,77 @@ impl Vm { } } match rule { - "ANY" => return state.skip(1), - "EOI" => return state.rule("EOI", |state| state.end_of_input()), - "SOI" => return state.start_of_input(), - "PEEK" => return state.stack_peek(), - "PEEK_ALL" => return state.stack_match_peek(), - "POP" => return state.stack_pop(), - "POP_ALL" => return state.stack_match_pop(), - "DROP" => return state.stack_drop(), - "ASCII_DIGIT" => return state.match_range('0'..'9'), - "ASCII_NONZERO_DIGIT" => return state.match_range('1'..'9'), - "ASCII_BIN_DIGIT" => return state.match_range('0'..'1'), - "ASCII_OCT_DIGIT" => return state.match_range('0'..'7'), + "ANY" => return state.trace_wrapper(rule, false, false, |state| state.skip(1)), + "EOI" => { + return state.trace_wrapper(rule, false, false, |state| { + state.rule("EOI", |state| state.end_of_input()) + }) + } + "SOI" => { + return state.trace_wrapper(rule, false, false, |state| state.start_of_input()) + } + "PEEK" => return state.trace_wrapper(rule, false, false, |state| state.stack_peek()), + "PEEK_ALL" => { + return state.trace_wrapper(rule, false, false, |state| state.stack_match_peek()) + } + "POP" => return state.trace_wrapper(rule, false, false, |state| state.stack_pop()), + "POP_ALL" => { + return state.trace_wrapper(rule, false, false, |state| state.stack_match_pop()) + } + "DROP" => return state.trace_wrapper(rule, false, false, |state| state.stack_drop()), + "ASCII_DIGIT" => { + return state.trace_wrapper(rule, false, false, |state| state.match_range('0'..'9')) + } + "ASCII_NONZERO_DIGIT" => { + return state.trace_wrapper(rule, false, false, |state| state.match_range('1'..'9')) + } + "ASCII_BIN_DIGIT" => { + return state.trace_wrapper(rule, false, false, |state| state.match_range('0'..'1')) + } + "ASCII_OCT_DIGIT" => { + return state.trace_wrapper(rule, false, false, |state| state.match_range('0'..'7')) + } "ASCII_HEX_DIGIT" => { - return state - .match_range('0'..'9') - .or_else(|state| state.match_range('a'..'f')) - .or_else(|state| state.match_range('A'..'F')); + return state.trace_wrapper(rule, false, false, |state| { + state + .match_range('0'..'9') + .or_else(|state| state.match_range('a'..'f')) + .or_else(|state| state.match_range('A'..'F')) + }) + } + "ASCII_ALPHA_LOWER" => { + return state.trace_wrapper(rule, false, false, |state| state.match_range('a'..'z')) + } + "ASCII_ALPHA_UPPER" => { + return state.trace_wrapper(rule, false, false, |state| state.match_range('A'..'Z')) } - "ASCII_ALPHA_LOWER" => return state.match_range('a'..'z'), - "ASCII_ALPHA_UPPER" => return state.match_range('A'..'Z'), "ASCII_ALPHA" => { - return state - .match_range('a'..'z') - .or_else(|state| state.match_range('A'..'Z')); + return state.trace_wrapper(rule, false, false, |state| { + state + .match_range('a'..'z') + .or_else(|state| state.match_range('A'..'Z')) + }) } "ASCII_ALPHANUMERIC" => { - return state - .match_range('a'..'z') - .or_else(|state| state.match_range('A'..'Z')) - .or_else(|state| state.match_range('0'..'9')); + return state.trace_wrapper(rule, false, false, |state| { + state + .match_range('a'..'z') + .or_else(|state| state.match_range('A'..'Z')) + .or_else(|state| state.match_range('0'..'9')) + }) + } + "ASCII" => { + return state.trace_wrapper(rule, false, false, |state| { + state.match_range('\x00'..'\x7f') + }) } - "ASCII" => return state.match_range('\x00'..'\x7f'), "NEWLINE" => { - return state - .match_string("\n") - .or_else(|state| state.match_string("\r\n")) - .or_else(|state| state.match_string("\r")); + return state.trace_wrapper(rule, false, false, |state| { + state + .match_string("\n") + .or_else(|state| state.match_string("\r\n")) + .or_else(|state| state.match_string("\r")) + }) } _ => (), }; @@ -128,44 +173,64 @@ impl Vm { if let Some(rule) = self.rules.get(rule) { if rule.name == "WHITESPACE" || rule.name == "COMMENT" { match rule.ty { - RuleType::Normal => state.rule(&rule.name, |state| { - state.atomic(Atomicity::Atomic, |state| { - self.parse_expr(&rule.expr, state) + RuleType::Normal => state.trace_wrapper(&rule.name, true, false, |state| { + state.rule(&rule.name, |state| { + state.atomic(Atomicity::Atomic, |state| { + self.parse_expr(&rule.expr, state) + }) }) }), - RuleType::Silent => state.atomic(Atomicity::Atomic, |state| { - self.parse_expr(&rule.expr, state) - }), - RuleType::Atomic => state.rule(&rule.name, |state| { + RuleType::Silent => state.trace_wrapper(&rule.name, true, true, |state| { state.atomic(Atomicity::Atomic, |state| { self.parse_expr(&rule.expr, state) }) }), - RuleType::CompoundAtomic => state.atomic(Atomicity::CompoundAtomic, |state| { - state.rule(&rule.name, |state| self.parse_expr(&rule.expr, state)) + RuleType::Atomic => state.trace_wrapper(&rule.name, true, false, |state| { + state.rule(&rule.name, |state| { + state.atomic(Atomicity::Atomic, |state| { + self.parse_expr(&rule.expr, state) + }) + }) }), - RuleType::NonAtomic => state.atomic(Atomicity::Atomic, |state| { - state.rule(&rule.name, |state| self.parse_expr(&rule.expr, state)) + RuleType::CompoundAtomic => { + state.trace_wrapper(&rule.name, true, false, |state| { + state.atomic(Atomicity::CompoundAtomic, |state| { + state.rule(&rule.name, |state| self.parse_expr(&rule.expr, state)) + }) + }) + } + RuleType::NonAtomic => state.trace_wrapper(&rule.name, true, false, |state| { + state.atomic(Atomicity::Atomic, |state| { + state.rule(&rule.name, |state| self.parse_expr(&rule.expr, state)) + }) }), } } else { match rule.ty { - RuleType::Normal => { + RuleType::Normal => state.trace_wrapper(&rule.name, false, false, |state| { state.rule(&rule.name, move |state| self.parse_expr(&rule.expr, state)) - } - RuleType::Silent => self.parse_expr(&rule.expr, state), - RuleType::Atomic => state.rule(&rule.name, move |state| { - state.atomic(Atomicity::Atomic, move |state| { - self.parse_expr(&rule.expr, state) + }), + RuleType::Silent => state.trace_wrapper(&rule.name, false, true, |state| { + self.parse_expr(&rule.expr, state) + }), + RuleType::Atomic => state.trace_wrapper(&rule.name, false, false, |state| { + state.rule(&rule.name, move |state| { + state.atomic(Atomicity::Atomic, move |state| { + self.parse_expr(&rule.expr, state) + }) }) }), RuleType::CompoundAtomic => { - state.atomic(Atomicity::CompoundAtomic, move |state| { - state.rule(&rule.name, |state| self.parse_expr(&rule.expr, state)) + state.trace_wrapper(&rule.name, false, false, |state| { + state.atomic(Atomicity::CompoundAtomic, move |state| { + state.rule(&rule.name, |state| self.parse_expr(&rule.expr, state)) + }) }) } - RuleType::NonAtomic => state.atomic(Atomicity::NonAtomic, move |state| { - state.rule(&rule.name, |state| self.parse_expr(&rule.expr, state)) + RuleType::NonAtomic => state.trace_wrapper(&rule.name, false, false, |state| { + state.atomic(Atomicity::NonAtomic, move |state| { + state.rule(&rule.name, |state| self.parse_expr(&rule.expr, state)) + }) }), } } @@ -263,34 +328,66 @@ impl Vm { (false, false) => Ok(state), (true, false) => { if state.atomicity() == Atomicity::NonAtomic { - state.repeat(|state| self.parse_rule("WHITESPACE", state)) + // This is to match the output of the tracer in non-debugger mode, which includes a "skip" rule + state.trace_wrapper("skip", true, false, |state| { + state.repeat(|state| { + state.trace_wrapper("WHITESPACE", true, false, |state| { + self.parse_rule("WHITESPACE", state) + }) + }) + }) } else { Ok(state) } } (false, true) => { if state.atomicity() == Atomicity::NonAtomic { - state.repeat(|state| self.parse_rule("COMMENT", state)) + // This is to match the output of the tracer in non-debugger mode, which includes a "skip" rule + state.trace_wrapper("skip", true, false, |state| { + state.repeat(|state| { + state.trace_wrapper("COMMENT", true, false, |state| { + self.parse_rule("COMMENT", state) + }) + }) + }) } else { Ok(state) } } (true, true) => { if state.atomicity() == Atomicity::NonAtomic { - state.sequence(|state| { - state - .repeat(|state| self.parse_rule("WHITESPACE", state)) - .and_then(|state| { - state.repeat(|state| { - state.sequence(|state| { - self.parse_rule("COMMENT", state).and_then(|state| { - state.repeat(|state| { - self.parse_rule("WHITESPACE", state) - }) + // This is to match the output of the tracer in non-debugger mode, which includes a "skip" rule + state.trace_wrapper("skip", true, false, |state| { + state.sequence(|state| { + state + .repeat(|state| { + state.trace_wrapper("WHITESPACE", true, false, |state| { + self.parse_rule("WHITESPACE", state) + }) + }) + .and_then(|state| { + state.repeat(|state| { + state.sequence(|state| { + state + .trace_wrapper("COMMENT", true, false, |state| { + self.parse_rule("COMMENT", state) + }) + .and_then(|state| { + state.repeat(|state| { + state.trace_wrapper( + "WHITESPACE", + true, + false, + |state| { + self.parse_rule("WHITESPACE", state) + }, + ) + }) + }) }) }) }) - }) + }) }) } else { Ok(state)