diff --git a/crates/ruff_python_formatter/tests/fixtures.rs b/crates/ruff_python_formatter/tests/fixtures.rs index 75bef10def55c..7e83d969cb9c0 100644 --- a/crates/ruff_python_formatter/tests/fixtures.rs +++ b/crates/ruff_python_formatter/tests/fixtures.rs @@ -1,20 +1,22 @@ +use crate::normalizer::Normalizer; +use itertools::Itertools; +use ruff_formatter::FormatOptions; +use ruff_python_ast::comparable::ComparableMod; +use ruff_python_formatter::{format_module_source, format_range, PreviewMode, PyFormatOptions}; +use ruff_python_parser::{parse, ParseOptions, UnsupportedSyntaxError}; +use ruff_source_file::{LineIndex, OneIndexed}; +use ruff_text_size::{Ranged, TextRange, TextSize}; +use rustc_hash::FxHashMap; +use similar::TextDiff; use std::borrow::Cow; +use std::collections::hash_map::Entry; use std::fmt::{Formatter, Write}; +use std::hash::{DefaultHasher, Hash, Hasher}; use std::io::BufReader; use std::ops::Range; use std::path::Path; use std::{fmt, fs}; -use similar::TextDiff; - -use crate::normalizer::Normalizer; -use ruff_formatter::FormatOptions; -use ruff_python_ast::comparable::ComparableMod; -use ruff_python_formatter::{format_module_source, format_range, PreviewMode, PyFormatOptions}; -use ruff_python_parser::{parse, ParseOptions}; -use ruff_source_file::{LineIndex, OneIndexed}; -use ruff_text_size::{TextRange, TextSize}; - mod normalizer; #[test] @@ -379,7 +381,7 @@ Formatted twice: } } -/// Ensure that formatting doesn't change the AST. +/// Ensure that formatting doesn't change the AST and doesn't introduce any new unsupported syntax errors. /// /// Like Black, there are a few exceptions to this "invariant" which are encoded in /// [`NormalizedMod`] and related structs. Namely, formatting can change indentation within strings, @@ -393,16 +395,53 @@ fn ensure_unchanged_ast( let source_type = options.source_type(); // Parse the unformatted code. - let mut unformatted_ast = parse(unformatted_code, ParseOptions::from(source_type)) - .expect("Unformatted code to be valid syntax") - .into_syntax(); + let unformatted_parsed = parse( + unformatted_code, + ParseOptions::from(source_type).with_target_version(options.target_version()), + ) + .expect("Unformatted code to be valid syntax"); + + let unformatted_unsupported_syntax_errors = + collect_unsupported_syntax_errors(unformatted_parsed.unsupported_syntax_errors()); + let mut unformatted_ast = unformatted_parsed.into_syntax(); + Normalizer.visit_module(&mut unformatted_ast); let unformatted_ast = ComparableMod::from(&unformatted_ast); // Parse the formatted code. - let mut formatted_ast = parse(formatted_code, ParseOptions::from(source_type)) - .expect("Formatted code to be valid syntax") - .into_syntax(); + let formatted_parsed = parse( + formatted_code, + ParseOptions::from(source_type).with_target_version(options.target_version()), + ) + .expect("Formatted code to be valid syntax"); + + // Assert that there are no new unsupported syntax errors + let mut formatted_unsupported_syntax_errors = + collect_unsupported_syntax_errors(formatted_parsed.unsupported_syntax_errors()); + + formatted_unsupported_syntax_errors + .retain(|fingerprint, _| !unformatted_unsupported_syntax_errors.contains_key(fingerprint)); + + if !formatted_unsupported_syntax_errors.is_empty() { + let index = LineIndex::from_source_text(formatted_code); + panic!( + "Formatted code `{}` introduced new unsupported syntax errors:\n---\n{}\n---", + input_path.display(), + formatted_unsupported_syntax_errors + .into_values() + .map(|error| { + let location = index.source_location(error.start(), formatted_code); + format!( + "{row}:{col} {error}", + row = location.row, + col = location.column + ) + }) + .join("\n") + ); + } + + let mut formatted_ast = formatted_parsed.into_syntax(); Normalizer.visit_module(&mut formatted_ast); let formatted_ast = ComparableMod::from(&formatted_ast); @@ -492,3 +531,49 @@ source_type = {source_type:?}"#, ) } } + +/// Collects the unsupported syntax errors and assigns a unique hash to each error. +fn collect_unsupported_syntax_errors( + errors: &[UnsupportedSyntaxError], +) -> FxHashMap { + let mut collected = FxHashMap::default(); + + for error in errors { + let mut error_fingerprint = fingerprint_unsupported_syntax_error(error, 0); + + // Make sure that we do not get a fingerprint that is already in use + // by adding in the previously generated one. + loop { + match collected.entry(error_fingerprint) { + Entry::Occupied(_) => { + error_fingerprint = + fingerprint_unsupported_syntax_error(error, error_fingerprint); + } + Entry::Vacant(entry) => { + entry.insert(error.clone()); + break; + } + } + } + } + + collected +} + +fn fingerprint_unsupported_syntax_error(error: &UnsupportedSyntaxError, salt: u64) -> u64 { + let mut hasher = DefaultHasher::new(); + + let UnsupportedSyntaxError { + kind, + target_version, + // Don't hash the range because the location between the formatted and unformatted code + // is likely to be different + range: _, + } = error; + + salt.hash(&mut hasher); + kind.hash(&mut hasher); + target_version.hash(&mut hasher); + + hasher.finish() +} diff --git a/crates/ruff_python_parser/src/error.rs b/crates/ruff_python_parser/src/error.rs index 1e8934d3d1235..6d0f6c38452c8 100644 --- a/crates/ruff_python_parser/src/error.rs +++ b/crates/ruff_python_parser/src/error.rs @@ -1,7 +1,7 @@ use std::fmt::{self, Display}; use ruff_python_ast::PythonVersion; -use ruff_text_size::TextRange; +use ruff_text_size::{Ranged, TextRange}; use crate::TokenKind; @@ -439,14 +439,20 @@ pub struct UnsupportedSyntaxError { pub target_version: PythonVersion, } +impl Ranged for UnsupportedSyntaxError { + fn range(&self) -> TextRange { + self.range + } +} + /// The type of tuple unpacking for [`UnsupportedSyntaxErrorKind::StarTuple`]. -#[derive(Debug, PartialEq, Clone, Copy)] +#[derive(Debug, PartialEq, Eq, Hash, Clone, Copy)] pub enum StarTupleKind { Return, Yield, } -#[derive(Debug, PartialEq, Clone, Copy)] +#[derive(Debug, PartialEq, Eq, Hash, Clone, Copy)] pub enum UnsupportedSyntaxErrorKind { Match, Walrus,