Skip to content

Commit

Permalink
[syntax-errors] Tuple unpacking in return and yield before Python…
Browse files Browse the repository at this point in the history 
… 3.8 (#16485)

Summary
--

Checks for tuple unpacking in `return` and `yield` statements before
Python 3.8, as described [here].

Test Plan
--
Inline tests.

[here]: python/cpython#76298
  • Loading branch information
@ntBre
ntBre authored Mar 6, 2025
1 parent 0a627ef commit 6c14225
Show file tree
Hide file tree
Showing 15 changed files with 1,217 additions and 8 deletions.
3 changes: 3 additions & 0 deletions crates/ruff_python_parser/resources/inline/err/iter_unpack_return_py37.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
# parse_options: {"target-version": "3.7"}
rest = (4, 5, 6)
def f(): return 1, 2, 3, *rest
4 changes: 4 additions & 0 deletions crates/ruff_python_parser/resources/inline/err/iter_unpack_yield_py37.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
# parse_options: {"target-version": "3.7"}
rest = (4, 5, 6)
def g(): yield 1, 2, 3, *rest
def h(): yield 1, (yield 2, *rest), 3
3 changes: 3 additions & 0 deletions crates/ruff_python_parser/resources/inline/ok/iter_unpack_return_py37.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
# parse_options: {"target-version": "3.7"}
rest = (4, 5, 6)
def f(): return (1, 2, 3, *rest)
3 changes: 3 additions & 0 deletions crates/ruff_python_parser/resources/inline/ok/iter_unpack_return_py38.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
# parse_options: {"target-version": "3.8"}
rest = (4, 5, 6)
def f(): return 1, 2, 3, *rest
3 changes: 3 additions & 0 deletions crates/ruff_python_parser/resources/inline/ok/iter_unpack_yield_py37.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
# parse_options: {"target-version": "3.7"}
rest = (4, 5, 6)
def g(): yield (1, 2, 3, *rest)
4 changes: 4 additions & 0 deletions crates/ruff_python_parser/resources/inline/ok/iter_unpack_yield_py38.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
# parse_options: {"target-version": "3.8"}
rest = (4, 5, 6)
def g(): yield 1, 2, 3, *rest
def h(): yield 1, (yield 2, *rest), 3
66 changes: 66 additions & 0 deletions crates/ruff_python_parser/src/error.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -444,11 +444,68 @@ pub struct UnsupportedSyntaxError {
pub target_version: PythonVersion,
}

/// The type of tuple unpacking for [`UnsupportedSyntaxErrorKind::StarTuple`].
#[derive(Debug, PartialEq, Clone, Copy)]
pub enum StarTupleKind {
Return,
Yield,
}

#[derive(Debug, PartialEq, Clone, Copy)]
pub enum UnsupportedSyntaxErrorKind {
Match,
Walrus,
ExceptStar,

/// Represents the use of unparenthesized tuple unpacking in a `return` statement or `yield`
/// expression before Python 3.8.
///
/// ## Examples
///
/// Before Python 3.8, this syntax was allowed:
///
/// ```python
/// rest = (4, 5, 6)
///
/// def f():
/// t = 1, 2, 3, *rest
/// return t
///
/// def g():
/// t = 1, 2, 3, *rest
/// yield t
/// ```
///
/// But this was not:
///
/// ```python
/// rest = (4, 5, 6)
///
/// def f():
/// return 1, 2, 3, *rest
///
/// def g():
/// yield 1, 2, 3, *rest
/// ```
///
/// Instead, parentheses were required in the `return` and `yield` cases:
///
/// ```python
/// rest = (4, 5, 6)
///
/// def f():
/// return (1, 2, 3, *rest)
///
/// def g():
/// yield (1, 2, 3, *rest)
/// ```
///
/// This was reported in [BPO 32117] and updated in Python 3.8 to allow the unparenthesized
/// form.
///
/// [BPO 32117]: https://github.com/python/cpython/issues/76298
StarTuple(StarTupleKind),

/// Represents the use of a "relaxed" [PEP 614] decorator before Python 3.9.
///
/// ## Examples
Expand Down Expand Up @@ -480,6 +537,7 @@ pub enum UnsupportedSyntaxErrorKind {
/// [`dotted_name`]: https://docs.python.org/3.8/reference/compound_stmts.html#grammar-token-dotted-name
/// [decorator grammar]: https://docs.python.org/3/reference/compound_stmts.html#grammar-token-python-grammar-decorator
RelaxedDecorator,

/// Represents the use of a [PEP 570] positional-only parameter before Python 3.8.
///
/// ## Examples
Expand All @@ -506,6 +564,7 @@ pub enum UnsupportedSyntaxErrorKind {
///
/// [PEP 570]: https://peps.python.org/pep-0570/
PositionalOnlyParameter,

/// Represents the use of a [type parameter list] before Python 3.12.
///
/// ## Examples
Expand Down Expand Up @@ -544,6 +603,12 @@ impl Display for UnsupportedSyntaxError {
UnsupportedSyntaxErrorKind::Match => "Cannot use `match` statement",
UnsupportedSyntaxErrorKind::Walrus => "Cannot use named assignment expression (`:=`)",
UnsupportedSyntaxErrorKind::ExceptStar => "Cannot use `except*`",
UnsupportedSyntaxErrorKind::StarTuple(StarTupleKind::Return) => {
"Cannot use iterable unpacking in return statements"
}
UnsupportedSyntaxErrorKind::StarTuple(StarTupleKind::Yield) => {
"Cannot use iterable unpacking in yield expressions"
}
UnsupportedSyntaxErrorKind::RelaxedDecorator => "Unsupported expression in decorators",
UnsupportedSyntaxErrorKind::PositionalOnlyParameter => {
"Cannot use positional-only parameter separator"
Expand All @@ -570,6 +635,7 @@ impl UnsupportedSyntaxErrorKind {
UnsupportedSyntaxErrorKind::Match => PythonVersion::PY310,
UnsupportedSyntaxErrorKind::Walrus => PythonVersion::PY38,
UnsupportedSyntaxErrorKind::ExceptStar => PythonVersion::PY311,
UnsupportedSyntaxErrorKind::StarTuple(_) => PythonVersion::PY38,
UnsupportedSyntaxErrorKind::RelaxedDecorator => PythonVersion::PY39,
UnsupportedSyntaxErrorKind::PositionalOnlyParameter => PythonVersion::PY38,
UnsupportedSyntaxErrorKind::TypeParameterList => PythonVersion::PY312,
Expand Down
26 changes: 22 additions & 4 deletions crates/ruff_python_parser/src/parser/expression.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ use ruff_python_ast::{
};
use ruff_text_size::{Ranged, TextLen, TextRange, TextSize};

use crate::error::StarTupleKind;
use crate::parser::progress::ParserProgress;
use crate::parser::{helpers, FunctionKind, Parser};
use crate::string::{parse_fstring_literal_element, parse_string_literal, StringType};
Expand Down Expand Up @@ -2089,10 +2090,27 @@ impl<'src> Parser<'src> {
}

let value = self.at_expr().then(|| {
Box::new(
self.parse_expression_list(ExpressionContext::starred_bitwise_or())
.expr,
)
let parsed_expr = self.parse_expression_list(ExpressionContext::starred_bitwise_or());

// test_ok iter_unpack_yield_py37
// # parse_options: {"target-version": "3.7"}
// rest = (4, 5, 6)
// def g(): yield (1, 2, 3, *rest)

// test_ok iter_unpack_yield_py38
// # parse_options: {"target-version": "3.8"}
// rest = (4, 5, 6)
// def g(): yield 1, 2, 3, *rest
// def h(): yield 1, (yield 2, *rest), 3

// test_err iter_unpack_yield_py37
// # parse_options: {"target-version": "3.7"}
// rest = (4, 5, 6)
// def g(): yield 1, 2, 3, *rest
// def h(): yield 1, (yield 2, *rest), 3
self.check_tuple_unpacking(&parsed_expr, StarTupleKind::Yield);

Box::new(parsed_expr.expr)
});

Expr::Yield(ast::ExprYield {
Expand Down
51 changes: 47 additions & 4 deletions crates/ruff_python_parser/src/parser/statement.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ use ruff_python_ast::{
};
use ruff_text_size::{Ranged, TextRange, TextSize};

use crate::error::StarTupleKind;
use crate::parser::expression::{ParsedExpr, EXPR_SET};
use crate::parser::progress::ParserProgress;
use crate::parser::{
Expand Down Expand Up @@ -389,10 +390,25 @@ impl<'src> Parser<'src> {
// return x := 1
// return *x and y
let value = self.at_expr().then(|| {
Box::new(
self.parse_expression_list(ExpressionContext::starred_bitwise_or())
.expr,
)
let parsed_expr = self.parse_expression_list(ExpressionContext::starred_bitwise_or());

// test_ok iter_unpack_return_py37
// # parse_options: {"target-version": "3.7"}
// rest = (4, 5, 6)
// def f(): return (1, 2, 3, *rest)

// test_ok iter_unpack_return_py38
// # parse_options: {"target-version": "3.8"}
// rest = (4, 5, 6)
// def f(): return 1, 2, 3, *rest

// test_err iter_unpack_return_py37
// # parse_options: {"target-version": "3.7"}
// rest = (4, 5, 6)
// def f(): return 1, 2, 3, *rest
self.check_tuple_unpacking(&parsed_expr, StarTupleKind::Return);

Box::new(parsed_expr.expr)
});

ast::StmtReturn {
Expand All @@ -401,6 +417,33 @@ impl<'src> Parser<'src> {
}
}

/// Report [`UnsupportedSyntaxError`]s for each starred element in `expr` if it is an
/// unparenthesized tuple.
///
/// This method can be used to check for tuple unpacking in return and yield statements, which
/// are only allowed in Python 3.8 and later: <https://github.com/python/cpython/issues/76298>.
pub(crate) fn check_tuple_unpacking(&mut self, expr: &Expr, kind: StarTupleKind) {
let kind = UnsupportedSyntaxErrorKind::StarTuple(kind);
if self.options.target_version >= kind.minimum_version() {
return;
}

let Expr::Tuple(ast::ExprTuple {
elts,
parenthesized: false,
..
}) = expr
else {
return;
};

for elt in elts {
if elt.is_starred_expr() {
self.add_unsupported_syntax_error(kind, elt.range());
}
}
}

/// Parses a `raise` statement.
///
/// # Panics
Expand Down
146 changes: 146 additions & 0 deletions crates/ruff_python_parser/tests/snapshots/invalid_syntax@iter_unpack_return_py37.py.snap
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,146 @@
---
source: crates/ruff_python_parser/tests/fixtures.rs
input_file: crates/ruff_python_parser/resources/inline/err/iter_unpack_return_py37.py
---
## AST

```
Module(
ModModule {
range: 0..91,
body: [
Assign(
StmtAssign {
range: 43..59,
targets: [
Name(
ExprName {
range: 43..47,
id: Name("rest"),
ctx: Store,
},
),
],
value: Tuple(
ExprTuple {
range: 50..59,
elts: [
NumberLiteral(
ExprNumberLiteral {
range: 51..52,
value: Int(
4,
),
},
),
NumberLiteral(
ExprNumberLiteral {
range: 54..55,
value: Int(
5,
),
},
),
NumberLiteral(
ExprNumberLiteral {
range: 57..58,
value: Int(
6,
),
},
),
],
ctx: Load,
parenthesized: true,
},
),
},
),
FunctionDef(
StmtFunctionDef {
range: 60..90,
is_async: false,
decorator_list: [],
name: Identifier {
id: Name("f"),
range: 64..65,
},
type_params: None,
parameters: Parameters {
range: 65..67,
posonlyargs: [],
args: [],
vararg: None,
kwonlyargs: [],
kwarg: None,
},
returns: None,
body: [
Return(
StmtReturn {
range: 69..90,
value: Some(
Tuple(
ExprTuple {
range: 76..90,
elts: [
NumberLiteral(
ExprNumberLiteral {
range: 76..77,
value: Int(
1,
),
},
),
NumberLiteral(
ExprNumberLiteral {
range: 79..80,
value: Int(
2,
),
},
),
NumberLiteral(
ExprNumberLiteral {
range: 82..83,
value: Int(
3,
),
},
),
Starred(
ExprStarred {
range: 85..90,
value: Name(
ExprName {
range: 86..90,
id: Name("rest"),
ctx: Load,
},
),
ctx: Load,
},
),
],
ctx: Load,
parenthesized: false,
},
),
),
},
),
],
},
),
],
},
)
```
## Unsupported Syntax Errors

|
1 | # parse_options: {"target-version": "3.7"}
2 | rest = (4, 5, 6)
3 | def f(): return 1, 2, 3, *rest
| ^^^^^ Syntax Error: Cannot use iterable unpacking in return statements on Python 3.7 (syntax was added in Python 3.8)
|
Loading

0 comments on commit 6c14225

Please sign in to comment.