From 38c074e67d8306e4635b8e9d365bf4c007c8d5b3 Mon Sep 17 00:00:00 2001 From: Brent Westbrook <36778786+ntBre@users.noreply.github.com> Date: Mon, 20 Oct 2025 09:03:13 -0400 Subject: [PATCH] Catch syntax errors in nested interpolations before Python 3.12 (#20949) Summary -- This PR fixes the issue I added in #20867 and noticed in #20930. Cases like this cause an error on any Python version: ```py f"{1:""}" ``` which gave me a false sense of security before. Cases like this are still invalid only before 3.12 and weren't flagged after the changes in #20867: ```py f'{1: abcd "{'aa'}" }' # ^ reused quote f'{1: abcd "{"\n"}" }' # ^ backslash ``` I didn't recognize these as nested interpolations that also need to be checked for invalid expressions, so filtering out the whole format spec wasn't quite right. And `elements.interpolations()` only iterates over the outermost interpolations, not the nested ones. There's basically no code change in this PR, I just moved the existing check from `parse_interpolated_string`, which parses the entire string, to `parse_interpolated_element`. This kind of seems more natural anyway and avoids having to try to recursively visit nested elements after the fact in `parse_interpolated_string`. So viewing the diff with something like ``` git diff --color-moved --ignore-space-change --color-moved-ws=allow-indentation-change main ``` should make this more clear. Test Plan -- New tests --- .../err/pep701_nested_interpolation_py311.py | 4 + .../src/parser/expression.rs | 196 +++++++-------- ...@pep701_nested_interpolation_py311.py.snap | 231 ++++++++++++++++++ 3 files changed, 335 insertions(+), 96 deletions(-) create mode 100644 crates/ruff_python_parser/resources/inline/err/pep701_nested_interpolation_py311.py create mode 100644 crates/ruff_python_parser/tests/snapshots/invalid_syntax@pep701_nested_interpolation_py311.py.snap diff --git a/crates/ruff_python_parser/resources/inline/err/pep701_nested_interpolation_py311.py b/crates/ruff_python_parser/resources/inline/err/pep701_nested_interpolation_py311.py new file mode 100644 index 0000000000..3fbddf2285 --- /dev/null +++ b/crates/ruff_python_parser/resources/inline/err/pep701_nested_interpolation_py311.py @@ -0,0 +1,4 @@ +# parse_options: {"target-version": "3.11"} +# nested interpolations also need to be checked +f'{1: abcd "{'aa'}" }' +f'{1: abcd "{"\n"}" }' diff --git a/crates/ruff_python_parser/src/parser/expression.rs b/crates/ruff_python_parser/src/parser/expression.rs index 6920e7cdd1..2ae786ce77 100644 --- a/crates/ruff_python_parser/src/parser/expression.rs +++ b/crates/ruff_python_parser/src/parser/expression.rs @@ -1539,103 +1539,9 @@ impl<'src> Parser<'src> { flags = flags.with_unclosed(true); } - // test_ok pep701_f_string_py312 - // # parse_options: {"target-version": "3.12"} - // f'Magic wand: { bag['wand'] }' # nested quotes - // f"{'\n'.join(a)}" # escape sequence - // f'''A complex trick: { - // bag['bag'] # comment - // }''' - // f"{f"{f"{f"{f"{f"{1+1}"}"}"}"}"}" # arbitrary nesting - // f"{f'''{"nested"} inner'''} outer" # nested (triple) quotes - // f"test {a \ - // } more" # line continuation - - // test_ok pep750_t_string_py314 - // # parse_options: {"target-version": "3.14"} - // t'Magic wand: { bag['wand'] }' # nested quotes - // t"{'\n'.join(a)}" # escape sequence - // t'''A complex trick: { - // bag['bag'] # comment - // }''' - // t"{t"{t"{t"{t"{t"{1+1}"}"}"}"}"}" # arbitrary nesting - // t"{t'''{"nested"} inner'''} outer" # nested (triple) quotes - // t"test {a \ - // } more" # line continuation - - // test_ok pep701_f_string_py311 - // # parse_options: {"target-version": "3.11"} - // f"outer {'# not a comment'}" - // f'outer {x:{"# not a comment"} }' - // f"""{f'''{f'{"# not a comment"}'}'''}""" - // f"""{f'''# before expression {f'# aro{f"#{1+1}#"}und #'}'''} # after expression""" - // f"escape outside of \t {expr}\n" - // f"test\"abcd" - // f"{1:\x64}" # escapes are valid in the format spec - // f"{1:\"d\"}" # this also means that escaped outer quotes are valid - - // test_err pep701_f_string_py311 - // # parse_options: {"target-version": "3.11"} - // f'Magic wand: { bag['wand'] }' # nested quotes - // f"{'\n'.join(a)}" # escape sequence - // f'''A complex trick: { - // bag['bag'] # comment - // }''' - // f"{f"{f"{f"{f"{f"{1+1}"}"}"}"}"}" # arbitrary nesting - // f"{f'''{"nested"} inner'''} outer" # nested (triple) quotes - // f"test {a \ - // } more" # line continuation - // f"""{f"""{x}"""}""" # mark the whole triple quote - // f"{'\n'.join(['\t', '\v', '\r'])}" # multiple escape sequences, multiple errors - - // test_err nested_quote_in_format_spec_py312 - // # parse_options: {"target-version": "3.12"} - // f"{1:""}" # this is a ParseError on all versions - - // test_ok non_nested_quote_in_format_spec_py311 - // # parse_options: {"target-version": "3.11"} - // f"{1:''}" # but this is okay on all versions - let range = self.node_range(start); - - if !self.options.target_version.supports_pep_701() - && matches!(kind, InterpolatedStringKind::FString) - { - let quote_bytes = flags.quote_str().as_bytes(); - let quote_len = flags.quote_len(); - for expr in elements.interpolations() { - // We need to check the whole expression range, including any leading or trailing - // debug text, but exclude the format spec, where escapes and escaped, reused quotes - // are allowed. - let range = expr - .format_spec - .as_ref() - .map(|format_spec| TextRange::new(expr.start(), format_spec.start())) - .unwrap_or(expr.range); - for slash_position in memchr::memchr_iter(b'\\', self.source[range].as_bytes()) { - let slash_position = TextSize::try_from(slash_position).unwrap(); - self.add_unsupported_syntax_error( - UnsupportedSyntaxErrorKind::Pep701FString(FStringKind::Backslash), - TextRange::at(range.start() + slash_position, '\\'.text_len()), - ); - } - - if let Some(quote_position) = - memchr::memmem::find(self.source[range].as_bytes(), quote_bytes) - { - let quote_position = TextSize::try_from(quote_position).unwrap(); - self.add_unsupported_syntax_error( - UnsupportedSyntaxErrorKind::Pep701FString(FStringKind::NestedQuote), - TextRange::at(range.start() + quote_position, quote_len), - ); - } - } - - self.check_fstring_comments(range); - } - InterpolatedStringData { elements, - range, + range: self.node_range(start), flags, } } @@ -1921,12 +1827,110 @@ impl<'src> Parser<'src> { ); } + // test_ok pep701_f_string_py312 + // # parse_options: {"target-version": "3.12"} + // f'Magic wand: { bag['wand'] }' # nested quotes + // f"{'\n'.join(a)}" # escape sequence + // f'''A complex trick: { + // bag['bag'] # comment + // }''' + // f"{f"{f"{f"{f"{f"{1+1}"}"}"}"}"}" # arbitrary nesting + // f"{f'''{"nested"} inner'''} outer" # nested (triple) quotes + // f"test {a \ + // } more" # line continuation + + // test_ok pep750_t_string_py314 + // # parse_options: {"target-version": "3.14"} + // t'Magic wand: { bag['wand'] }' # nested quotes + // t"{'\n'.join(a)}" # escape sequence + // t'''A complex trick: { + // bag['bag'] # comment + // }''' + // t"{t"{t"{t"{t"{t"{1+1}"}"}"}"}"}" # arbitrary nesting + // t"{t'''{"nested"} inner'''} outer" # nested (triple) quotes + // t"test {a \ + // } more" # line continuation + + // test_ok pep701_f_string_py311 + // # parse_options: {"target-version": "3.11"} + // f"outer {'# not a comment'}" + // f'outer {x:{"# not a comment"} }' + // f"""{f'''{f'{"# not a comment"}'}'''}""" + // f"""{f'''# before expression {f'# aro{f"#{1+1}#"}und #'}'''} # after expression""" + // f"escape outside of \t {expr}\n" + // f"test\"abcd" + // f"{1:\x64}" # escapes are valid in the format spec + // f"{1:\"d\"}" # this also means that escaped outer quotes are valid + + // test_err pep701_f_string_py311 + // # parse_options: {"target-version": "3.11"} + // f'Magic wand: { bag['wand'] }' # nested quotes + // f"{'\n'.join(a)}" # escape sequence + // f'''A complex trick: { + // bag['bag'] # comment + // }''' + // f"{f"{f"{f"{f"{f"{1+1}"}"}"}"}"}" # arbitrary nesting + // f"{f'''{"nested"} inner'''} outer" # nested (triple) quotes + // f"test {a \ + // } more" # line continuation + // f"""{f"""{x}"""}""" # mark the whole triple quote + // f"{'\n'.join(['\t', '\v', '\r'])}" # multiple escape sequences, multiple errors + + // test_err pep701_nested_interpolation_py311 + // # parse_options: {"target-version": "3.11"} + // # nested interpolations also need to be checked + // f'{1: abcd "{'aa'}" }' + // f'{1: abcd "{"\n"}" }' + + // test_err nested_quote_in_format_spec_py312 + // # parse_options: {"target-version": "3.12"} + // f"{1:""}" # this is a ParseError on all versions + + // test_ok non_nested_quote_in_format_spec_py311 + // # parse_options: {"target-version": "3.11"} + // f"{1:''}" # but this is okay on all versions + let range = self.node_range(start); + + if !self.options.target_version.supports_pep_701() + && matches!(string_kind, InterpolatedStringKind::FString) + { + // We need to check the whole expression range, including any leading or trailing + // debug text, but exclude the format spec, where escapes and escaped, reused quotes + // are allowed. + let range = format_spec + .as_ref() + .map(|format_spec| TextRange::new(range.start(), format_spec.start())) + .unwrap_or(range); + + let quote_bytes = flags.quote_str().as_bytes(); + let quote_len = flags.quote_len(); + for slash_position in memchr::memchr_iter(b'\\', self.source[range].as_bytes()) { + let slash_position = TextSize::try_from(slash_position).unwrap(); + self.add_unsupported_syntax_error( + UnsupportedSyntaxErrorKind::Pep701FString(FStringKind::Backslash), + TextRange::at(range.start() + slash_position, '\\'.text_len()), + ); + } + + if let Some(quote_position) = + memchr::memmem::find(self.source[range].as_bytes(), quote_bytes) + { + let quote_position = TextSize::try_from(quote_position).unwrap(); + self.add_unsupported_syntax_error( + UnsupportedSyntaxErrorKind::Pep701FString(FStringKind::NestedQuote), + TextRange::at(range.start() + quote_position, quote_len), + ); + } + + self.check_fstring_comments(range); + } + ast::InterpolatedElement { expression: Box::new(value.expr), debug_text, conversion, format_spec, - range: self.node_range(start), + range, node_index: AtomicNodeIndex::NONE, } } diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@pep701_nested_interpolation_py311.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@pep701_nested_interpolation_py311.py.snap new file mode 100644 index 0000000000..5699059e91 --- /dev/null +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@pep701_nested_interpolation_py311.py.snap @@ -0,0 +1,231 @@ +--- +source: crates/ruff_python_parser/tests/fixtures.rs +input_file: crates/ruff_python_parser/resources/inline/err/pep701_nested_interpolation_py311.py +--- +## AST + +``` +Module( + ModModule { + node_index: NodeIndex(None), + range: 0..138, + body: [ + Expr( + StmtExpr { + node_index: NodeIndex(None), + range: 92..114, + value: FString( + ExprFString { + node_index: NodeIndex(None), + range: 92..114, + value: FStringValue { + inner: Single( + FString( + FString { + range: 92..114, + node_index: NodeIndex(None), + elements: [ + Interpolation( + InterpolatedElement { + range: 94..113, + node_index: NodeIndex(None), + expression: NumberLiteral( + ExprNumberLiteral { + node_index: NodeIndex(None), + range: 95..96, + value: Int( + 1, + ), + }, + ), + debug_text: None, + conversion: None, + format_spec: Some( + InterpolatedStringFormatSpec { + range: 97..112, + node_index: NodeIndex(None), + elements: [ + Literal( + InterpolatedStringLiteralElement { + range: 97..104, + node_index: NodeIndex(None), + value: " abcd \"", + }, + ), + Interpolation( + InterpolatedElement { + range: 104..110, + node_index: NodeIndex(None), + expression: StringLiteral( + ExprStringLiteral { + node_index: NodeIndex(None), + range: 105..109, + value: StringLiteralValue { + inner: Single( + StringLiteral { + range: 105..109, + node_index: NodeIndex(None), + value: "aa", + flags: StringLiteralFlags { + quote_style: Single, + prefix: Empty, + triple_quoted: false, + unclosed: false, + }, + }, + ), + }, + }, + ), + debug_text: None, + conversion: None, + format_spec: None, + }, + ), + Literal( + InterpolatedStringLiteralElement { + range: 110..112, + node_index: NodeIndex(None), + value: "\" ", + }, + ), + ], + }, + ), + }, + ), + ], + flags: FStringFlags { + quote_style: Single, + prefix: Regular, + triple_quoted: false, + unclosed: false, + }, + }, + ), + ), + }, + }, + ), + }, + ), + Expr( + StmtExpr { + node_index: NodeIndex(None), + range: 115..137, + value: FString( + ExprFString { + node_index: NodeIndex(None), + range: 115..137, + value: FStringValue { + inner: Single( + FString( + FString { + range: 115..137, + node_index: NodeIndex(None), + elements: [ + Interpolation( + InterpolatedElement { + range: 117..136, + node_index: NodeIndex(None), + expression: NumberLiteral( + ExprNumberLiteral { + node_index: NodeIndex(None), + range: 118..119, + value: Int( + 1, + ), + }, + ), + debug_text: None, + conversion: None, + format_spec: Some( + InterpolatedStringFormatSpec { + range: 120..135, + node_index: NodeIndex(None), + elements: [ + Literal( + InterpolatedStringLiteralElement { + range: 120..127, + node_index: NodeIndex(None), + value: " abcd \"", + }, + ), + Interpolation( + InterpolatedElement { + range: 127..133, + node_index: NodeIndex(None), + expression: StringLiteral( + ExprStringLiteral { + node_index: NodeIndex(None), + range: 128..132, + value: StringLiteralValue { + inner: Single( + StringLiteral { + range: 128..132, + node_index: NodeIndex(None), + value: "\n", + flags: StringLiteralFlags { + quote_style: Double, + prefix: Empty, + triple_quoted: false, + unclosed: false, + }, + }, + ), + }, + }, + ), + debug_text: None, + conversion: None, + format_spec: None, + }, + ), + Literal( + InterpolatedStringLiteralElement { + range: 133..135, + node_index: NodeIndex(None), + value: "\" ", + }, + ), + ], + }, + ), + }, + ), + ], + flags: FStringFlags { + quote_style: Single, + prefix: Regular, + triple_quoted: false, + unclosed: false, + }, + }, + ), + ), + }, + }, + ), + }, + ), + ], + }, +) +``` +## Unsupported Syntax Errors + + | +1 | # parse_options: {"target-version": "3.11"} +2 | # nested interpolations also need to be checked +3 | f'{1: abcd "{'aa'}" }' + | ^ Syntax Error: Cannot reuse outer quote character in f-strings on Python 3.11 (syntax was added in Python 3.12) +4 | f'{1: abcd "{"\n"}" }' + | + + + | +2 | # nested interpolations also need to be checked +3 | f'{1: abcd "{'aa'}" }' +4 | f'{1: abcd "{"\n"}" }' + | ^ Syntax Error: Cannot use an escape sequence (backslash) in f-strings on Python 3.11 (syntax was added in Python 3.12) + |