mirrors/RustPython-Parser
1
0
Fork 0
mirror of https://github.com/RustPython/Parser.git synced 2025-07-08 05:35:22 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'RustPython:main' into main

Browse source
This commit is contained in:
Micha Reiser 2023-05-25 12:28:15 +02:00
commit 7a3eedbf6f
No known key found for this signature in database
29 changed files with 9170 additions and 8810 deletions
Download patch file Download diff file Expand all files Collapse all files

1
Cargo.toml
View file

@ -12,6 +12,7 @@ include = ["LICENSE", "Cargo.toml", "src/**/*.rs"]
resolver = "2"
members = [
"ast", "core", "format", "literal", "parser",
"ast-pyo3",
"ruff_text_size", "ruff_source_location",
]

72
ast-pyo3/.gitignore vendored Normal file
View file

@ -0,0 +1,72 @@
/target
# Byte-compiled / optimized / DLL files
__pycache__/
.pytest_cache/
*.py[cod]
# C extensions
*.so
# Distribution / packaging
.Python
.venv/
env/
bin/
build/
develop-eggs/
dist/
eggs/
lib/
lib64/
parts/
sdist/
var/
include/
man/
venv/
*.egg-info/
.installed.cfg
*.egg
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
pip-selfcheck.json
# Unit test / coverage reports
htmlcov/
.tox/
.coverage
.cache
nosetests.xml
coverage.xml
# Translations
*.mo
# Mr Developer
.mr.developer.cfg
.project
.pydevproject
# Rope
.ropeproject
# Django stuff:
*.log
*.pot
.DS_Store
# Sphinx documentation
docs/_build/
# PyCharm
.idea/
# VSCode
.vscode/
# Pyenv
.python-version

21
ast-pyo3/Cargo.toml Normal file
View file

@ -0,0 +1,21 @@
[package]
name = "rustpython-ast-pyo3"
version = "0.1.0"
edition = "2021"
[features]
# This feature is experimental
# It reimplements AST types, but currently both slower than python AST types and limited to use in other API
wrapper = []
[lib]
name = "rustpython_ast"
crate-type = ["cdylib"]
[dependencies]
rustpython-ast = { workspace = true, features = ["location"] }
rustpython-parser = { workspace = true }
num-complex = { workspace = true }
once_cell = { workspace = true }
pyo3 = { workspace = true, features = ["num-bigint", "num-complex"] }

15
ast-pyo3/pyproject.toml Normal file
View file

@ -0,0 +1,15 @@
[build-system]
requires = ["maturin>=0.15,<0.16"]
build-backend = "maturin"
[project]
name = "rustpython_ast"
requires-python = ">=3.7"
classifiers = [
"Programming Language :: Rust",
"Programming Language :: Python :: Implementation :: CPython",
]
[tool.maturin]
features = ["pyo3/extension-module"]

4841
ast-pyo3/src/gen/to_py_ast.rs Normal file
View file

File diff suppressed because it is too large Load diff

1968
ast/src/gen/pyo3_wrapper_located.rs → ast-pyo3/src/gen/wrapper_located.rs
View file

File diff suppressed because it is too large Load diff

1820
ast/src/gen/pyo3_wrapper_ranged.rs → ast-pyo3/src/gen/wrapper_ranged.rs
View file

File diff suppressed because it is too large Load diff

49
ast-pyo3/src/lib.rs Normal file
View file

@ -0,0 +1,49 @@
mod py_ast;
#[cfg(feature = "wrapper")]
pub mod wrapper;
pub use py_ast::{init, PyNode, ToPyAst};
use pyo3::prelude::*;
use rustpython_parser::ast::{source_code::SourceLocator, Fold};
#[pyfunction]
#[pyo3(signature = (source, filename="<unknown>", *, type_comments=false, locate=true))]
pub fn parse<'py>(
source: &str,
filename: &str,
type_comments: bool,
locate: bool,
py: Python<'py>,
) -> PyResult<&'py PyAny> {
if type_comments {
todo!("'type_comments' is not implemented yet");
}
let parsed = rustpython_parser::parse(source, rustpython_parser::Mode::Module, filename)
.map_err(|e| PyErr::new::<pyo3::exceptions::PySyntaxError, _>(e.to_string()))?;
if locate {
let parsed = SourceLocator::new(source).fold(parsed).unwrap();
parsed.module().unwrap().to_py_ast(py)
} else {
parsed.module().unwrap().to_py_ast(py)
}
}
#[pymodule]
fn rustpython_ast(py: Python, m: &PyModule) -> PyResult<()> {
py_ast::init(py)?;
#[cfg(feature = "wrapper")]
{
let ast = PyModule::new(py, "ast")?;
wrapper::located::add_to_module(py, ast)?;
m.add_submodule(ast)?;
let ast = PyModule::new(py, "unlocated_ast")?;
wrapper::ranged::add_to_module(py, ast)?;
m.add_submodule(ast)?;
}
m.add_function(wrap_pyfunction!(parse, m)?)?;
Ok(())
}

203
ast-pyo3/src/py_ast.rs Normal file
View file

@ -0,0 +1,203 @@
use num_complex::Complex64;
use once_cell::sync::OnceCell;
use pyo3::{
prelude::*,
types::{PyBool, PyBytes, PyList, PyString, PyTuple},
ToPyObject,
};
use rustpython_ast::{
self as ast, source_code::SourceRange, text_size::TextRange, ConversionFlag, Node,
};
pub trait PyNode {
fn py_type_cache() -> &'static OnceCell<(Py<PyAny>, Py<PyAny>)> {
{
static PY_TYPE: OnceCell<(Py<PyAny>, Py<PyAny>)> = OnceCell::new();
&PY_TYPE
}
}
}
pub trait ToPyAst {
fn to_py_ast<'py>(&self, py: Python<'py>) -> PyResult<&'py PyAny>;
}
impl<T: ToPyAst> ToPyAst for Box<T> {
#[inline]
fn to_py_ast<'py>(&self, py: Python<'py>) -> PyResult<&'py PyAny> {
(**self).to_py_ast(py)
}
}
impl<T: ToPyAst> ToPyAst for Option<T> {
#[inline]
fn to_py_ast<'py>(&self, py: Python<'py>) -> PyResult<&'py PyAny> {
match self {
Some(ast) => ast.to_py_ast(py),
None => Ok(ast_cache().none_ref(py)),
}
}
}
impl<T: ToPyAst> ToPyAst for Vec<T> {
fn to_py_ast<'py>(&self, py: Python<'py>) -> PyResult<&'py PyAny> {
let elts = self
.iter()
.map(|item| item.to_py_ast(py))
.collect::<Result<Vec<_>, _>>()?;
let list = PyList::new(py, elts);
Ok(list.into())
}
}
impl ToPyAst for ast::Identifier {
#[inline]
fn to_py_ast<'py>(&self, py: Python<'py>) -> PyResult<&'py PyAny> {
Ok(PyString::new(py, self.as_str()).into())
}
}
impl ToPyAst for ast::String {
#[inline]
fn to_py_ast<'py>(&self, py: Python<'py>) -> PyResult<&'py PyAny> {
Ok(PyString::new(py, self.as_str()).into())
}
}
impl ToPyAst for bool {
#[inline]
fn to_py_ast<'py>(&self, py: Python<'py>) -> PyResult<&'py PyAny> {
Ok(ast_cache().bool_int(py, *self))
}
}
impl ToPyAst for ConversionFlag {
#[inline]
fn to_py_ast<'py>(&self, py: Python<'py>) -> PyResult<&'py PyAny> {
Ok(ast_cache().conversion_flag(py, *self))
}
}
fn constant_to_object(constant: &ast::Constant, py: Python) -> PyObject {
let cache = ast_cache();
match constant {
ast::Constant::None => cache.none.clone_ref(py),
ast::Constant::Bool(bool) => cache.bool(py, *bool).into(),
ast::Constant::Str(string) => string.to_object(py),
ast::Constant::Bytes(bytes) => PyBytes::new(py, bytes).into(),
ast::Constant::Int(int) => int.to_object(py),
ast::Constant::Tuple(elts) => {
let elts: Vec<_> = elts.iter().map(|c| constant_to_object(c, py)).collect();
PyTuple::new(py, elts).into()
}
ast::Constant::Float(f64) => f64.to_object(py),
ast::Constant::Complex { real, imag } => Complex64::new(*real, *imag).to_object(py),
ast::Constant::Ellipsis => py.Ellipsis(),
}
}
#[pyclass(module = "rustpython_ast", subclass)]
pub struct Ast;
#[pymethods]
impl Ast {
#[new]
fn new() -> Self {
Self
}
}
fn cache_py_type<N: PyNode + Node>(ast_module: &PyAny) -> PyResult<()> {
let class = ast_module.getattr(N::NAME)?;
let base = if std::mem::size_of::<N>() == 0 {
class.call0()?
} else {
class.getattr("__new__")?
};
N::py_type_cache().get_or_init(|| (class.into(), base.into()));
Ok(())
}
// TODO: This cache must be bound to 'py
struct AstCache {
lineno: Py<PyString>,
col_offset: Py<PyString>,
end_lineno: Py<PyString>,
end_col_offset: Py<PyString>,
none: Py<PyAny>,
bool_values: (Py<PyBool>, Py<PyBool>),
bool_int_values: (Py<PyAny>, Py<PyAny>),
conversion_flags: (Py<PyAny>, Py<PyAny>, Py<PyAny>, Py<PyAny>),
}
impl AstCache {
// fn location_vec<'py>(&'static self, py: Python<'py>, range: &SourceRange) -> &'py PyDict {
// let attributes = PyDict::new(py);
// attributes.set_item(self.lineno.as_ref(py), range.start.row.get()).unwrap();
// attributes.set_item(self.col_offset.as_ref(py), range.start.column.to_zero_indexed()).unwrap();
// if let Some(end) = range.end {
// attributes.set_item(self.end_lineno.as_ref(py), end.row.get()).unwrap();
// attributes.set_item(
// self.end_col_offset.as_ref(py),
// end.column.to_zero_indexed(),
// ).unwrap();
// }
// attributes
// }
#[inline]
fn none_ref<'py>(&'static self, py: Python<'py>) -> &'py PyAny {
Py::<PyAny>::as_ref(&self.none, py)
}
#[inline]
fn bool_int<'py>(&'static self, py: Python<'py>, value: bool) -> &'py PyAny {
let v = &self.bool_int_values;
Py::<PyAny>::as_ref(if value { &v.1 } else { &v.0 }, py)
}
#[inline]
fn bool(&'static self, py: Python, value: bool) -> Py<PyBool> {
let v = &self.bool_values;
(if value { &v.1 } else { &v.0 }).clone_ref(py)
}
fn conversion_flag<'py>(&'static self, py: Python<'py>, value: ConversionFlag) -> &'py PyAny {
let v = &self.conversion_flags;
match value {
ConversionFlag::None => v.0.as_ref(py),
ConversionFlag::Str => v.1.as_ref(py),
ConversionFlag::Ascii => v.2.as_ref(py),
ConversionFlag::Repr => v.3.as_ref(py),
}
}
}
fn ast_cache_cell() -> &'static OnceCell<AstCache> {
{
static PY_TYPE: OnceCell<AstCache> = OnceCell::new();
&PY_TYPE
}
}
fn ast_cache() -> &'static AstCache {
ast_cache_cell().get().unwrap()
}
pub fn init(py: Python) -> PyResult<()> {
ast_cache_cell().get_or_init(|| AstCache {
lineno: pyo3::intern!(py, "lineno").into_py(py),
col_offset: pyo3::intern!(py, "col_offset").into_py(py),
end_lineno: pyo3::intern!(py, "end_lineno").into_py(py),
end_col_offset: pyo3::intern!(py, "end_col_offset").into_py(py),
none: py.None(),
bool_values: (PyBool::new(py, false).into(), PyBool::new(py, true).into()),
bool_int_values: ((0).to_object(py), (1).to_object(py)),
conversion_flags: (
(-1).to_object(py),
(b's').to_object(py),
(b'a').to_object(py),
(b'r').to_object(py),
),
});
init_types(py)
}
include!("gen/to_py_ast.rs");

141
ast-pyo3/src/wrapper.rs Normal file
View file

@ -0,0 +1,141 @@
use crate::PyNode;
use num_complex::Complex64;
use pyo3::prelude::*;
use pyo3::types::{PyBytes, PyList, PyTuple};
use rustpython_ast::{
self as ast, source_code::SourceRange, text_size::TextRange, ConversionFlag, Node,
};
pub trait ToPyWrapper {
fn to_py_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>>;
}
impl<T: ToPyWrapper> ToPyWrapper for Box<T> {
#[inline]
fn to_py_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
(**self).to_py_wrapper(py)
}
}
impl<T: ToPyWrapper> ToPyWrapper for Option<T> {
#[inline]
fn to_py_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
match self {
Some(ast) => ast.to_py_wrapper(py),
None => Ok(py.None()),
}
}
}
impl ToPyWrapper for ast::Identifier {
#[inline]
fn to_py_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
Ok(self.as_str().to_object(py))
}
}
impl ToPyWrapper for ast::String {
#[inline]
fn to_py_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
Ok(self.as_str().to_object(py))
}
}
impl ToPyWrapper for ast::Int {
#[inline]
fn to_py_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
Ok((self.to_u32()).to_object(py))
}
}
impl ToPyWrapper for bool {
#[inline]
fn to_py_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
Ok((*self as u32).to_object(py))
}
}
impl ToPyWrapper for ConversionFlag {
#[inline]
fn to_py_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
Ok((*self as i8).to_object(py))
}
}
impl ToPyWrapper for ast::Constant {
fn to_py_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
let value = match self {
ast::Constant::None => py.None(),
ast::Constant::Bool(bool) => bool.to_object(py),
ast::Constant::Str(string) => string.to_object(py),
ast::Constant::Bytes(bytes) => PyBytes::new(py, bytes).into(),
ast::Constant::Int(int) => int.to_object(py),
ast::Constant::Tuple(elts) => {
let elts: PyResult<Vec<_>> = elts.iter().map(|c| c.to_py_wrapper(py)).collect();
PyTuple::new(py, elts?).into()
}
ast::Constant::Float(f64) => f64.to_object(py),
ast::Constant::Complex { real, imag } => Complex64::new(*real, *imag).to_object(py),
ast::Constant::Ellipsis => py.Ellipsis(),
};
Ok(value)
}
}
impl<T: ToPyWrapper> ToPyWrapper for Vec<T> {
fn to_py_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
let list = PyList::empty(py);
for item in self {
let py_item = item.to_py_wrapper(py)?;
list.append(py_item)?;
}
Ok(list.into())
}
}
#[pyclass(module = "rustpython_ast", name = "AST", subclass)]
pub struct Ast;
#[pymethods]
impl Ast {
#[new]
fn new() -> Self {
Self
}
}
pub mod located {
pub use super::Ast;
use super::*;
include!("gen/wrapper_located.rs");
}
pub mod ranged {
pub use super::Ast;
use super::*;
include!("gen/wrapper_ranged.rs");
}
fn init_type<P: pyo3::PyClass, N: PyNode + Node>(py: Python, m: &PyModule) -> PyResult<()> {
m.add_class::<P>()?;
let node = m.getattr(P::NAME)?;
if P::NAME != N::NAME {
// TODO: no idea how to escape rust keyword on #[pyclass]
m.setattr(P::NAME, node)?;
}
let names: Vec<&'static str> = N::FIELD_NAMES.to_vec();
let fields = PyTuple::new(py, names);
node.setattr("_fields", fields)?;
Ok(())
}
/// A Python module implemented in Rust.
fn init_module(py: Python, m: &PyModule) -> PyResult<()> {
m.add_class::<Ast>()?;
let ast = m.getattr("AST")?;
let fields = PyTuple::empty(py);
ast.setattr("_fields", fields)?;
Ok(())
}

56
ast-pyo3/test_ast.py Normal file
View file

@ -0,0 +1,56 @@
import re
import difflib
import pytest
import ast as py_ast
import rustpython_ast as rust_ast
from glob import glob
files = {}
for path in glob("../../cpython/Lib/**/*.py"):
try:
txt = open(path, "r").read()
except UnicodeDecodeError:
continue
# try:
# if py_ast.dump(py_ast.parse(txt)) != py_ast.dump(rust_ast.parse(txt)):
# continue
# except SyntaxError:
# continue
files[path] = txt
@pytest.mark.parametrize("path", files.keys())
def test_roundtrip(path):
txt = files[path]
module_p = py_ast.parse(txt)
dump_p = py_ast.dump(module_p, indent=True)
module_r = rust_ast.parse(txt)
dump_r = py_ast.dump(module_r, indent=True)
p = re.compile("object at 0x[0-9a-f]+")
dump_p2 = re.sub(p, "object at 0x????????", dump_p)
dump_r2 = re.sub(p, "object at 0x????????", dump_r)
try:
assert dump_p2 == dump_r2
except AssertionError:
last_sign = ' '
for s in difflib.ndiff(dump_p2, dump_r2):
if s[0]==' ': continue
if s[0] == last_sign:
print(s[2:], end='')
else:
print()
print(s, end='')
last_sign = s[0]
# with open("dump_code.py", "w") as f:
# f.write(path)
# f.write('\n')
# f.write(txt)
# with open("dump_p.txt", "w") as f:
# f.write(dump_p2)
# with open("dump_r.txt", "w") as f:
# f.write(dump_r2)
raise

8
ast/Cargo.toml
View file

@ -15,11 +15,6 @@ fold = []
unparse = ["rustpython-literal"]
visitor = []
all-nodes-with-ranges = []
pyo3 = ["dep:pyo3", "num-complex", "once_cell"]
# This feature is experimental
# It reimplements AST types, but currently both slower than python AST types and limited to use in other API
pyo3-wrapper = ["pyo3"]
[dependencies]
rustpython-parser-core = { workspace = true }
@ -28,6 +23,5 @@ rustpython-literal = { workspace = true, optional = true }
is-macro = { workspace = true }
num-bigint = { workspace = true }
static_assertions = "1.1.0"
num-complex = { workspace = true, optional = true }
once_cell = { workspace = true, optional = true }
pyo3 = { workspace = true, optional = true, features = ["num-bigint", "num-complex"] }

164
ast/asdl_rs.py
View file

@ -300,10 +300,13 @@ class StructVisitor(EmitVisitor):
def visitModule(self, mod):
self.emit_attrs(0)
self.emit("""
self.emit(
"""
#[derive(is_macro::Is)]
pub enum Ast<R=TextRange> {
""", 0)
""",
0,
)
for dfn in mod.dfns:
rust_name = rust_type_name(dfn.name)
generics = "" if self.type_info[dfn.name].is_simple else "<R>"
@ -315,23 +318,29 @@ class StructVisitor(EmitVisitor):
# "ast_" prefix to everywhere seems less useful.
self.emit('#[is(name = "module")]', 1)
self.emit(f"{rust_name}({rust_name}{generics}),", 1)
self.emit("""
}
impl<R> Node for Ast<R> {
const NAME: &'static str = "AST";
const FIELD_NAMES: &'static [&'static str] = &[];
}
""", 0)
self.emit(
"""
}
impl<R> Node for Ast<R> {
const NAME: &'static str = "AST";
const FIELD_NAMES: &'static [&'static str] = &[];
}
""",
0,
)
for dfn in mod.dfns:
rust_name = rust_type_name(dfn.name)
generics = "" if self.type_info[dfn.name].is_simple else "<R>"
self.emit(f"""
self.emit(
f"""
impl<R> From<{rust_name}{generics}> for Ast<R> {{
fn from(node: {rust_name}{generics}) -> Self {{
Ast::{rust_name}(node)
}}
}}
""", 0)
""",
0,
)
for dfn in mod.dfns:
self.visit(dfn)
@ -663,9 +672,7 @@ class FoldImplVisitor(EmitVisitor):
cons_type_name = f"{enum_name}{cons.name}"
self.emit(
f"impl<T, U> Foldable<T, U> for {cons_type_name}{apply_t} {{", depth
)
self.emit(f"impl<T, U> Foldable<T, U> for {cons_type_name}{apply_t} {{", depth)
self.emit(f"type Mapped = {cons_type_name}{apply_u};", depth + 1)
self.emit(
"fn fold<F: Fold<T, TargetU = U> + ?Sized>(self, folder: &mut F) -> Result<Self::Mapped, F::Error> {",
@ -1071,11 +1078,6 @@ class ToPyo3AstVisitor(EmitVisitor):
else:
assert False, self.namespace
@property
def location(self):
# lineno, col_offset
pass
def visitModule(self, mod):
for dfn in mod.dfns:
self.visit(dfn)
@ -1095,16 +1097,16 @@ class ToPyo3AstVisitor(EmitVisitor):
self.emit(
f"""
impl ToPyo3Ast for crate::generic::{rust_name}{self.generics} {{
impl ToPyAst for ast::{rust_name}{self.generics} {{
#[inline]
fn to_pyo3_ast(&self, {"_" if simple else ""}py: Python) -> PyResult<Py<PyAny>> {{
fn to_py_ast<'py>(&self, {"_" if simple else ""}py: Python<'py>) -> PyResult<&'py PyAny> {{
let instance = match &self {{
""",
0,
)
for cons in sum.types:
self.emit(
f"crate::{rust_name}::{cons.name}(cons) => cons.to_pyo3_ast(py)?,",
f"ast::{rust_name}::{cons.name}(cons) => cons.to_py_ast(py)?,",
1,
)
self.emit(
@ -1126,12 +1128,13 @@ class ToPyo3AstVisitor(EmitVisitor):
def emit_to_pyo3_with_fields(self, cons, type, name):
type_info = self.type_info[type.name]
self.emit(
f"""
impl ToPyo3Ast for crate::{name}{self.generics} {{
impl ToPyAst for ast::{name}{self.generics} {{
#[inline]
fn to_pyo3_ast(&self, py: Python) -> PyResult<Py<PyAny>> {{
let cache = Self::py_type_cache().get().unwrap();
fn to_py_ast<'py>(&self, py: Python<'py>) -> PyResult<&'py PyAny> {{
let cache = Self::py_type_cache().get().unwrap();
""",
0,
)
@ -1144,12 +1147,34 @@ class ToPyo3AstVisitor(EmitVisitor):
1,
)
self.emit(
"let instance = cache.0.call1(py, (",
"""
let instance = Py::<PyAny>::as_ref(&cache.0, py).call1((
""",
1,
)
for field in cons.fields:
if field.type == "constant":
self.emit(
f"constant_to_object({rust_field(field.name)}, py),",
3,
)
continue
if field.type == "int":
if field.name == "level":
assert field.opt
self.emit(
f"{rust_field(field.name)}.map_or_else(|| py.None(), |level| level.to_u32().to_object(py)),",
3,
)
continue
if field.name == "lineno":
self.emit(
f"{rust_field(field.name)}.to_u32().to_object(py),",
3,
)
continue
self.emit(
f"{rust_field(field.name)}.to_pyo3_ast(py)?,",
f"{rust_field(field.name)}.to_py_ast(py)?,",
3,
)
self.emit(
@ -1158,25 +1183,25 @@ class ToPyo3AstVisitor(EmitVisitor):
)
else:
self.emit(
"let instance = cache.0.call0(py)?;",
"let Self { range: _range } = self;",
1,
)
self.emit(
"let Self { range: _range } = self;",
"""let instance = Py::<PyAny>::as_ref(&cache.0, py).call0()?;""",
1,
)
if type.value.attributes and self.namespace == "located":
self.emit(
"""
let cache = ast_key_cache().get().unwrap();
instance.setattr(py, cache.lineno.as_ref(py), _range.start.row.get())?;
instance.setattr(py, cache.col_offset.as_ref(py), _range.start.column.get())?;
let cache = ast_cache();
instance.setattr(cache.lineno.as_ref(py), _range.start.row.get())?;
instance.setattr(cache.col_offset.as_ref(py), _range.start.column.get())?;
if let Some(end) = _range.end {
instance.setattr(py, cache.end_lineno.as_ref(py), end.row.get())?;
instance.setattr(py, cache.end_col_offset.as_ref(py), end.column.get())?;
instance.setattr(cache.end_lineno.as_ref(py), end.row.get())?;
instance.setattr(cache.end_col_offset.as_ref(py), end.column.get())?;
}
""",
1,
0,
)
self.emit(
"""
@ -1218,7 +1243,7 @@ class Pyo3StructVisitor(EmitVisitor):
def ref(self):
return "&" if self.borrow else ""
def emit_class(self, name, rust_name, simple, base="super::AST"):
def emit_class(self, name, rust_name, simple, base="super::Ast"):
info = self.type_info[name]
if simple:
generics = ""
@ -1230,7 +1255,7 @@ class Pyo3StructVisitor(EmitVisitor):
into = f"{rust_name}"
else:
subclass = ""
body = f"(pub {self.ref_def} crate::{rust_name}{generics})"
body = f"(pub {self.ref_def} ast::{rust_name}{generics})"
into = f"{rust_name}(node)"
self.emit(
@ -1239,8 +1264,8 @@ class Pyo3StructVisitor(EmitVisitor):
#[derive(Clone, Debug)]
pub struct {rust_name} {body};
impl From<{self.ref_def} crate::{rust_name}{generics}> for {rust_name} {{
fn from({"" if body else "_"}node: {self.ref_def} crate::{rust_name}{generics}) -> Self {{
impl From<{self.ref_def} ast::{rust_name}{generics}> for {rust_name} {{
fn from({"" if body else "_"}node: {self.ref_def} ast::{rust_name}{generics}) -> Self {{
{into}
}}
}}
@ -1254,7 +1279,7 @@ class Pyo3StructVisitor(EmitVisitor):
impl {rust_name} {{
#[new]
fn new() -> PyClassInitializer<Self> {{
PyClassInitializer::from(AST)
PyClassInitializer::from(Ast)
.add_subclass(Self)
}}
@ -1269,7 +1294,7 @@ class Pyo3StructVisitor(EmitVisitor):
0,
)
else:
if base != "super::AST":
if base != "super::Ast":
add_subclass = f".add_subclass({base})"
else:
add_subclass = ""
@ -1277,7 +1302,7 @@ class Pyo3StructVisitor(EmitVisitor):
f"""
impl ToPyObject for {rust_name} {{
fn to_object(&self, py: Python) -> PyObject {{
let initializer = PyClassInitializer::from(AST)
let initializer = PyClassInitializer::from(Ast)
{add_subclass}
.add_subclass(self.clone());
Py::new(py, initializer).unwrap().into_py(py)
@ -1305,7 +1330,7 @@ class Pyo3StructVisitor(EmitVisitor):
#[getter]
#[inline]
fn get_{field.name}(&self, py: Python) -> PyResult<PyObject> {{
self.0.{rust_field(field.name)}.to_pyo3_wrapper(py)
self.0.{rust_field(field.name)}.to_py_wrapper(py)
}}
""",
3,
@ -1331,7 +1356,7 @@ class Pyo3StructVisitor(EmitVisitor):
for field in owner.fields:
self.emit(
f'"{field.name}" => self.0.{rust_field(field.name)}.to_pyo3_wrapper(py)?,',
f'"{field.name}" => self.0.{rust_field(field.name)}.to_py_wrapper(py)?,',
3,
)
@ -1349,9 +1374,9 @@ class Pyo3StructVisitor(EmitVisitor):
def emit_wrapper(self, rust_name):
self.emit(
f"""
impl ToPyo3Wrapper for crate::{rust_name}{self.generics} {{
impl ToPyWrapper for ast::{rust_name}{self.generics} {{
#[inline]
fn to_pyo3_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {{
fn to_py_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {{
Ok({rust_name}(self).to_object(py))
}}
}}
@ -1375,16 +1400,16 @@ class Pyo3StructVisitor(EmitVisitor):
if not simple:
self.emit(
f"""
impl ToPyo3Wrapper for crate::{rust_name}{self.generics} {{
impl ToPyWrapper for ast::{rust_name}{self.generics} {{
#[inline]
fn to_pyo3_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {{
fn to_py_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {{
match &self {{
""",
0,
)
for cons in sum.types:
self.emit(f"Self::{cons.name}(cons) => cons.to_pyo3_wrapper(py),", 3)
self.emit(f"Self::{cons.name}(cons) => cons.to_py_wrapper(py),", 3)
self.emit(
"""
@ -1413,7 +1438,7 @@ class Pyo3StructVisitor(EmitVisitor):
impl ToPyObject for {parent}{cons.name} {{
fn to_object(&self, py: Python) -> PyObject {{
let initializer = PyClassInitializer::from(AST)
let initializer = PyClassInitializer::from(Ast)
.add_subclass({parent})
.add_subclass(Self);
Py::new(py, initializer).unwrap().into_py(py)
@ -1462,9 +1487,7 @@ class Pyo3PymoduleVisitor(EmitVisitor):
self.emit_fields(cons.name, rust_name, simple)
def emit_fields(self, name, rust_name, simple):
self.emit(
f"super::init_type::<{rust_name}, crate::generic::{rust_name}>(py, m)?;", 1
)
self.emit(f"super::init_type::<{rust_name}, ast::{rust_name}>(py, m)?;", 1)
class StdlibClassDefVisitor(EmitVisitor):
@ -1812,7 +1835,7 @@ def write_pyo3_node(type_info, f):
f.write(
f"""
impl{generics} Pyo3Node for crate::generic::{rust_name}{generics} {{
impl{generics} PyNode for ast::{rust_name}{generics} {{
#[inline]
fn py_type_cache() -> &'static OnceCell<(Py<PyAny>, Py<PyAny>)> {{
static PY_TYPE: OnceCell<(Py<PyAny>, Py<PyAny>)> = OnceCell::new();
@ -1842,7 +1865,7 @@ def write_to_pyo3(mod, type_info, f):
for info in type_info.values():
rust_name = info.rust_sum_name
f.write(f"cache_py_type::<crate::generic::{rust_name}>(ast_module)?;\n")
f.write(f"cache_py_type::<ast::{rust_name}>(ast_module)?;\n")
f.write("Ok(())\n}")
@ -1856,20 +1879,20 @@ def write_to_pyo3_simple(type_info, f):
rust_name = type_info.rust_sum_name
f.write(
f"""
impl ToPyo3Ast for crate::generic::{rust_name} {{
impl ToPyAst for ast::{rust_name} {{
#[inline]
fn to_pyo3_ast(&self, _py: Python) -> PyResult<Py<PyAny>> {{
fn to_py_ast<'py>(&self, py: Python<'py>) -> PyResult<&'py PyAny> {{
let cell = match &self {{
""",
)
for cons in type_info.type.value.types:
f.write(
f"""crate::{rust_name}::{cons.name} => crate::{rust_name}{cons.name}::py_type_cache(),""",
f"""ast::{rust_name}::{cons.name} => ast::{rust_name}{cons.name}::py_type_cache(),""",
)
f.write(
"""
};
Ok(cell.get().unwrap().1.clone())
Ok(Py::<PyAny>::as_ref(&cell.get().unwrap().1, py))
}
}
""",
@ -1887,9 +1910,9 @@ def write_pyo3_wrapper(mod, type_info, namespace, f):
rust_name = type_info.rust_sum_name
f.write(
f"""
impl ToPyo3Wrapper for crate::generic::{rust_name} {{
impl ToPyWrapper for ast::{rust_name} {{
#[inline]
fn to_pyo3_wrapper(&self, py: Python) -> PyResult<Py<PyAny>> {{
fn to_py_wrapper(&self, py: Python) -> PyResult<Py<PyAny>> {{
match &self {{
""",
)
@ -1908,9 +1931,9 @@ def write_pyo3_wrapper(mod, type_info, namespace, f):
for cons in type_info.type.value.types:
f.write(
f"""
impl ToPyo3Wrapper for crate::generic::{rust_name}{cons.name} {{
impl ToPyWrapper for ast::{rust_name}{cons.name} {{
#[inline]
fn to_pyo3_wrapper(&self, py: Python) -> PyResult<Py<PyAny>> {{
fn to_py_wrapper(&self, py: Python) -> PyResult<Py<PyAny>> {{
Ok({rust_name}{cons.name}.to_object(py))
}}
}}
@ -1949,6 +1972,7 @@ def write_ast_mod(mod, type_info, f):
def main(
input_filename,
ast_dir,
ast_pyo3_dir,
module_filename,
dump_module=False,
):
@ -1971,14 +1995,20 @@ def main(
("ranged", p(write_ranged_def, mod, type_info)),
("located", p(write_located_def, mod, type_info)),
("visitor", p(write_visitor_def, mod, type_info)),
("to_pyo3", p(write_to_pyo3, mod, type_info)),
("pyo3_wrapper_located", p(write_pyo3_wrapper, mod, type_info, "located")),
("pyo3_wrapper_ranged", p(write_pyo3_wrapper, mod, type_info, "ranged")),
]:
with (ast_dir / f"{filename}.rs").open("w") as f:
f.write(auto_gen_msg)
write(f)
for filename, write in [
("to_py_ast", p(write_to_pyo3, mod, type_info)),
("wrapper_located", p(write_pyo3_wrapper, mod, type_info, "located")),
("wrapper_ranged", p(write_pyo3_wrapper, mod, type_info, "ranged")),
]:
with (ast_pyo3_dir / f"{filename}.rs").open("w") as f:
f.write(auto_gen_msg)
write(f)
with module_filename.open("w") as module_file:
module_file.write(auto_gen_msg)
write_ast_mod(mod, type_info, module_file)
@ -1990,6 +2020,7 @@ if __name__ == "__main__":
parser = ArgumentParser()
parser.add_argument("input_file", type=Path)
parser.add_argument("-A", "--ast-dir", type=Path, required=True)
parser.add_argument("-O", "--ast-pyo3-dir", type=Path, required=True)
parser.add_argument("-M", "--module-file", type=Path, required=True)
parser.add_argument("-d", "--dump-module", action="store_true")
@ -1997,6 +2028,7 @@ if __name__ == "__main__":
main(
args.input_file,
args.ast_dir,
args.ast_pyo3_dir,
args.module_file,
args.dump_module,
)

4937
ast/src/gen/to_pyo3.rs
View file

File diff suppressed because it is too large Load diff

17
ast/src/generic.rs
View file

@ -45,4 +45,21 @@ impl<R> Default for EmptyRange<R> {
}
}
impl Cmpop {
pub fn as_str(&self) -> &'static str {
match self {
Cmpop::Eq => "==",
Cmpop::NotEq => "!=",
Cmpop::Lt => "<",
Cmpop::LtE => "<=",
Cmpop::Gt => ">",
Cmpop::GtE => ">=",
Cmpop::Is => "is",
Cmpop::IsNot => "is not",
Cmpop::In => "in",
Cmpop::NotIn => "not in",
}
}
}
include!("gen/generic.rs");

5
ast/src/lib.rs
View file

@ -41,8 +41,3 @@ pub use visitor::Visitor;
mod optimizer;
#[cfg(feature = "constant-optimization")]
pub use optimizer::ConstantOptimizer;
#[cfg(feature = "pyo3")]
pub mod pyo3;
#[cfg(feature = "pyo3-wrapper")]
pub mod pyo3_wrapper;

153
ast/src/pyo3.rs
View file

@ -1,153 +0,0 @@
use crate::{source_code::SourceRange, text_size::TextRange, ConversionFlag, Node};
use num_complex::Complex64;
use once_cell::sync::OnceCell;
use pyo3::{
prelude::*,
types::{PyBytes, PyList, PyString, PyTuple},
};
pub trait Pyo3Node {
fn py_type_cache() -> &'static OnceCell<(Py<PyAny>, Py<PyAny>)> {
{
static PY_TYPE: OnceCell<(Py<PyAny>, Py<PyAny>)> = OnceCell::new();
&PY_TYPE
}
}
}
pub trait ToPyo3Ast {
fn to_pyo3_ast(&self, py: Python) -> PyResult<Py<PyAny>>;
}
impl<T: ToPyo3Ast> ToPyo3Ast for Box<T> {
#[inline]
fn to_pyo3_ast(&self, py: Python) -> PyResult<Py<PyAny>> {
(**self).to_pyo3_ast(py)
}
}
impl<T: ToPyo3Ast> ToPyo3Ast for Option<T> {
#[inline]
fn to_pyo3_ast(&self, py: Python) -> PyResult<Py<PyAny>> {
match self {
Some(ast) => ast.to_pyo3_ast(py),
None => Ok(py.None()),
}
}
}
impl<T: ToPyo3Ast> ToPyo3Ast for Vec<T> {
fn to_pyo3_ast(&self, py: Python) -> PyResult<Py<PyAny>> {
let elts = self
.iter()
.map(|item| item.to_pyo3_ast(py))
.collect::<Result<Vec<_>, _>>()?;
let list = PyList::new(py, elts);
Ok(list.into())
}
}
impl ToPyo3Ast for crate::Identifier {
#[inline]
fn to_pyo3_ast(&self, py: Python) -> PyResult<Py<PyAny>> {
Ok(self.as_str().to_object(py))
}
}
impl ToPyo3Ast for crate::String {
#[inline]
fn to_pyo3_ast(&self, py: Python) -> PyResult<Py<PyAny>> {
Ok(self.as_str().to_object(py))
}
}
impl ToPyo3Ast for crate::Int {
#[inline]
fn to_pyo3_ast(&self, py: Python) -> PyResult<Py<PyAny>> {
Ok((self.to_u32()).to_object(py))
}
}
impl ToPyo3Ast for bool {
#[inline]
fn to_pyo3_ast(&self, py: Python) -> PyResult<Py<PyAny>> {
Ok((*self as u32).to_object(py))
}
}
impl ToPyo3Ast for ConversionFlag {
#[inline]
fn to_pyo3_ast(&self, py: Python) -> PyResult<Py<PyAny>> {
Ok((*self as i8).to_object(py))
}
}
impl ToPyo3Ast for crate::Constant {
#[inline]
fn to_pyo3_ast(&self, py: Python) -> PyResult<Py<PyAny>> {
let value = match self {
crate::Constant::None => py.None(),
crate::Constant::Bool(bool) => bool.to_object(py),
crate::Constant::Str(string) => string.to_object(py),
crate::Constant::Bytes(bytes) => PyBytes::new(py, bytes).into(),
crate::Constant::Int(int) => int.to_object(py),
crate::Constant::Tuple(elts) => {
let elts: PyResult<Vec<_>> = elts.iter().map(|c| c.to_pyo3_ast(py)).collect();
PyTuple::new(py, elts?).into()
}
crate::Constant::Float(f64) => f64.to_object(py),
crate::Constant::Complex { real, imag } => Complex64::new(*real, *imag).to_object(py),
crate::Constant::Ellipsis => py.Ellipsis(),
};
Ok(value)
}
}
#[pyclass(module = "rustpython_ast", subclass)]
pub struct AST;
#[pymethods]
impl AST {
#[new]
fn new() -> Self {
Self
}
}
fn cache_py_type<N: Pyo3Node + Node>(ast_module: &PyAny) -> PyResult<()> {
let class = ast_module.getattr(N::NAME)?;
let base = if std::mem::size_of::<N>() == 0 {
class.call0()?
} else {
class.getattr("__new__")?
};
N::py_type_cache().get_or_init(|| (class.into(), base.into()));
Ok(())
}
struct AstKeyCache {
lineno: Py<PyString>,
col_offset: Py<PyString>,
end_lineno: Py<PyString>,
end_col_offset: Py<PyString>,
}
fn ast_key_cache() -> &'static OnceCell<AstKeyCache> {
{
static PY_TYPE: OnceCell<AstKeyCache> = OnceCell::new();
&PY_TYPE
}
}
pub fn init(py: Python) -> PyResult<()> {
ast_key_cache().get_or_init(|| AstKeyCache {
lineno: pyo3::intern!(py, "lineno").into_py(py),
col_offset: pyo3::intern!(py, "col_offset").into_py(py),
end_lineno: pyo3::intern!(py, "end_lineno").into_py(py),
end_col_offset: pyo3::intern!(py, "end_col_offset").into_py(py),
});
init_types(py)
}
include!("gen/to_pyo3.rs");

128
ast/src/pyo3_wrapper.rs
View file

@ -1,128 +0,0 @@
use crate::pyo3::{Pyo3Node, AST};
use crate::{source_code::SourceRange, text_size::TextRange, ConversionFlag, Node};
use num_complex::Complex64;
use pyo3::prelude::*;
use pyo3::types::{PyBytes, PyList, PyTuple};
pub trait ToPyo3Wrapper {
fn to_pyo3_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>>;
}
impl<T: ToPyo3Wrapper> ToPyo3Wrapper for Box<T> {
#[inline]
fn to_pyo3_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
(**self).to_pyo3_wrapper(py)
}
}
impl<T: ToPyo3Wrapper> ToPyo3Wrapper for Option<T> {
#[inline]
fn to_pyo3_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
match self {
Some(ast) => ast.to_pyo3_wrapper(py),
None => Ok(py.None()),
}
}
}
impl ToPyo3Wrapper for crate::Identifier {
#[inline]
fn to_pyo3_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
Ok(self.as_str().to_object(py))
}
}
impl ToPyo3Wrapper for crate::String {
#[inline]
fn to_pyo3_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
Ok(self.as_str().to_object(py))
}
}
impl ToPyo3Wrapper for crate::Int {
#[inline]
fn to_pyo3_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
Ok((self.to_u32()).to_object(py))
}
}
impl ToPyo3Wrapper for bool {
#[inline]
fn to_pyo3_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
Ok((*self as u32).to_object(py))
}
}
impl ToPyo3Wrapper for ConversionFlag {
#[inline]
fn to_pyo3_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
Ok((*self as i8).to_object(py))
}
}
impl ToPyo3Wrapper for crate::Constant {
fn to_pyo3_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
let value = match self {
crate::Constant::None => py.None(),
crate::Constant::Bool(bool) => bool.to_object(py),
crate::Constant::Str(string) => string.to_object(py),
crate::Constant::Bytes(bytes) => PyBytes::new(py, bytes).into(),
crate::Constant::Int(int) => int.to_object(py),
crate::Constant::Tuple(elts) => {
let elts: PyResult<Vec<_>> = elts.iter().map(|c| c.to_pyo3_wrapper(py)).collect();
PyTuple::new(py, elts?).into()
}
crate::Constant::Float(f64) => f64.to_object(py),
crate::Constant::Complex { real, imag } => Complex64::new(*real, *imag).to_object(py),
crate::Constant::Ellipsis => py.Ellipsis(),
};
Ok(value)
}
}
impl<T: ToPyo3Wrapper> ToPyo3Wrapper for Vec<T> {
fn to_pyo3_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
let list = PyList::empty(py);
for item in self {
let py_item = item.to_pyo3_wrapper(py)?;
list.append(py_item)?;
}
Ok(list.into())
}
}
pub mod located {
use super::*;
pub use crate::pyo3::AST;
include!("gen/pyo3_wrapper_located.rs");
}
pub mod ranged {
use super::*;
pub use crate::pyo3::AST;
include!("gen/pyo3_wrapper_ranged.rs");
}
fn init_type<P: pyo3::PyClass, N: Pyo3Node + Node>(py: Python, m: &PyModule) -> PyResult<()> {
m.add_class::<P>()?;
let node = m.getattr(P::NAME)?;
if P::NAME != N::NAME {
// TODO: no idea how to escape rust keyword on #[pyclass]
m.setattr(P::NAME, node)?;
}
let names: Vec<&'static str> = N::FIELD_NAMES.to_vec();
let fields = PyTuple::new(py, names);
node.setattr("_fields", fields)?;
Ok(())
}
/// A Python module implemented in Rust.
fn init_module(py: Python, m: &PyModule) -> PyResult<()> {
m.add_class::<AST>()?;
let ast = m.getattr("AST")?;
let fields = PyTuple::empty(py);
ast.setattr("_fields", fields)?;
Ok(())
}

18
ast/src/unparse.rs
View file

@ -1,5 +1,5 @@
use crate::ConversionFlag;
use crate::{Arg, Arguments, Boolop, Cmpop, Comprehension, Constant, Expr, Identifier, Operator};
use crate::{Arg, Arguments, Boolop, Comprehension, Constant, Expr, Identifier, Operator};
use std::fmt;
mod precedence {
@ -285,19 +285,9 @@ impl<'a> Unparser<'a> {
let new_lvl = precedence::CMP + 1;
self.unparse_expr(left, new_lvl)?;
for (op, cmp) in ops.iter().zip(comparators) {
let op = match op {
Cmpop::Eq => " == ",
Cmpop::NotEq => " != ",
Cmpop::Lt => " < ",
Cmpop::LtE => " <= ",
Cmpop::Gt => " > ",
Cmpop::GtE => " >= ",
Cmpop::Is => " is ",
Cmpop::IsNot => " is not ",
Cmpop::In => " in ",
Cmpop::NotIn => " not in ",
};
self.p(op)?;
self.p(" ")?;
self.p(op.as_str())?;
self.p(" ")?;
self.unparse_expr(cmp, new_lvl)?;
}
})

5
parser/README.md
View file

@ -60,7 +60,8 @@ Execution mode check. Allowed modes are `exec`, `eval` or `single`.
For example, one could do this:
```
use rustpython_parser::{parser, ast};
use rustpython_parser::{Parse, ast};
let python_source = "print('Hello world')";
let python_ast = parser::parse_expression(python_source).unwrap();
let python_statements = ast::Suite::parse(python_source).unwrap(); // statements
let python_expr = ast::Expr::parse(python_source).unwrap(); // or expr
```

38
parser/src/context.rs
View file

@ -49,19 +49,19 @@ pub(crate) fn set_context(expr: Expr, ctx: ExprContext) -> Expr {
#[cfg(test)]
mod tests {
use crate::parser::parse_program;
use crate::{ast, Parse};
#[test]
fn test_assign_name() {
let source = "x = (1, 2, 3)";
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_assign_tuple() {
let source = "(x, y) = (1, 2, 3)";
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
@ -69,35 +69,35 @@ mod tests {
#[cfg(not(feature = "all-nodes-with-ranges"))]
fn test_assign_list() {
let source = "[x, y] = (1, 2, 3)";
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_assign_attribute() {
let source = "x.y = (1, 2, 3)";
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_assign_subscript() {
let source = "x[y] = (1, 2, 3)";
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_assign_starred() {
let source = "(x, *y) = (1, 2, 3)";
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_assign_for() {
let source = "for x in (1, 2, 3): pass";
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
@ -105,7 +105,7 @@ mod tests {
#[cfg(not(feature = "all-nodes-with-ranges"))]
fn test_assign_list_comp() {
let source = "x = [y for y in (1, 2, 3)]";
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
@ -113,7 +113,7 @@ mod tests {
#[cfg(not(feature = "all-nodes-with-ranges"))]
fn test_assign_set_comp() {
let source = "x = {y for y in (1, 2, 3)}";
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
@ -121,63 +121,63 @@ mod tests {
#[cfg(not(feature = "all-nodes-with-ranges"))]
fn test_assign_with() {
let source = "with 1 as x: pass";
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_assign_named_expr() {
let source = "if x:= 1: pass";
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_ann_assign_name() {
let source = "x: int = 1";
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_aug_assign_name() {
let source = "x += 1";
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_aug_assign_attribute() {
let source = "x.y += (1, 2, 3)";
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_aug_assign_subscript() {
let source = "x[y] += (1, 2, 3)";
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_del_name() {
let source = "del x";
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_del_attribute() {
let source = "del x.y";
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_del_subscript() {
let source = "del x[y]";
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
}

6
parser/src/function.rs
View file

@ -155,7 +155,7 @@ const fn is_starred(exp: &ast::Expr) -> bool {
#[cfg(test)]
mod tests {
use super::*;
use crate::parser::{parse_program, ParseErrorType};
use crate::{ast, parser::ParseErrorType, Parse};
#[cfg(not(feature = "all-nodes-with-ranges"))]
macro_rules! function_and_lambda {
@ -163,7 +163,7 @@ mod tests {
$(
#[test]
fn $name() {
let parse_ast = parse_program($code, "<test>");
let parse_ast = ast::Suite::parse($code, "<test>");
insta::assert_debug_snapshot!(parse_ast);
}
)*
@ -190,7 +190,7 @@ mod tests {
}
fn function_parse_error(src: &str) -> LexicalErrorType {
let parse_ast = parse_program(src, "<test>");
let parse_ast = ast::Suite::parse(src, "<test>");
parse_ast
.map_err(|e| match e.error {
ParseErrorType::Lexical(e) => e,

2
parser/src/lexer.rs
View file

@ -212,7 +212,7 @@ pub fn lex_starts_at(
source: &str,
mode: Mode,
start_offset: TextSize,
) -> impl Iterator<Item = LexResult> + '_ {
) -> SoftKeywordTransformer<Lexer<std::str::Chars<'_>>> {
SoftKeywordTransformer::new(Lexer::new(source.chars(), start_offset), mode)
}

12
parser/src/lib.rs
View file

@ -94,13 +94,13 @@
//! mode or tokenizing the source beforehand:
//!
//! ```
//! use rustpython_parser::parse_program;
//! use rustpython_parser::{Parse, ast};
//!
//! let python_source = r#"
//! def is_odd(i):
//! return bool(i & 1)
//! "#;
//! let ast = parse_program(python_source, "<embedded>");
//! let ast = ast::Suite::parse(python_source, "<embedded>");
//!
//! assert!(ast.is_ok());
//! ```
@ -126,13 +126,13 @@ mod soft_keywords;
mod string;
mod token;
pub use parser::{
parse, parse_expression, parse_expression_starts_at, parse_program, parse_starts_at,
parse_tokens, ParseError, ParseErrorType,
};
pub use parser::{parse, parse_starts_at, parse_tokens, Parse, ParseError, ParseErrorType};
pub use string::FStringErrorType;
pub use token::{StringKind, Tok};
#[allow(deprecated)]
pub use parser::{parse_expression, parse_expression_starts_at, parse_program};
#[rustfmt::skip]
mod python {
#![allow(clippy::all)]

262
parser/src/parser.rs
View file

@ -13,7 +13,7 @@
//! [`Mode`]: crate::mode
use crate::{
ast,
ast::{self, OptionalRange, Ranged},
lexer::{self, LexResult, LexicalError, LexicalErrorType},
python,
text_size::TextSize,
@ -23,9 +23,179 @@ use crate::{
use itertools::Itertools;
use std::iter;
use crate::text_size::TextRange;
use crate::{lexer::Lexer, soft_keywords::SoftKeywordTransformer, text_size::TextRange};
pub(super) use lalrpop_util::ParseError as LalrpopError;
use rustpython_ast::OptionalRange;
/// Parse Python code string to implementor's type.
///
/// # Example
///
/// For example, parsing a simple function definition and a call to that function:
///
/// ```
/// use rustpython_parser::{self as parser, ast, Parse};
/// let source = r#"
/// def foo():
/// return 42
///
/// print(foo())
/// "#;
/// let program = ast::Suite::parse(source, "<embedded>");
/// assert!(program.is_ok());
/// ```
///
/// Parsing a single expression denoting the addition of two numbers, but this time specifying a different,
/// somewhat silly, location:
///
/// ```
/// use rustpython_parser::{self as parser, ast, Parse, text_size::TextSize};
///
/// let expr = ast::Expr::parse_starts_at("1 + 2", "<embedded>", TextSize::from(400));
/// assert!(expr.is_ok());
pub trait Parse
where
Self: Sized,
{
fn parse(source: &str, source_path: &str) -> Result<Self, ParseError> {
Self::parse_starts_at(source, source_path, TextSize::default())
}
fn parse_starts_at(
source: &str,
source_path: &str,
offset: TextSize,
) -> Result<Self, ParseError> {
let lxr = Self::lex_starts_at(source, offset);
#[cfg(feature = "full-lexer")]
let lxr =
lxr.filter_ok(|(tok, _)| !matches!(tok, Tok::Comment { .. } | Tok::NonLogicalNewline));
Self::parse_tokens(lxr, source_path)
}
fn lex_starts_at(
source: &str,
offset: TextSize,
) -> SoftKeywordTransformer<Lexer<std::str::Chars>>;
fn parse_tokens(
lxr: impl IntoIterator<Item = LexResult>,
source_path: &str,
) -> Result<Self, ParseError>;
}
impl Parse for ast::ModModule {
fn lex_starts_at(
source: &str,
offset: TextSize,
) -> SoftKeywordTransformer<Lexer<std::str::Chars>> {
lexer::lex_starts_at(source, Mode::Module, offset)
}
fn parse_tokens(
lxr: impl IntoIterator<Item = LexResult>,
source_path: &str,
) -> Result<Self, ParseError> {
match parse_filtered_tokens(lxr, Mode::Module, source_path)? {
ast::Mod::Module(m) => Ok(m),
_ => unreachable!("Mode::Module doesn't return other variant"),
}
}
}
impl Parse for ast::ModExpression {
fn lex_starts_at(
source: &str,
offset: TextSize,
) -> SoftKeywordTransformer<Lexer<std::str::Chars>> {
lexer::lex_starts_at(source, Mode::Expression, offset)
}
fn parse_tokens(
lxr: impl IntoIterator<Item = LexResult>,
source_path: &str,
) -> Result<Self, ParseError> {
match parse_filtered_tokens(lxr, Mode::Expression, source_path)? {
ast::Mod::Expression(m) => Ok(m),
_ => unreachable!("Mode::Module doesn't return other variant"),
}
}
}
impl Parse for ast::ModInteractive {
fn lex_starts_at(
source: &str,
offset: TextSize,
) -> SoftKeywordTransformer<Lexer<std::str::Chars>> {
lexer::lex_starts_at(source, Mode::Interactive, offset)
}
fn parse_tokens(
lxr: impl IntoIterator<Item = LexResult>,
source_path: &str,
) -> Result<Self, ParseError> {
match parse_filtered_tokens(lxr, Mode::Interactive, source_path)? {
ast::Mod::Interactive(m) => Ok(m),
_ => unreachable!("Mode::Module doesn't return other variant"),
}
}
}
impl Parse for ast::Suite {
fn lex_starts_at(
source: &str,
offset: TextSize,
) -> SoftKeywordTransformer<Lexer<std::str::Chars>> {
ast::ModModule::lex_starts_at(source, offset)
}
fn parse_tokens(
lxr: impl IntoIterator<Item = LexResult>,
source_path: &str,
) -> Result<Self, ParseError> {
Ok(ast::ModModule::parse_tokens(lxr, source_path)?.body)
}
}
impl Parse for ast::Stmt {
fn lex_starts_at(
source: &str,
offset: TextSize,
) -> SoftKeywordTransformer<Lexer<std::str::Chars>> {
ast::ModModule::lex_starts_at(source, offset)
}
fn parse_tokens(
lxr: impl IntoIterator<Item = LexResult>,
source_path: &str,
) -> Result<Self, ParseError> {
let mut statements = ast::ModModule::parse_tokens(lxr, source_path)?.body;
let statement = match statements.len() {
0 => {
return Err(ParseError {
error: ParseErrorType::Eof,
offset: TextSize::default(),
source_path: source_path.to_owned(),
})
}
1 => statements.pop().unwrap(),
_ => {
return Err(ParseError {
error: ParseErrorType::InvalidToken,
offset: statements[1].range().start(),
source_path: source_path.to_owned(),
})
}
};
Ok(statement)
}
}
impl Parse for ast::Expr {
fn lex_starts_at(
source: &str,
offset: TextSize,
) -> SoftKeywordTransformer<Lexer<std::str::Chars>> {
ast::ModExpression::lex_starts_at(source, offset)
}
fn parse_tokens(
lxr: impl IntoIterator<Item = LexResult>,
source_path: &str,
) -> Result<Self, ParseError> {
Ok(*ast::ModExpression::parse_tokens(lxr, source_path)?.body)
}
}
/// Parse a full Python program usually consisting of multiple lines.
///
@ -47,6 +217,7 @@ use rustpython_ast::OptionalRange;
/// let program = parser::parse_program(source, "<embedded>");
/// assert!(program.is_ok());
/// ```
#[deprecated = "Use ast::Suite::parse from rustpython_parser::Parse trait."]
pub fn parse_program(source: &str, source_path: &str) -> Result<ast::Suite, ParseError> {
parse(source, Mode::Module, source_path).map(|top| match top {
ast::Mod::Module(ast::ModModule { body, .. }) => body,
@ -71,8 +242,9 @@ pub fn parse_program(source: &str, source_path: &str) -> Result<ast::Suite, Pars
/// assert!(expr.is_ok());
///
/// ```
#[deprecated = "Use ast::Expr::parse from rustpython_parser::Parse trait."]
pub fn parse_expression(source: &str, path: &str) -> Result<ast::Expr, ParseError> {
parse_expression_starts_at(source, path, TextSize::default())
ast::Expr::parse(source, path)
}
/// Parses a Python expression from a given location.
@ -91,15 +263,13 @@ pub fn parse_expression(source: &str, path: &str) -> Result<ast::Expr, ParseErro
/// let expr = parse_expression_starts_at("1 + 2", "<embedded>", TextSize::from(400));
/// assert!(expr.is_ok());
/// ```
#[deprecated = "Use ast::Expr::parse_starts_at from rustpython_parser::Parse trait."]
pub fn parse_expression_starts_at(
source: &str,
path: &str,
offset: TextSize,
) -> Result<ast::Expr, ParseError> {
parse_starts_at(source, Mode::Expression, path, offset).map(|top| match top {
ast::Mod::Expression(ast::ModExpression { body, .. }) => *body,
_ => unreachable!(),
})
ast::Expr::parse_starts_at(source, path, offset)
}
/// Parse the given Python source code using the specified [`Mode`].
@ -188,12 +358,21 @@ pub fn parse_tokens(
lxr: impl IntoIterator<Item = LexResult>,
mode: Mode,
source_path: &str,
) -> Result<ast::Mod, ParseError> {
let lxr = lxr.into_iter();
#[cfg(feature = "full-lexer")]
let lxr =
lxr.filter_ok(|(tok, _)| !matches!(tok, Tok::Comment { .. } | Tok::NonLogicalNewline));
parse_filtered_tokens(lxr, mode, source_path)
}
fn parse_filtered_tokens(
lxr: impl IntoIterator<Item = LexResult>,
mode: Mode,
source_path: &str,
) -> Result<ast::Mod, ParseError> {
let marker_token = (Tok::start_marker(mode), Default::default());
let lexer = iter::once(Ok(marker_token)).chain(lxr);
#[cfg(feature = "full-lexer")]
let lexer =
lexer.filter_ok(|(tok, _)| !matches!(tok, Tok::Comment { .. } | Tok::NonLogicalNewline));
python::TopParser::new()
.parse(
lexer
@ -329,52 +508,53 @@ pub(super) fn optional_range(start: TextSize, end: TextSize) -> OptionalRange<Te
#[cfg(test)]
mod tests {
use super::*;
use crate::{ast, Parse};
#[test]
fn test_parse_empty() {
let parse_ast = parse_program("", "<test>").unwrap();
let parse_ast = ast::Suite::parse("", "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_string() {
let source = "'Hello world'";
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_f_string() {
let source = "f'Hello world'";
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_print_hello() {
let source = "print('Hello world')";
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_print_2() {
let source = "print('Hello world', 2)";
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_kwargs() {
let source = "my_func('positional', keyword=2)";
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_if_elif_else() {
let source = "if 1: 10\nelif 2: 20\nelse: 30";
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
@ -382,7 +562,7 @@ mod tests {
#[cfg(not(feature = "all-nodes-with-ranges"))]
fn test_parse_lambda() {
let source = "lambda x, y: x * y"; // lambda(x, y): x * y";
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
@ -390,7 +570,7 @@ mod tests {
fn test_parse_tuples() {
let source = "a, b = 4, 5";
insta::assert_debug_snapshot!(parse_program(source, "<test>").unwrap());
insta::assert_debug_snapshot!(ast::Suite::parse(source, "<test>").unwrap());
}
#[test]
@ -403,14 +583,14 @@ class Foo(A, B):
def method_with_default(self, arg='default'):
pass
";
insta::assert_debug_snapshot!(parse_program(source, "<test>").unwrap());
insta::assert_debug_snapshot!(ast::Suite::parse(source, "<test>").unwrap());
}
#[test]
#[cfg(not(feature = "all-nodes-with-ranges"))]
fn test_parse_dict_comprehension() {
let source = "{x1: x2 for y in z}";
let parse_ast = parse_expression(source, "<test>").unwrap();
let parse_ast = ast::Expr::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
@ -418,7 +598,7 @@ class Foo(A, B):
#[cfg(not(feature = "all-nodes-with-ranges"))]
fn test_parse_list_comprehension() {
let source = "[x for y in z]";
let parse_ast = parse_expression(source, "<test>").unwrap();
let parse_ast = ast::Expr::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
@ -426,7 +606,7 @@ class Foo(A, B):
#[cfg(not(feature = "all-nodes-with-ranges"))]
fn test_parse_double_list_comprehension() {
let source = "[x for y, y2 in z for a in b if a < 5 if a > 10]";
let parse_ast = parse_expression(source, "<test>").unwrap();
let parse_ast = ast::Expr::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
@ -434,7 +614,7 @@ class Foo(A, B):
#[cfg(not(feature = "all-nodes-with-ranges"))]
fn test_parse_generator_comprehension() {
let source = "(x for y in z)";
let parse_ast = parse_expression(source, "<test>").unwrap();
let parse_ast = ast::Expr::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
@ -442,7 +622,7 @@ class Foo(A, B):
#[cfg(not(feature = "all-nodes-with-ranges"))]
fn test_parse_named_expression_generator_comprehension() {
let source = "(x := y + 1 for y in z)";
let parse_ast = parse_expression(source, "<test>").unwrap();
let parse_ast = ast::Expr::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
@ -450,28 +630,28 @@ class Foo(A, B):
#[cfg(not(feature = "all-nodes-with-ranges"))]
fn test_parse_if_else_generator_comprehension() {
let source = "(x if y else y for y in z)";
let parse_ast = parse_expression(source, "<test>").unwrap();
let parse_ast = ast::Expr::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_bool_op_or() {
let source = "x or y";
let parse_ast = parse_expression(source, "<test>").unwrap();
let parse_ast = ast::Expr::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_bool_op_and() {
let source = "x and y";
let parse_ast = parse_expression(source, "<test>").unwrap();
let parse_ast = ast::Expr::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_slice() {
let source = "x[1:2:3]";
let parse_ast = parse_expression(source, "<test>").unwrap();
let parse_ast = ast::Expr::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
@ -506,7 +686,7 @@ with (0 as a,): pass
with (0 as a, 1 as b): pass
with (0 as a, 1 as b,): pass
";
insta::assert_debug_snapshot!(parse_program(source, "<test>").unwrap());
insta::assert_debug_snapshot!(ast::Suite::parse(source, "<test>").unwrap());
}
#[test]
@ -529,7 +709,7 @@ with (0 as a, 1 as b,): pass
"with a := 0 as x: pass",
"with (a := 0 as x): pass",
] {
assert!(parse_program(source, "<test>").is_err());
assert!(ast::Suite::parse(source, "<test>").is_err());
}
}
@ -541,7 +721,7 @@ array[0, *indexes, -1] = array_slice
array[*indexes_to_select, *indexes_to_select]
array[3:5, *indexes_to_select]
";
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
@ -555,13 +735,13 @@ array[3:5, *indexes_to_select]
("OFFSET %d" % offset) if offset else None,
)
)"#;
let parse_ast = parse_expression(source, "<test>").unwrap();
let parse_ast = ast::Expr::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_try() {
let parse_ast = parse_program(
let parse_ast = ast::Suite::parse(
r#"try:
raise ValueError(1)
except TypeError as e:
@ -576,7 +756,7 @@ except OSError as e:
#[test]
fn test_try_star() {
let parse_ast = parse_program(
let parse_ast = ast::Suite::parse(
r#"try:
raise ExceptionGroup("eg",
[ValueError(1), TypeError(2), OSError(3), OSError(4)])
@ -592,7 +772,7 @@ except* OSError as e:
#[test]
fn test_dict_unpacking() {
let parse_ast = parse_expression(r#"{"a": "b", **c, "d": "e"}"#, "<test>").unwrap();
let parse_ast = ast::Expr::parse(r#"{"a": "b", **c, "d": "e"}"#, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
@ -608,7 +788,7 @@ except* OSError as e:
#[test]
#[cfg(not(feature = "all-nodes-with-ranges"))]
fn test_match_as_identifier() {
let parse_ast = parse_program(
let parse_ast = ast::Suite::parse(
r#"
match *a + b, c # ((match * a) + b), c
match *(a + b), c # (match * (a + b)), c
@ -806,14 +986,14 @@ match w := x,:
case y as v,:
z = 0
"#;
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
#[cfg(not(feature = "all-nodes-with-ranges"))]
fn test_match() {
let parse_ast = parse_program(
let parse_ast = ast::Suite::parse(
r#"
match {"test": 1}:
case {
@ -844,7 +1024,7 @@ match x:
#[test]
#[cfg(not(feature = "all-nodes-with-ranges"))]
fn test_variadic_generics() {
let parse_ast = parse_program(
let parse_ast = ast::Suite::parse(
r#"
def args_to_tuple(*args: *Ts) -> Tuple[*Ts]: ...
"#,

12
parser/src/python.lalrpop
View file

@ -114,11 +114,11 @@ ExpressionStatement: ast::Stmt = {
let mut targets = vec![set_context(expression, ast::ExprContext::Store)];
let mut values = suffix;
while values.len() > 1 {
targets.push(set_context(values.remove(0), ast::ExprContext::Store));
}
let value = Box::new(values.pop().unwrap());
let value = Box::new(values.into_iter().next().unwrap());
for target in values {
targets.push(set_context(target, ast::ExprContext::Store));
}
ast::Stmt::Assign(
ast::StmtAssign { targets, value, type_comment: None, range: (location..end_location).into() }
@ -387,7 +387,9 @@ MatchStatement: ast::Stmt = {
}
MatchCase: ast::MatchCase = {
<start:@L> "case" <pattern:Patterns> <guard:(Guard)?> ":" <body:Suite> <end:@R> => {
<start:@L> "case" <pattern:Patterns> <guard:(Guard)?> ":" <body:Suite> => {
// SAFETY: `body` is never empty because it is non-optional and `Suite` matches one or more statements.
let end = body.last().unwrap().end();
ast::MatchCase {
pattern,
guard: guard.map(Box::new),

2972
parser/src/python.rs generated
View file

File diff suppressed because it is too large Load diff

50
parser/src/string.rs
View file

@ -7,7 +7,7 @@ use crate::text_size::TextRange;
use crate::{
ast::{self, Constant, Expr},
lexer::{LexicalError, LexicalErrorType},
parser::{parse_expression_starts_at, LalrpopError, ParseError, ParseErrorType},
parser::{LalrpopError, Parse, ParseError, ParseErrorType},
token::{StringKind, Tok},
};
use itertools::Itertools;
@ -591,7 +591,7 @@ impl<'a> StringParser<'a> {
fn parse_fstring_expr(source: &str, location: TextSize) -> Result<Expr, ParseError> {
let fstring_body = format!("({source})");
let start = location - TextSize::from(1);
parse_expression_starts_at(&fstring_body, "<fstring>", start)
ast::Expr::parse_starts_at(&fstring_body, "<fstring>", start)
}
fn parse_string(
@ -815,7 +815,7 @@ impl From<FStringError> for LalrpopError<TextSize, Tok, LexicalError> {
#[cfg(test)]
mod tests {
use super::*;
use crate::parser::parse_program;
use crate::{ast, Parse};
fn parse_fstring(source: &str) -> Result<Vec<Expr>, LexicalError> {
StringParser::new(
@ -957,63 +957,63 @@ mod tests {
#[test]
fn test_parse_string_concat() {
let source = "'Hello ' 'world'";
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_u_string_concat_1() {
let source = "'Hello ' u'world'";
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_u_string_concat_2() {
let source = "u'Hello ' 'world'";
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_f_string_concat_1() {
let source = "'Hello ' f'world'";
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_f_string_concat_2() {
let source = "'Hello ' f'world'";
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_f_string_concat_3() {
let source = "'Hello ' f'world{\"!\"}'";
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_u_f_string_concat_1() {
let source = "u'Hello ' f'world'";
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_u_f_string_concat_2() {
let source = "u'Hello ' f'world' '!'";
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_parse_string_triple_quotes_with_kind() {
let source = "u'''Hello, world!'''";
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
@ -1021,7 +1021,7 @@ mod tests {
fn test_single_quoted_byte() {
// single quote
let source = r##"b'\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !"#$%&\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff'"##;
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
@ -1029,7 +1029,7 @@ mod tests {
fn test_double_quoted_byte() {
// double quote
let source = r##"b"\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff""##;
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
@ -1037,35 +1037,35 @@ mod tests {
fn test_escape_char_in_byte_literal() {
// backslash does not escape
let source = r##"b"omkmok\Xaa""##; // spell-checker:ignore omkmok
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_raw_byte_literal_1() {
let source = r"rb'\x1z'";
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_raw_byte_literal_2() {
let source = r"rb'\\'";
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_escape_octet() {
let source = r##"b'\43a\4\1234'"##;
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_fstring_escaped_newline() {
let source = r#"f"\n{x}""#;
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
@ -1073,28 +1073,28 @@ mod tests {
fn test_fstring_unescaped_newline() {
let source = r#"f"""
{x}""""#;
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_fstring_escaped_character() {
let source = r#"f"\\{x}""#;
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_raw_fstring() {
let source = r#"rf"{x}""#;
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
#[test]
fn test_triple_quoted_raw_fstring() {
let source = r#"rf"""{x}""""#;
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
@ -1102,7 +1102,7 @@ mod tests {
fn test_fstring_line_continuation() {
let source = r#"rf"\
{x}""#;
let parse_ast = parse_program(source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
@ -1112,7 +1112,7 @@ mod tests {
#[test]
fn $name() {
let source = format!(r#""\N{{{0}}}""#, $alias);
let parse_ast = parse_program(&source, "<test>").unwrap();
let parse_ast = ast::Suite::parse(&source, "<test>").unwrap();
insta::assert_debug_snapshot!(parse_ast);
}
)*

4
scripts/update_asdl.sh
View file

@ -4,5 +4,5 @@ set -e
cd "$(dirname "$(dirname "$0")")"
# rm ast/src/gen/*.rs
python ast/asdl_rs.py --ast-dir ast/src/gen/ --module-file ../RustPython/vm/src/stdlib/ast/gen.rs ast/Python.asdl
rustfmt ast/src/gen/*.rs ../RustPython/vm/src/stdlib/ast/gen.rs
python ast/asdl_rs.py --ast-dir ast/src/gen/ --ast-pyo3-dir ast-pyo3/src/gen/ --module-file ../RustPython/vm/src/stdlib/ast/gen.rs ast/Python.asdl
rustfmt ast/src/gen/*.rs ast-pyo3/src/gen/*.rs ../RustPython/vm/src/stdlib/ast/gen.rs