mirrors/RustPython-Parser
1
0
Fork 0
mirror of https://github.com/RustPython/Parser.git synced 2025-07-07 13:15:21 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Remove ast-pyo3 from repository

Browse source
This commit is contained in:
Jeong YunWon 2024-04-07 02:34:35 +09:00
parent 9ce55aefde
commit 37a1190dfb
12 changed files with 0 additions and 14656 deletions
Download patch file Download diff file Expand all files Collapse all files

1
Cargo.toml
View file

@ -12,7 +12,6 @@ include = ["LICENSE", "Cargo.toml", "src/**/*.rs"]
resolver = "2"
members = [
"ast", "core", "format", "literal", "parser", "vendored",
"ast-pyo3",
]
[workspace.dependencies]

72
ast-pyo3/.gitignore vendored
View file

@ -1,72 +0,0 @@
/target
# Byte-compiled / optimized / DLL files
__pycache__/
.pytest_cache/
*.py[cod]
# C extensions
*.so
# Distribution / packaging
.Python
.venv/
env/
bin/
build/
develop-eggs/
dist/
eggs/
lib/
lib64/
parts/
sdist/
var/
include/
man/
venv/
*.egg-info/
.installed.cfg
*.egg
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
pip-selfcheck.json
# Unit test / coverage reports
htmlcov/
.tox/
.coverage
.cache
nosetests.xml
coverage.xml
# Translations
*.mo
# Mr Developer
.mr.developer.cfg
.project
.pydevproject
# Rope
.ropeproject
# Django stuff:
*.log
*.pot
.DS_Store
# Sphinx documentation
docs/_build/
# PyCharm
.idea/
# VSCode
.vscode/
# Pyenv
.python-version

24
ast-pyo3/Cargo.toml
View file

@ -1,24 +0,0 @@
[package]
name = "rustpython-ast-pyo3"
version = "0.0.1"
edition = { workspace = true }
rust-version = { workspace = true }
[features]
abi3 = ["pyo3/abi3-py37"] # will be supported from next pyo3 version
# This feature is experimental
# It reimplements AST types, but currently both slower than python AST types and limited to use in other API
wrapper = []
[lib]
name = "rustpython_ast"
crate-type = ["cdylib"]
[dependencies]
rustpython-ast = { workspace = true, features = ["location"] }
rustpython-parser = { workspace = true, features = ["num-bigint"] }
num-complex = { workspace = true }
num-traits = { workspace = true }
once_cell = { workspace = true }
pyo3 = { workspace = true, features = ["num-bigint", "num-complex"] }

15
ast-pyo3/pyproject.toml
View file

@ -1,15 +0,0 @@
[build-system]
requires = ["maturin>=0.15,<0.16"]
build-backend = "maturin"
[project]
name = "rustpython_ast"
requires-python = ">=3.7"
classifiers = [
"Programming Language :: Rust",
"Programming Language :: Python :: Implementation :: CPython",
]
[tool.maturin]
# module-name = "_rustpython_ast"
features = ["pyo3/extension-module"]

3
ast-pyo3/rustpython_ast/__init__.py
View file

@ -1,3 +0,0 @@
from rustpython_ast.rustpython_ast import parse
__all__ = ["parse"]

5100
ast-pyo3/src/gen/to_py_ast.rs
View file

File diff suppressed because it is too large Load diff

4630
ast-pyo3/src/gen/wrapper_located.rs
View file

File diff suppressed because it is too large Load diff

4334
ast-pyo3/src/gen/wrapper_ranged.rs
View file

File diff suppressed because it is too large Load diff

49
ast-pyo3/src/lib.rs
View file

@ -1,49 +0,0 @@
mod py_ast;
#[cfg(feature = "wrapper")]
pub mod wrapper;
pub use py_ast::{init, PyNode, ToPyAst};
use pyo3::prelude::*;
use rustpython_parser::ast::{source_code::LinearLocator, Fold};
#[pyfunction]
#[pyo3(signature = (source, filename="<unknown>", *, type_comments=false, locate=true))]
pub fn parse<'py>(
source: &str,
filename: &str,
type_comments: bool,
locate: bool,
py: Python<'py>,
) -> PyResult<&'py PyAny> {
if type_comments {
todo!("'type_comments' is not implemented yet");
}
let parsed = rustpython_parser::parse(source, rustpython_parser::Mode::Module, filename)
.map_err(|e| PyErr::new::<pyo3::exceptions::PySyntaxError, _>(e.to_string()))?;
if locate {
let parsed = LinearLocator::new(source).fold(parsed).unwrap();
parsed.module().unwrap().to_py_ast(py)
} else {
parsed.module().unwrap().to_py_ast(py)
}
}
#[pymodule]
fn rustpython_ast(py: Python, m: &PyModule) -> PyResult<()> {
py_ast::init(py)?;
#[cfg(feature = "wrapper")]
{
let ast = PyModule::new(py, "ast")?;
wrapper::located::add_to_module(py, ast)?;
m.add_submodule(ast)?;
let ast = PyModule::new(py, "unlocated_ast")?;
wrapper::ranged::add_to_module(py, ast)?;
m.add_submodule(ast)?;
}
m.add_function(wrap_pyfunction!(parse, m)?)?;
Ok(())
}

219
ast-pyo3/src/py_ast.rs
View file

@ -1,219 +0,0 @@
use num_complex::Complex64;
use num_traits::cast::ToPrimitive;
use once_cell::sync::OnceCell;
use pyo3::{
prelude::*,
types::{PyBool, PyBytes, PyList, PyString, PyTuple},
ToPyObject,
};
use rustpython_ast::{
self as ast, source_code::SourceRange, text_size::TextRange, ConversionFlag, Node,
};
pub trait PyNode {
fn py_type_cache() -> &'static OnceCell<(Py<PyAny>, Py<PyAny>)> {
{
static PY_TYPE: OnceCell<(Py<PyAny>, Py<PyAny>)> = OnceCell::new();
&PY_TYPE
}
}
}
pub trait ToPyAst {
fn to_py_ast<'py>(&self, py: Python<'py>) -> PyResult<&'py PyAny>;
}
impl<T: ToPyAst> ToPyAst for Box<T> {
#[inline]
fn to_py_ast<'py>(&self, py: Python<'py>) -> PyResult<&'py PyAny> {
(**self).to_py_ast(py)
}
}
impl<T: ToPyAst> ToPyAst for Option<T> {
#[inline]
fn to_py_ast<'py>(&self, py: Python<'py>) -> PyResult<&'py PyAny> {
match self {
Some(ast) => ast.to_py_ast(py),
None => Ok(ast_cache().none_ref(py)),
}
}
}
impl<T: ToPyAst> ToPyAst for Vec<T> {
fn to_py_ast<'py>(&self, py: Python<'py>) -> PyResult<&'py PyAny> {
let elts = self
.iter()
.map(|item| item.to_py_ast(py))
.collect::<Result<Vec<_>, _>>()?;
let list = PyList::new(py, elts);
Ok(list.into())
}
}
impl ToPyAst for ast::Identifier {
#[inline]
fn to_py_ast<'py>(&self, py: Python<'py>) -> PyResult<&'py PyAny> {
Ok(PyString::new(py, self.as_str()).into())
}
}
impl ToPyAst for ast::String {
#[inline]
fn to_py_ast<'py>(&self, py: Python<'py>) -> PyResult<&'py PyAny> {
Ok(PyString::new(py, self.as_str()).into())
}
}
impl ToPyAst for bool {
#[inline]
fn to_py_ast<'py>(&self, py: Python<'py>) -> PyResult<&'py PyAny> {
Ok(ast_cache().bool_int(py, *self))
}
}
impl ToPyAst for ConversionFlag {
#[inline]
fn to_py_ast<'py>(&self, py: Python<'py>) -> PyResult<&'py PyAny> {
Ok(ast_cache().conversion_flag(py, *self))
}
}
impl<R> ToPyAst for ast::Arguments<R>
where
R: Clone,
ast::PythonArguments<R>: ToPyAst,
{
#[inline]
fn to_py_ast<'py>(&self, py: Python<'py>) -> PyResult<&'py PyAny> {
let arguments = self.to_python_arguments();
arguments.to_py_ast(py)
}
}
fn constant_to_object(constant: &ast::Constant, py: Python) -> PyObject {
let cache = ast_cache();
match constant {
ast::Constant::None => cache.none.clone_ref(py),
ast::Constant::Bool(bool) => cache.bool(py, *bool).into(),
ast::Constant::Str(string) => string.to_object(py),
ast::Constant::Bytes(bytes) => PyBytes::new(py, bytes).into(),
ast::Constant::Int(int) => match int.to_i64() {
Some(small_int) => small_int.to_object(py),
None => int.to_object(py),
},
ast::Constant::Tuple(elts) => {
let elts: Vec<_> = elts.iter().map(|c| constant_to_object(c, py)).collect();
PyTuple::new(py, elts).into()
}
ast::Constant::Float(f64) => f64.to_object(py),
ast::Constant::Complex { real, imag } => Complex64::new(*real, *imag).to_object(py),
ast::Constant::Ellipsis => py.Ellipsis(),
}
}
#[pyclass(module = "rustpython_ast", subclass)]
pub struct Ast;
#[pymethods]
impl Ast {
#[new]
fn new() -> Self {
Self
}
}
fn cache_py_type<N: PyNode + Node>(ast_module: &PyAny) -> PyResult<()> {
let class = ast_module.getattr(N::NAME)?;
let base = if std::mem::size_of::<N>() == 0 {
class.call0()?
} else {
class.getattr("__new__")?
};
N::py_type_cache().get_or_init(|| (class.into(), base.into()));
Ok(())
}
// TODO: This cache must be bound to 'py
struct AstCache {
lineno: Py<PyString>,
col_offset: Py<PyString>,
end_lineno: Py<PyString>,
end_col_offset: Py<PyString>,
none: Py<PyAny>,
bool_values: (Py<PyBool>, Py<PyBool>),
bool_int_values: (Py<PyAny>, Py<PyAny>),
conversion_flags: (Py<PyAny>, Py<PyAny>, Py<PyAny>, Py<PyAny>),
}
impl AstCache {
// fn location_vec<'py>(&'static self, py: Python<'py>, range: &SourceRange) -> &'py PyDict {
// let attributes = PyDict::new(py);
// attributes.set_item(self.lineno.as_ref(py), range.start.row.get()).unwrap();
// attributes.set_item(self.col_offset.as_ref(py), range.start.column.to_zero_indexed()).unwrap();
// if let Some(end) = range.end {
// attributes.set_item(self.end_lineno.as_ref(py), end.row.get()).unwrap();
// attributes.set_item(
// self.end_col_offset.as_ref(py),
// end.column.to_zero_indexed(),
// ).unwrap();
// }
// attributes
// }
#[inline]
fn none_ref<'py>(&'static self, py: Python<'py>) -> &'py PyAny {
Py::<PyAny>::as_ref(&self.none, py)
}
#[inline]
fn bool_int<'py>(&'static self, py: Python<'py>, value: bool) -> &'py PyAny {
let v = &self.bool_int_values;
Py::<PyAny>::as_ref(if value { &v.1 } else { &v.0 }, py)
}
#[inline]
fn bool(&'static self, py: Python, value: bool) -> Py<PyBool> {
let v = &self.bool_values;
(if value { &v.1 } else { &v.0 }).clone_ref(py)
}
fn conversion_flag<'py>(&'static self, py: Python<'py>, value: ConversionFlag) -> &'py PyAny {
let v = &self.conversion_flags;
match value {
ConversionFlag::None => v.0.as_ref(py),
ConversionFlag::Str => v.1.as_ref(py),
ConversionFlag::Ascii => v.2.as_ref(py),
ConversionFlag::Repr => v.3.as_ref(py),
}
}
}
fn ast_cache_cell() -> &'static OnceCell<AstCache> {
{
static PY_TYPE: OnceCell<AstCache> = OnceCell::new();
&PY_TYPE
}
}
fn ast_cache() -> &'static AstCache {
ast_cache_cell().get().unwrap()
}
pub fn init(py: Python) -> PyResult<()> {
ast_cache_cell().get_or_init(|| AstCache {
lineno: pyo3::intern!(py, "lineno").into_py(py),
col_offset: pyo3::intern!(py, "col_offset").into_py(py),
end_lineno: pyo3::intern!(py, "end_lineno").into_py(py),
end_col_offset: pyo3::intern!(py, "end_col_offset").into_py(py),
none: py.None(),
bool_values: (PyBool::new(py, false).into(), PyBool::new(py, true).into()),
bool_int_values: ((0).to_object(py), (1).to_object(py)),
conversion_flags: (
(-1).to_object(py),
(b's').to_object(py),
(b'a').to_object(py),
(b'r').to_object(py),
),
});
init_types(py)
}
include!("gen/to_py_ast.rs");

153
ast-pyo3/src/wrapper.rs
View file

@ -1,153 +0,0 @@
use crate::PyNode;
use num_complex::Complex64;
use pyo3::prelude::*;
use pyo3::types::{PyBytes, PyList, PyTuple};
use rustpython_ast::{
self as ast, source_code::SourceRange, text_size::TextRange, ConversionFlag, Node,
};
pub trait ToPyWrapper {
fn to_py_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>>;
}
impl<T: ToPyWrapper> ToPyWrapper for Box<T> {
#[inline]
fn to_py_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
(**self).to_py_wrapper(py)
}
}
impl<T: ToPyWrapper> ToPyWrapper for Option<T> {
#[inline]
fn to_py_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
match self {
Some(ast) => ast.to_py_wrapper(py),
None => Ok(py.None()),
}
}
}
impl ToPyWrapper for ast::Identifier {
#[inline]
fn to_py_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
Ok(self.as_str().to_object(py))
}
}
impl ToPyWrapper for ast::String {
#[inline]
fn to_py_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
Ok(self.as_str().to_object(py))
}
}
impl ToPyWrapper for ast::Int {
#[inline]
fn to_py_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
Ok((self.to_u32()).to_object(py))
}
}
impl ToPyWrapper for bool {
#[inline]
fn to_py_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
Ok((*self as u32).to_object(py))
}
}
impl ToPyWrapper for ConversionFlag {
#[inline]
fn to_py_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
Ok((*self as i8).to_object(py))
}
}
impl ToPyWrapper for ast::Constant {
fn to_py_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
let value = match self {
ast::Constant::None => py.None(),
ast::Constant::Bool(bool) => bool.to_object(py),
ast::Constant::Str(string) => string.to_object(py),
ast::Constant::Bytes(bytes) => PyBytes::new(py, bytes).into(),
ast::Constant::Int(int) => int.to_object(py),
ast::Constant::Tuple(elts) => {
let elts: PyResult<Vec<_>> = elts.iter().map(|c| c.to_py_wrapper(py)).collect();
PyTuple::new(py, elts?).into()
}
ast::Constant::Float(f64) => f64.to_object(py),
ast::Constant::Complex { real, imag } => Complex64::new(*real, *imag).to_object(py),
ast::Constant::Ellipsis => py.Ellipsis(),
};
Ok(value)
}
}
impl<T: ToPyWrapper> ToPyWrapper for Vec<T> {
fn to_py_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
let list = PyList::empty(py);
for item in self {
let py_item = item.to_py_wrapper(py)?;
list.append(py_item)?;
}
Ok(list.into())
}
}
impl<R> ToPyWrapper for ast::Arguments<R>
where
Self: Clone,
ast::PythonArguments<R>: ToPyWrapper,
{
#[inline]
fn to_py_wrapper(&'static self, _py: Python) -> PyResult<Py<PyAny>> {
todo!()
// Ok(FunctionArguments(self).to_object(py))
}
}
#[pyclass(module = "rustpython_ast", name = "AST", subclass)]
pub struct Ast;
#[pymethods]
impl Ast {
#[new]
fn new() -> Self {
Self
}
}
pub mod located {
pub use super::Ast;
use super::*;
include!("gen/wrapper_located.rs");
}
pub mod ranged {
pub use super::Ast;
use super::*;
include!("gen/wrapper_ranged.rs");
}
fn init_type<P: pyo3::PyClass, N: PyNode + Node>(py: Python, m: &PyModule) -> PyResult<()> {
m.add_class::<P>()?;
let node = m.getattr(P::NAME)?;
if P::NAME != N::NAME {
// TODO: no idea how to escape rust keyword on #[pyclass]
m.setattr(P::NAME, node)?;
}
let names: Vec<&'static str> = N::FIELD_NAMES.to_vec();
let fields = PyTuple::new(py, names);
node.setattr("_fields", fields)?;
Ok(())
}
/// A Python module implemented in Rust.
fn init_module(py: Python, m: &PyModule) -> PyResult<()> {
m.add_class::<Ast>()?;
let ast = m.getattr("AST")?;
let fields = PyTuple::empty(py);
ast.setattr("_fields", fields)?;
Ok(())
}

56
ast-pyo3/test_ast.py
View file

@ -1,56 +0,0 @@
import re
import difflib
import pytest
import ast as py_ast
import rustpython_ast as rust_ast
from glob import glob
files = {}
for path in glob("../../cpython/Lib/**/*.py"):
try:
txt = open(path, "r").read()
except UnicodeDecodeError:
continue
# try:
# if py_ast.dump(py_ast.parse(txt)) != py_ast.dump(rust_ast.parse(txt)):
# continue
# except SyntaxError:
# continue
files[path] = txt
@pytest.mark.parametrize("path", files.keys())
def test_roundtrip(path):
txt = files[path]
module_p = py_ast.parse(txt)
dump_p = py_ast.dump(module_p, indent=True)
module_r = rust_ast.parse(txt)
dump_r = py_ast.dump(module_r, indent=True)
p = re.compile("object at 0x[0-9a-f]+")
dump_p2 = re.sub(p, "object at 0x????????", dump_p)
dump_r2 = re.sub(p, "object at 0x????????", dump_r)
try:
assert dump_p2 == dump_r2
except AssertionError:
last_sign = ' '
for s in difflib.ndiff(dump_p2, dump_r2):
if s[0]==' ': continue
if s[0] == last_sign:
print(s[2:], end='')
else:
print()
print(s, end='')
last_sign = s[0]
# with open("dump_code.py", "w") as f:
# f.write(path)
# f.write('\n')
# f.write(txt)
# with open("dump_p.txt", "w") as f:
# f.write(dump_p2)
# with open("dump_r.txt", "w") as f:
# f.write(dump_r2)
raise