diff --git a/Cargo.toml b/Cargo.toml index 8e43b18..8f0741c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,7 +11,7 @@ include = ["LICENSE", "Cargo.toml", "src/**/*.rs"] [workspace] resolver = "2" members = [ - "ast", "core", "format", "literal", "parser", + "ast", "core", "format", "literal", "parser", "parser-pyo3", "ruff_text_size", "ruff_source_location", ] diff --git a/parser-pyo3/.github/workflows/CI.yml b/parser-pyo3/.github/workflows/CI.yml new file mode 100644 index 0000000..a1a573a --- /dev/null +++ b/parser-pyo3/.github/workflows/CI.yml @@ -0,0 +1,120 @@ +# This file is autogenerated by maturin v0.15.1 +# To update, run +# +# maturin generate-ci github +# +name: CI + +on: + push: + branches: + - main + - master + tags: + - '*' + pull_request: + workflow_dispatch: + +permissions: + contents: read + +jobs: + linux: + runs-on: ubuntu-latest + strategy: + matrix: + target: [x86_64, x86, aarch64, armv7, s390x, ppc64le] + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: '3.10' + - name: Build wheels + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.target }} + args: --release --out dist --find-interpreter + sccache: 'true' + manylinux: auto + - name: Upload wheels + uses: actions/upload-artifact@v3 + with: + name: wheels + path: dist + + windows: + runs-on: windows-latest + strategy: + matrix: + target: [x64, x86] + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: '3.10' + architecture: ${{ matrix.target }} + - name: Build wheels + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.target }} + args: --release --out dist --find-interpreter + sccache: 'true' + - name: Upload wheels + uses: actions/upload-artifact@v3 + with: + name: wheels + path: dist + + macos: + runs-on: macos-latest + strategy: + matrix: + target: [x86_64, aarch64] + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: '3.10' + - name: Build wheels + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.target }} + args: --release --out dist --find-interpreter + sccache: 'true' + - name: Upload wheels + uses: actions/upload-artifact@v3 + with: + name: wheels + path: dist + + sdist: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Build sdist + uses: PyO3/maturin-action@v1 + with: + command: sdist + args: --out dist + - name: Upload sdist + uses: actions/upload-artifact@v3 + with: + name: wheels + path: dist + + release: + name: Release + runs-on: ubuntu-latest + if: "startsWith(github.ref, 'refs/tags/')" + needs: [linux, windows, macos, sdist] + steps: + - uses: actions/download-artifact@v3 + with: + name: wheels + - name: Publish to PyPI + uses: PyO3/maturin-action@v1 + env: + MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_API_TOKEN }} + with: + command: upload + args: --skip-existing * diff --git a/parser-pyo3/.gitignore b/parser-pyo3/.gitignore new file mode 100644 index 0000000..af3ca5e --- /dev/null +++ b/parser-pyo3/.gitignore @@ -0,0 +1,72 @@ +/target + +# Byte-compiled / optimized / DLL files +__pycache__/ +.pytest_cache/ +*.py[cod] + +# C extensions +*.so + +# Distribution / packaging +.Python +.venv/ +env/ +bin/ +build/ +develop-eggs/ +dist/ +eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +include/ +man/ +venv/ +*.egg-info/ +.installed.cfg +*.egg + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt +pip-selfcheck.json + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.cache +nosetests.xml +coverage.xml + +# Translations +*.mo + +# Mr Developer +.mr.developer.cfg +.project +.pydevproject + +# Rope +.ropeproject + +# Django stuff: +*.log +*.pot + +.DS_Store + +# Sphinx documentation +docs/_build/ + +# PyCharm +.idea/ + +# VSCode +.vscode/ + +# Pyenv +.python-version \ No newline at end of file diff --git a/parser-pyo3/Cargo.toml b/parser-pyo3/Cargo.toml new file mode 100644 index 0000000..7f655fa --- /dev/null +++ b/parser-pyo3/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "rustpython-parser-pyo3" +version = "0.2.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[lib] +name = "rustpython_parser_pyo3" +crate-type = ["cdylib"] + +[dependencies] +rustpython-parser = { workspace = true } +rustpython-ast = { workspace = true, features = ["pyo3"]} +pyo3 = { version = "0.18.3", features = ["num-bigint", "num-complex"] } diff --git a/parser-pyo3/baembal.py b/parser-pyo3/baembal.py new file mode 100644 index 0000000..f0e53ad --- /dev/null +++ b/parser-pyo3/baembal.py @@ -0,0 +1,21 @@ +"""뱀발, 畵蛇添足, The legs on the snake. + +To make it sprint. +Let's start with walking. +""" + +import ast +import _ast + +import rustpython_parser_pyo3 + +orig = _ast.AST + +class ASTType(type): + def __instancecheck__(self, instance): + return isinstance(instance, (orig, rustpython_parser_pyo3.unlocated_ast.AST)) + +class AST(ast.AST, metaclass=ASTType): + pass + +ast.AST = AST diff --git a/parser-pyo3/bench1.py b/parser-pyo3/bench1.py new file mode 100644 index 0000000..0e42d6b --- /dev/null +++ b/parser-pyo3/bench1.py @@ -0,0 +1,62 @@ +import baembal + +import sys +import ast +import _ast + +ast.AST = baembal.AST +_ast.AST = baembal.AST + +import ast as py_ast +import rustpython_parser_pyo3 as rust_ast + + +dump = ast.dump + +import timeit + +from glob import glob + + +files = {} +for path in glob("../../cpython/Lib/**/*.py"): + try: + txt = open(path, 'r').read() + except UnicodeDecodeError: + continue + # try: + # if py_ast.dump(py_ast.parse(txt)) != py_ast.dump(rust_ast.parse(txt)): + # continue + # except SyntaxError: + # continue + files[path] = txt + + +t1 = 0.0 +t2 = 0.0 +t3 = 0.0 + +for path, txt in files.items(): + try: + txt = open(path, 'r').read() + except UnicodeDecodeError: + continue + + # p = py_ast.parse(txt) + # r = rust_ast.parse(txt) + + # compile(p, 'x', 'exec') + # compile(r, 'x', 'exec') + + # break + try: + p = timeit.timeit(lambda: dump(py_ast.parse(txt)), number=10) + r1 = timeit.timeit(lambda: dump(rust_ast.parse(txt)), number=10) + r2 = timeit.timeit(lambda: dump(rust_ast.parse_wrap(txt)), number=10) + except Exception as e: + print('error:', path, e) + continue + t1 += p + t2 += r1 + t3 += r2 + print(f'accum: {t1:.2f} {t2:.2f} {t3:.2f} {path}') diff --git a/parser-pyo3/bench2.py b/parser-pyo3/bench2.py new file mode 100644 index 0000000..4fcbb66 --- /dev/null +++ b/parser-pyo3/bench2.py @@ -0,0 +1,27 @@ +import sys +import baembal + +import ast +ast.AST = baembal.AST + +from ast import dump + + +arg = sys.argv[1] # python or rustpython +if arg == "python": + import ast +elif arg == "rustpython": + import rustpython_parser_pyo3 as ast +else: + assert False + +from glob import glob + +for path in glob("../../cpython/Lib/*.py"): + try: + txt = open(path, 'r').read() + except UnicodeDecodeError: + continue + m = ast.parse(txt) + code = compile(m, path, 'exec') + # d = dump(m, indent=True) diff --git a/parser-pyo3/pyproject.toml b/parser-pyo3/pyproject.toml new file mode 100644 index 0000000..98cad9a --- /dev/null +++ b/parser-pyo3/pyproject.toml @@ -0,0 +1,16 @@ +[build-system] +requires = ["maturin>=0.15,<0.16"] +build-backend = "maturin" + +[project] +name = "rustpython-parser" +requires-python = ">=3.7" +classifiers = [ + "Programming Language :: Rust", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", +] + + +[tool.maturin] +features = ["pyo3/extension-module"] diff --git a/parser-pyo3/src/lib.rs b/parser-pyo3/src/lib.rs new file mode 100644 index 0000000..5e01cc8 --- /dev/null +++ b/parser-pyo3/src/lib.rs @@ -0,0 +1,46 @@ +use pyo3::prelude::*; + +use rustpython_ast::{ + pyo3::{ToPyo3Ast, ToPyo3Wrapper}, + Fold, +}; + +#[pyfunction] +fn parse_wrap(a: &str, py: Python) -> PyResult { + let parsed = rustpython_parser::parse(a, rustpython_parser::Mode::Module, "") + .map_err(|e| PyErr::new::(e.to_string()))?; + // let parsed = rustpython_ast::source_code::SourceLocator::new(a).fold(parsed).unwrap(); + let parsed = parsed.module().unwrap(); + let ast_module = PyModule::import(py, "_ast")?; + + let parsed = Box::leak(Box::new(parsed)); + parsed.to_pyo3_wrapper(py) +} + +#[pyfunction] +fn parse(a: &str, py: Python) -> PyResult { + use rustpython_parser::{ast::fold::Fold, source_code::SourceLocator}; + let parsed = rustpython_parser::parse(a, rustpython_parser::Mode::Module, "") + .map_err(|e| PyErr::new::(e.to_string()))? + // .module().unwrap() + ; + // let located = SourceLocator::new(a).fold(parsed).unwrap(); + let x = parsed.module().unwrap(); + x.to_pyo3_ast(py) +} + +#[pymodule] +fn rustpython_parser_pyo3(py: Python, m: &PyModule) -> PyResult<()> { + // let ast = PyModule::new(py, "ast")?; + // rustpython_ast::pyo3::located::add_to_module(py, ast)?; + // m.add_submodule(ast)?; + + let ast = PyModule::new(py, "unlocated_ast")?; + rustpython_ast::pyo3::ranged::add_to_module(py, ast)?; + m.add_submodule(ast)?; + + m.add_function(wrap_pyfunction!(parse, m)?)?; + m.add_function(wrap_pyfunction!(parse_wrap, m)?)?; + + Ok(()) +} diff --git a/parser-pyo3/test_ast.py b/parser-pyo3/test_ast.py new file mode 100644 index 0000000..67239b5 --- /dev/null +++ b/parser-pyo3/test_ast.py @@ -0,0 +1,47 @@ +import re +import baembal + +import ast + +ast.AST = baembal.AST + +import pytest + +import ast as py_ast +import rustpython_parser_pyo3 as rust_ast + + +from glob import glob + +files = {} +for path in glob("../../cpython/Lib/**/*.py"): + try: + txt = open(path, "r").read() + except UnicodeDecodeError: + continue + # try: + # if py_ast.dump(py_ast.parse(txt)) != py_ast.dump(rust_ast.parse(txt)): + # continue + # except SyntaxError: + # continue + files[path] = txt + + +@pytest.mark.parametrize("path", files.keys()) +def test_roundtrip(path): + txt = files[path] + module_p = py_ast.parse(txt) + dump_p = py_ast.dump(module_p, indent=True) + module_r = rust_ast.parse(txt) + dump_r = py_ast.dump(module_r, indent=True) + p = re.compile("object at 0x[0-9a-f]+") + dump_p2 = re.sub(p, "object at 0x????????", dump_p) + dump_r2 = re.sub(p, "object at 0x????????", dump_r) + try: + assert dump_p2 == dump_r2, dump_r2 + except AssertionError: + with open("dump_p.txt", "w") as f: + f.write(dump_p2) + with open("dump_r.txt", "w") as f: + f.write(dump_r2) + raise