parser-pyo3

Cargo.toml

@@ -11,7 +11,7 @@ include = ["LICENSE", "Cargo.toml", "src/**/*.rs"]
 [workspace]
 resolver = "2"
 members = [
-    "ast", "core", "format", "literal", "parser",
+    "ast", "core", "format", "literal", "parser", "parser-pyo3",
     "ruff_text_size", "ruff_source_location",
 ]
 

parser-pyo3/.github/workflows/CI.yml

@@ -0,0 +1,120 @@
+# This file is autogenerated by maturin v0.15.1
+# To update, run
+#
+#    maturin generate-ci github
+#
+name: CI
+
+on:
+  push:
+    branches:
+      - main
+      - master
+    tags:
+      - '*'
+  pull_request:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  linux:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        target: [x86_64, x86, aarch64, armv7, s390x, ppc64le]
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+      - name: Build wheels
+        uses: PyO3/maturin-action@v1
+        with:
+          target: ${{ matrix.target }}
+          args: --release --out dist --find-interpreter
+          sccache: 'true'
+          manylinux: auto
+      - name: Upload wheels
+        uses: actions/upload-artifact@v3
+        with:
+          name: wheels
+          path: dist
+
+  windows:
+    runs-on: windows-latest
+    strategy:
+      matrix:
+        target: [x64, x86]
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+          architecture: ${{ matrix.target }}
+      - name: Build wheels
+        uses: PyO3/maturin-action@v1
+        with:
+          target: ${{ matrix.target }}
+          args: --release --out dist --find-interpreter
+          sccache: 'true'
+      - name: Upload wheels
+        uses: actions/upload-artifact@v3
+        with:
+          name: wheels
+          path: dist
+
+  macos:
+    runs-on: macos-latest
+    strategy:
+      matrix:
+        target: [x86_64, aarch64]
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+      - name: Build wheels
+        uses: PyO3/maturin-action@v1
+        with:
+          target: ${{ matrix.target }}
+          args: --release --out dist --find-interpreter
+          sccache: 'true'
+      - name: Upload wheels
+        uses: actions/upload-artifact@v3
+        with:
+          name: wheels
+          path: dist
+
+  sdist:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Build sdist
+        uses: PyO3/maturin-action@v1
+        with:
+          command: sdist
+          args: --out dist
+      - name: Upload sdist
+        uses: actions/upload-artifact@v3
+        with:
+          name: wheels
+          path: dist
+
+  release:
+    name: Release
+    runs-on: ubuntu-latest
+    if: "startsWith(github.ref, 'refs/tags/')"
+    needs: [linux, windows, macos, sdist]
+    steps:
+      - uses: actions/download-artifact@v3
+        with:
+          name: wheels
+      - name: Publish to PyPI
+        uses: PyO3/maturin-action@v1
+        env:
+          MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_API_TOKEN }}
+        with:
+          command: upload
+          args: --skip-existing *

parser-pyo3/.gitignore

@@ -0,0 +1,72 @@
+/target
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+.pytest_cache/
+*.py[cod]
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+.venv/
+env/
+bin/
+build/
+develop-eggs/
+dist/
+eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+include/
+man/
+venv/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+pip-selfcheck.json
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.cache
+nosetests.xml
+coverage.xml
+
+# Translations
+*.mo
+
+# Mr Developer
+.mr.developer.cfg
+.project
+.pydevproject
+
+# Rope
+.ropeproject
+
+# Django stuff:
+*.log
+*.pot
+
+.DS_Store
+
+# Sphinx documentation
+docs/_build/
+
+# PyCharm
+.idea/
+
+# VSCode
+.vscode/
+
+# Pyenv
+.python-version

parser-pyo3/Cargo.toml

@@ -0,0 +1,14 @@
+[package]
+name = "rustpython-parser-pyo3"
+version = "0.2.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+[lib]
+name = "rustpython_parser_pyo3"
+crate-type = ["cdylib"]
+
+[dependencies]
+rustpython-parser = { workspace = true }
+rustpython-ast = { workspace = true, features = ["pyo3"]}
+pyo3 = { version = "0.18.3", features = ["num-bigint", "num-complex"] }

parser-pyo3/baembal.py

@@ -0,0 +1,21 @@
+"""뱀발, 畵蛇添足, The legs on the snake.
+
+To make it sprint.
+Let's start with walking.
+"""
+
+import ast
+import _ast
+
+import rustpython_parser_pyo3
+
+orig = _ast.AST
+        
+class ASTType(type):
+    def __instancecheck__(self, instance):
+        return isinstance(instance, (orig, rustpython_parser_pyo3.unlocated_ast.AST))
+
+class AST(ast.AST, metaclass=ASTType):
+    pass
+    
+ast.AST = AST

parser-pyo3/bench1.py

@@ -0,0 +1,62 @@
+import baembal
+
+import sys
+import ast
+import _ast
+
+ast.AST = baembal.AST
+_ast.AST = baembal.AST
+
+import ast as py_ast
+import rustpython_parser_pyo3 as rust_ast
+
+
+dump = ast.dump
+
+import timeit
+
+from glob import glob
+
+
+files = {}
+for path in glob("../../cpython/Lib/**/*.py"):
+    try:
+        txt = open(path, 'r').read()
+    except UnicodeDecodeError:
+        continue
+    # try:
+    #     if py_ast.dump(py_ast.parse(txt)) != py_ast.dump(rust_ast.parse(txt)):
+    #         continue
+    # except SyntaxError:
+    #     continue
+    files[path] = txt
+
+
+t1 = 0.0
+t2 = 0.0
+t3 = 0.0
+
+for path, txt in files.items():
+    try:
+        txt = open(path, 'r').read()
+    except UnicodeDecodeError:
+        continue
+
+    # p = py_ast.parse(txt)
+    # r = rust_ast.parse(txt)
+
+    # compile(p, 'x', 'exec')
+    # compile(r, 'x', 'exec')
+
+    # break
+    try:
+        p = timeit.timeit(lambda: dump(py_ast.parse(txt)), number=10)
+        r1 = timeit.timeit(lambda: dump(rust_ast.parse(txt)), number=10)
+        r2 = timeit.timeit(lambda: dump(rust_ast.parse_wrap(txt)), number=10)
+    except Exception as e:
+        print('error:', path, e)
+        continue
+    t1 += p
+    t2 += r1
+    t3 += r2
+    print(f'accum: {t1:.2f} {t2:.2f} {t3:.2f} {path}')

parser-pyo3/bench2.py

@@ -0,0 +1,27 @@
+import sys
+import baembal
+
+import ast
+ast.AST = baembal.AST
+
+from ast import dump
+
+
+arg = sys.argv[1]  # python or rustpython
+if arg == "python":
+    import ast
+elif arg == "rustpython":
+    import rustpython_parser_pyo3 as ast
+else:
+    assert False
+
+from glob import glob
+
+for path in glob("../../cpython/Lib/*.py"):
+    try:
+        txt = open(path, 'r').read()
+    except UnicodeDecodeError:
+        continue
+    m = ast.parse(txt)
+    code = compile(m, path, 'exec')
+    # d = dump(m, indent=True)

parser-pyo3/pyproject.toml

@@ -0,0 +1,16 @@
+[build-system]
+requires = ["maturin>=0.15,<0.16"]
+build-backend = "maturin"
+
+[project]
+name = "rustpython-parser"
+requires-python = ">=3.7"
+classifiers = [
+    "Programming Language :: Rust",
+    "Programming Language :: Python :: Implementation :: CPython",
+    "Programming Language :: Python :: Implementation :: PyPy",
+]
+
+
+[tool.maturin]
+features = ["pyo3/extension-module"]

parser-pyo3/src/lib.rs

@@ -0,0 +1,46 @@
+use pyo3::prelude::*;
+
+use rustpython_ast::{
+    pyo3::{ToPyo3Ast, ToPyo3Wrapper},
+    Fold,
+};
+
+#[pyfunction]
+fn parse_wrap(a: &str, py: Python) -> PyResult<PyObject> {
+    let parsed = rustpython_parser::parse(a, rustpython_parser::Mode::Module, "<unknown>")
+        .map_err(|e| PyErr::new::<pyo3::exceptions::PySyntaxError, _>(e.to_string()))?;
+    // let parsed = rustpython_ast::source_code::SourceLocator::new(a).fold(parsed).unwrap();
+    let parsed = parsed.module().unwrap();
+    let ast_module = PyModule::import(py, "_ast")?;
+
+    let parsed = Box::leak(Box::new(parsed));
+    parsed.to_pyo3_wrapper(py)
+}
+
+#[pyfunction]
+fn parse(a: &str, py: Python) -> PyResult<PyObject> {
+    use rustpython_parser::{ast::fold::Fold, source_code::SourceLocator};
+    let parsed = rustpython_parser::parse(a, rustpython_parser::Mode::Module, "<unknown>")
+        .map_err(|e| PyErr::new::<pyo3::exceptions::PySyntaxError, _>(e.to_string()))?
+        // .module().unwrap()
+        ;
+    // let located = SourceLocator::new(a).fold(parsed).unwrap();
+    let x = parsed.module().unwrap();
+    x.to_pyo3_ast(py)
+}
+
+#[pymodule]
+fn rustpython_parser_pyo3(py: Python, m: &PyModule) -> PyResult<()> {
+    // let ast = PyModule::new(py, "ast")?;
+    // rustpython_ast::pyo3::located::add_to_module(py, ast)?;
+    // m.add_submodule(ast)?;
+
+    let ast = PyModule::new(py, "unlocated_ast")?;
+    rustpython_ast::pyo3::ranged::add_to_module(py, ast)?;
+    m.add_submodule(ast)?;
+
+    m.add_function(wrap_pyfunction!(parse, m)?)?;
+    m.add_function(wrap_pyfunction!(parse_wrap, m)?)?;
+
+    Ok(())
+}

parser-pyo3/test_ast.py

@@ -0,0 +1,47 @@
+import re
+import baembal
+
+import ast
+
+ast.AST = baembal.AST
+
+import pytest
+
+import ast as py_ast
+import rustpython_parser_pyo3 as rust_ast
+
+
+from glob import glob
+
+files = {}
+for path in glob("../../cpython/Lib/**/*.py"):
+    try:
+        txt = open(path, "r").read()
+    except UnicodeDecodeError:
+        continue
+    # try:
+    #     if py_ast.dump(py_ast.parse(txt)) != py_ast.dump(rust_ast.parse(txt)):
+    #         continue
+    # except SyntaxError:
+    #     continue
+    files[path] = txt
+
+
+@pytest.mark.parametrize("path", files.keys())
+def test_roundtrip(path):
+    txt = files[path]
+    module_p = py_ast.parse(txt)
+    dump_p = py_ast.dump(module_p, indent=True)
+    module_r = rust_ast.parse(txt)
+    dump_r = py_ast.dump(module_r, indent=True)
+    p = re.compile("object at 0x[0-9a-f]+")
+    dump_p2 = re.sub(p, "object at 0x????????", dump_p)
+    dump_r2 = re.sub(p, "object at 0x????????", dump_r)
+    try:
+        assert dump_p2 == dump_r2, dump_r2
+    except AssertionError:
+        with open("dump_p.txt", "w") as f:
+            f.write(dump_p2)
+        with open("dump_r.txt", "w") as f:
+            f.write(dump_r2)
+        raise