feat: make erg_parser available as a Python lib

.gitignore

@@ -18,6 +18,7 @@
 /.idea/
 /timeit.dat
 **/__pycache__/
+**/.venv
 
 # Nix
 .direnv

Cargo.lock

@@ -125,6 +125,7 @@ dependencies = [
  "backtrace-on-stack-overflow",
  "crossterm",
  "parking_lot",
+ "pyo3",
  "thread_local",
 ]
 
@@ -141,6 +142,8 @@ name = "erg_parser"
 version = "0.6.25"
 dependencies = [
  "erg_common",
+ "erg_proc_macros",
+ "pyo3",
  "unicode-xid",
 ]
 
@@ -168,6 +171,12 @@ version = "0.28.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0"
 
+[[package]]
+name = "heck"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8"
+
 [[package]]
 name = "idna"
 version = "0.4.0"
@@ -178,6 +187,12 @@ dependencies = [
  "unicode-normalization",
 ]
 
+[[package]]
+name = "indoc"
+version = "2.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e186cfbae8084e513daff4240b4797e342f988cecda4fb6c939150f96315fd8"
+
 [[package]]
 name = "itoa"
 version = "1.0.9"
@@ -234,6 +249,15 @@ dependencies = [
  "autocfg",
 ]
 
+[[package]]
+name = "memoffset"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c"
+dependencies = [
+ "autocfg",
+]
+
 [[package]]
 name = "miniz_oxide"
 version = "0.7.1"
@@ -276,7 +300,7 @@ dependencies = [
  "cc",
  "cfg-if",
  "libc",
- "memoffset",
+ "memoffset 0.6.5",
 ]
 
 [[package]]
@@ -332,6 +356,67 @@ dependencies = [
  "unicode-ident",
 ]
 
+[[package]]
+name = "pyo3"
+version = "0.20.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "04e8453b658fe480c3e70c8ed4e3d3ec33eb74988bd186561b0cc66b85c3bc4b"
+dependencies = [
+ "cfg-if",
+ "indoc",
+ "libc",
+ "memoffset 0.9.0",
+ "parking_lot",
+ "pyo3-build-config",
+ "pyo3-ffi",
+ "pyo3-macros",
+ "unindent",
+]
+
+[[package]]
+name = "pyo3-build-config"
+version = "0.20.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a96fe70b176a89cff78f2fa7b3c930081e163d5379b4dcdf993e3ae29ca662e5"
+dependencies = [
+ "once_cell",
+ "target-lexicon",
+]
+
+[[package]]
+name = "pyo3-ffi"
+version = "0.20.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "214929900fd25e6604661ed9cf349727c8920d47deff196c4e28165a6ef2a96b"
+dependencies = [
+ "libc",
+ "pyo3-build-config",
+]
+
+[[package]]
+name = "pyo3-macros"
+version = "0.20.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dac53072f717aa1bfa4db832b39de8c875b7c7af4f4a6fe93cdbf9264cf8383b"
+dependencies = [
+ "proc-macro2",
+ "pyo3-macros-backend",
+ "quote",
+ "syn 2.0.38",
+]
+
+[[package]]
+name = "pyo3-macros-backend"
+version = "0.20.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7774b5a8282bd4f25f803b1f0d945120be959a36c72e08e7cd031c792fdfd424"
+dependencies = [
+ "heck",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.38",
+]
+
 [[package]]
 name = "quote"
 version = "1.0.33"
@@ -468,6 +553,12 @@ dependencies = [
  "unicode-ident",
 ]
 
+[[package]]
+name = "target-lexicon"
+version = "0.12.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "14c39fd04924ca3a864207c66fc2cd7d22d7c016007f9ce846cbb9326331930a"
+
 [[package]]
 name = "thread_local"
 version = "1.1.7"
@@ -520,6 +611,12 @@ version = "0.2.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c"
 
+[[package]]
+name = "unindent"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c7de7d73e1754487cb58364ee906a499937a0dfabd86bcb980fa99ec8c8fa2ce"
+
 [[package]]
 name = "url"
 version = "2.4.1"

Cargo.toml

@@ -71,6 +71,7 @@ erg_parser = { version = "0.6.25", path = "./crates/erg_parser" }
 erg_compiler = { version = "0.6.25", path = "./crates/erg_compiler" }
 els = { version = "0.1.37", path = "./crates/els" }
 erg_proc_macros = { version = "0.6.25", path = "./crates/erg_proc_macros" }
+pyo3 = { version = "0.20", features = ["extension-module"] }
 
 [dependencies]
 erg_common = { workspace = true }

crates/erg_common/Cargo.toml

@@ -24,6 +24,7 @@ gal = []
 no_std = []
 full-repl = ["dep:crossterm"]
 experimental = []
+pylib = ["dep:pyo3"]
 
 [target.'cfg(unix)'.dependencies]
 backtrace-on-stack-overflow = { version = "0.3.0", optional = true }
@@ -32,6 +33,7 @@ backtrace-on-stack-overflow = { version = "0.3.0", optional = true }
 crossterm = { optional = true, version = "0.25.0" }
 parking_lot = "0.12"
 thread_local = "1.1"
+pyo3 = { workspace = true, optional = true }
 
 [lib]
 path = "lib.rs"

crates/erg_common/str.rs

@@ -3,6 +3,9 @@ use std::fmt;
 use std::hash::{Hash, Hasher};
 use std::ops::{Add, Deref};
 
+#[cfg(feature = "pylib")]
+use pyo3::{IntoPy, PyObject, Python};
+
 pub type ArcStr = std::sync::Arc<str>;
 
 /// Used to hold an immutable string.
@@ -14,6 +17,13 @@ pub enum Str {
     Static(&'static str),
 }
 
+#[cfg(feature = "pylib")]
+impl IntoPy<PyObject> for Str {
+    fn into_py(self, py: Python<'_>) -> PyObject {
+        (&self[..]).into_py(py)
+    }
+}
+
 impl PartialEq for Str {
     #[inline]
     fn eq(&self, other: &Str) -> bool {

crates/erg_parser/Cargo.toml

@@ -18,12 +18,16 @@ unicode = ["erg_common/unicode"]
 pretty = ["erg_common/pretty"]
 large_thread = ["erg_common/large_thread"]
 experimental = ["erg_common/experimental"]
+pylib = ["dep:pyo3", "erg_common/pylib"]
 
 [dependencies]
 erg_common = { workspace = true }
+erg_proc_macros = { workspace = true }
 unicode-xid = "0.2.4"
+pyo3 = { workspace = true, optional = true }
 
 [lib]
+crate-type = ["cdylib", "rlib"]
 path = "lib.rs"
 
 [[bin]]

crates/erg_parser/README.md

@@ -1,5 +1,22 @@
 # Erg parser
 
-## Why isn't this module but crate?
+## Use `erg_parser` as a Python library
 
-For maintainability. This crate has tests.
+`erg_parser` can be built as a Python library by using pyo3/maturin.
+
+### Example
+
+```python
+import erg_parser
+
+module = erg_parser.parse("x = 1")
+for chunk in module:
+    if isinstance(chunk, erg_parser.expr.Def):
+        assert chunk.sig.inspect() == "x"
+```
+
+### Debug install
+
+```python
+maturin develop --features pylib
+```

crates/erg_parser/error.rs

@@ -25,6 +25,13 @@ impl fmt::Display for LexError {
 
 impl std::error::Error for LexError {}
 
+#[cfg(feature = "pylib")]
+impl std::convert::From<LexError> for pyo3::PyErr {
+    fn from(err: LexError) -> pyo3::PyErr {
+        pyo3::exceptions::PyOSError::new_err(err.to_string())
+    }
+}
+
 impl From<ErrorCore> for LexError {
     fn from(core: ErrorCore) -> Self {
         Self(Box::new(core))
@@ -37,6 +44,7 @@ impl From<LexError> for ErrorCore {
     }
 }
 
+#[cfg_attr(feature = "pylib", pyo3::pyclass)]
 #[derive(Debug)]
 pub struct LexErrors(Vec<LexError>);
 
@@ -50,6 +58,13 @@ impl fmt::Display for LexErrors {
 
 impl std::error::Error for LexErrors {}
 
+#[cfg(feature = "pylib")]
+impl std::convert::From<LexErrors> for pyo3::PyErr {
+    fn from(errs: LexErrors) -> pyo3::PyErr {
+        pyo3::exceptions::PyOSError::new_err(errs[0].to_string())
+    }
+}
+
 const ERR: Color = THEME.colors.error;
 const WARN: Color = THEME.colors.warning;
 const HINT: Color = THEME.colors.hint;

crates/erg_parser/lib.rs

@@ -17,3 +17,74 @@ pub mod visitor;
 
 pub use parse::{Parser, ParserRunner};
 pub use visitor::ASTVisitor;
+
+#[cfg(feature = "pylib")]
+use pyo3::prelude::*;
+
+#[cfg(feature = "pylib")]
+#[pyfunction]
+#[pyo3(name = "parse")]
+fn _parse(code: String) -> Result<ast::Module, error::ParseErrors> {
+    parse::SimpleParser::parse(code)
+        .map(|art| art.ast)
+        .map_err(|iart| iart.errors)
+}
+
+#[cfg(feature = "pylib")]
+#[pymodule]
+fn erg_parser(py: Python<'_>, m: &PyModule) -> PyResult<()> {
+    m.add_function(wrap_pyfunction!(_parse, m)?)?;
+    let expr = PyModule::new(py, "expr")?;
+    expr.add_class::<ast::Literal>()?;
+    expr.add_class::<ast::NormalArray>()?;
+    expr.add_class::<ast::NormalTuple>()?;
+    expr.add_class::<ast::NormalDict>()?;
+    expr.add_class::<ast::NormalSet>()?;
+    expr.add_class::<ast::NormalRecord>()?;
+    expr.add_class::<ast::BinOp>()?;
+    expr.add_class::<ast::UnaryOp>()?;
+    expr.add_class::<ast::Call>()?;
+    expr.add_class::<ast::DataPack>()?;
+    expr.add_class::<ast::Lambda>()?;
+    expr.add_class::<ast::TypeAscription>()?;
+    expr.add_class::<ast::Def>()?;
+    expr.add_class::<ast::Methods>()?;
+    expr.add_class::<ast::ClassDef>()?;
+    expr.add_class::<ast::PatchDef>()?;
+    expr.add_class::<ast::ReDef>()?;
+    expr.add_class::<ast::Compound>()?;
+    expr.add_class::<ast::InlineModule>()?;
+    expr.add_class::<ast::Dummy>()?;
+    m.add_submodule(expr)?;
+
+    let ast = PyModule::new(py, "ast")?;
+    ast.add_class::<ast::Literal>()?;
+    ast.add_class::<ast::Identifier>()?;
+    ast.add_class::<ast::Attribute>()?;
+    ast.add_class::<ast::TupleAttribute>()?;
+    ast.add_class::<ast::Subscript>()?;
+    ast.add_class::<ast::TypeApp>()?;
+    ast.add_class::<ast::NormalArray>()?;
+    ast.add_class::<ast::NormalTuple>()?;
+    ast.add_class::<ast::NormalDict>()?;
+    ast.add_class::<ast::NormalSet>()?;
+    ast.add_class::<ast::NormalRecord>()?;
+    ast.add_class::<ast::BinOp>()?;
+    ast.add_class::<ast::UnaryOp>()?;
+    ast.add_class::<ast::Call>()?;
+    ast.add_class::<ast::Args>()?;
+    ast.add_class::<ast::Block>()?;
+    ast.add_class::<ast::DataPack>()?;
+    ast.add_class::<ast::Lambda>()?;
+    ast.add_class::<ast::TypeAscription>()?;
+    ast.add_class::<ast::Def>()?;
+    ast.add_class::<ast::Methods>()?;
+    ast.add_class::<ast::ClassDef>()?;
+    ast.add_class::<ast::PatchDef>()?;
+    ast.add_class::<ast::ReDef>()?;
+    ast.add_class::<ast::Compound>()?;
+    ast.add_class::<ast::InlineModule>()?;
+    ast.add_class::<ast::Dummy>()?;
+    m.add_submodule(ast)?;
+    Ok(())
+}

crates/erg_parser/parse.rs

@@ -102,6 +102,7 @@ pub trait Parsable: 'static {
     fn parse(code: String) -> Result<CompleteArtifact, IncompleteArtifact<Module, ParseErrors>>;
 }
 
+#[cfg_attr(feature = "pylib", pyo3::pyclass)]
 pub struct SimpleParser {}
 
 impl Parsable for SimpleParser {
@@ -353,11 +354,11 @@ impl Parser {
         ParseError::unclosed_error(line as usize, loc, closer, ty)
     }
 
-    fn skip_and_throw_invalid_seq_err<S: std::fmt::Display>(
+    fn skip_and_throw_invalid_seq_err(
         &mut self,
         caused_by: &str,
         errno: usize,
-        expected: &[S],
+        expected: &[impl std::fmt::Display],
         found: TokenKind,
     ) -> ParseError {
         log!(err "error caused by: {caused_by}");

crates/erg_parser/token.rs

@@ -14,7 +14,13 @@ use erg_common::traits::{DequeStream, Locational};
 // use erg_common::typaram::OpKind;
 // use erg_common::value::ValueObj;
 
+#[cfg(not(feature = "pylib"))]
+use erg_proc_macros::pyclass;
+#[cfg(feature = "pylib")]
+use pyo3::prelude::*;
+
 /// 意味論的名前と記号自体の名前が混在しているが、Pythonの名残である
+#[pyclass]
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
 #[repr(u8)]
 pub enum TokenKind {
@@ -188,6 +194,7 @@ pub enum TokenKind {
 
 use TokenKind::*;
 
+#[pyclass]
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub enum TokenCategory {
     Symbol,
@@ -333,6 +340,7 @@ impl From<TokenKind> for BinOpCode {
     }
 }
 
+#[pyclass(get_all)]
 #[derive(Clone, Eq)]
 pub struct Token {
     pub kind: TokenKind,
@@ -547,6 +555,7 @@ impl Token {
     }
 }
 
+#[pyclass]
 #[derive(Debug, Clone)]
 pub struct TokenStream(VecDeque<Token>);
 

crates/erg_proc_macros/src/lib.rs

@@ -54,3 +54,21 @@ pub fn exec_new_thread(_attr: TokenStream, item: TokenStream) -> TokenStream {
     let item = quote! { #item_fn };
     item.into()
 }
+
+/// dummy attribute
+#[proc_macro_attribute]
+pub fn pyo3(_attr: TokenStream, item: TokenStream) -> TokenStream {
+    item
+}
+
+/// dummy attribute
+#[proc_macro_attribute]
+pub fn pyclass(_attr: TokenStream, item: TokenStream) -> TokenStream {
+    item
+}
+
+/// dummy attribute
+#[proc_macro_attribute]
+pub fn pymethods(_attr: TokenStream, item: TokenStream) -> TokenStream {
+    item
+}