numerous refactoring

- Split parser core and compiler core. Fix #14
- AST int type to `u32`
- Updated asdl_rs.py and update_asdl.sh fix #6
- Use `ruff_python_ast::SourceLocation` for Python source location. Deleted our own Location.
- Renamed ast::Located to ast::Attributed to distinguish terms for TextSize and SourceLocation
- `ast::<Node>`s for TextSize located ast. `ast::located::<Node>` for Python source located ast.
- And also strictly renaming `located` to refer only python location related interfaces.
- `SourceLocator` to convert locations.
- New `source-code` features of to disable python locations when unnecessary.
- Also including fully merging https://github.com/astral-sh/RustPython/pull/4 closes #9

ast/src/attributed.rs

@@ -1,5 +1,7 @@
-use crate::location::{SourceLocation, SourceRange};
-use rustpython_compiler_core::text_size::{TextRange, TextSize};
+use rustpython_parser_core::{
+    source_code::{SourceLocation, SourceRange},
+    text_size::{TextRange, TextSize},
+};
 
 #[derive(Clone, Debug, PartialEq)]
 pub struct Attributed<T, U = ()> {

ast/src/constant.rs

@@ -1,5 +1,4 @@
 use num_bigint::BigInt;
-pub use rustpython_compiler_core::ConversionFlag;
 
 #[derive(Clone, Debug, PartialEq)]
 pub enum Constant {
@@ -137,7 +136,7 @@ impl<U> crate::fold::Fold<U> for ConstantOptimizer {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use rustpython_compiler_core::text_size::TextRange;
+    use rustpython_parser_core::text_size::TextRange;
 
     #[cfg(feature = "constant-optimization")]
     #[test]

ast/src/fold_helpers.rs

@@ -62,4 +62,4 @@ macro_rules! simple_fold {
     };
 }
 
-simple_fold!(usize, String, bool, constant::Constant);
+simple_fold!(u32, String, bool, constant::Constant);

ast/src/gen/generic.rs

@@ -158,7 +158,7 @@ pub struct StmtAnnAssign<U = ()> {
     pub target: Box<Expr<U>>,
     pub annotation: Box<Expr<U>>,
     pub value: Option<Box<Expr<U>>>,
-    pub simple: usize,
+    pub simple: u32,
 }
 
 impl<U> From<StmtAnnAssign<U>> for StmtKind<U> {
@@ -328,7 +328,7 @@ impl<U> From<StmtImport<U>> for StmtKind<U> {
 pub struct StmtImportFrom<U = ()> {
     pub module: Option<Ident>,
     pub names: Vec<Alias<U>>,
-    pub level: Option<usize>,
+    pub level: Option<u32>,
 }
 
 impl<U> From<StmtImportFrom<U>> for StmtKind<U> {
@@ -610,7 +610,7 @@ impl<U> From<ExprCall<U>> for ExprKind<U> {
 #[derive(Clone, Debug, PartialEq)]
 pub struct ExprFormattedValue<U = ()> {
     pub value: Box<Expr<U>>,
-    pub conversion: usize,
+    pub conversion: u32,
     pub format_spec: Option<Box<Expr<U>>>,
 }
 
@@ -819,7 +819,7 @@ pub struct Comprehension<U = ()> {
     pub target: Expr<U>,
     pub iter: Expr<U>,
     pub ifs: Vec<Expr<U>>,
-    pub is_async: usize,
+    pub is_async: u32,
 }
 
 #[derive(Clone, Debug, PartialEq)]
@@ -996,7 +996,7 @@ pub type Pattern<U = ()> = Attributed<PatternKind<U>, U>;
 
 #[derive(Clone, Debug, PartialEq)]
 pub struct TypeIgnoreTypeIgnore {
-    pub lineno: usize,
+    pub lineno: u32,
     pub tag: String,
 }
 
@@ -1019,6 +1019,7 @@ pub mod fold {
         type TargetU;
         type Error;
         fn map_user(&mut self, user: U) -> Result<Self::TargetU, Self::Error>;
+
         fn map_located<T>(
             &mut self,
             located: Attributed<T, U>,
@@ -1030,6 +1031,13 @@ pub mod fold {
                 node: located.node,
             })
         }
+
+        fn fold<X: Foldable<U, Self::TargetU>>(
+            &mut self,
+            node: X,
+        ) -> Result<X::Mapped, Self::Error> {
+            node.fold(self)
+        }
         fn fold_mod(&mut self, node: Mod<U>) -> Result<Mod<Self::TargetU>, Self::Error> {
             fold_mod(self, node)
         }

ast/src/gen/located.rs

@@ -1,6 +1,6 @@
 // File automatically generated by ast/asdl_rs.py.
 
-use crate::location::SourceRange;
+use rustpython_parser_core::source_code::SourceRange;
 
 pub type Located<T> = super::generic::Attributed<T, SourceRange>;
 pub type Mod = super::generic::Mod<SourceRange>;

ast/src/lib.rs

@@ -7,53 +7,21 @@ mod generic {
     include!("gen/generic.rs");
 }
 mod impls;
-#[cfg(feature = "location")]
-pub mod located {
-    include!("gen/located.rs");
-}
-#[cfg(feature = "location")]
-mod locator;
-#[cfg(feature = "location")]
-pub use crate::locator::locate;
-#[cfg(feature = "location")]
-pub use rustpython_compiler_core::SourceLocator;
-
+#[cfg(feature = "source-code")]
+mod source_locator;
 #[cfg(feature = "unparse")]
 mod unparse;
 
 pub use attributed::Attributed;
-pub use constant::{Constant, ConversionFlag};
+pub use constant::Constant;
 pub use generic::*;
+pub use rustpython_parser_core::{text_size, ConversionFlag};
 
 pub type Suite<U = ()> = Vec<Stmt<U>>;
 
-pub mod location {
-    pub use rustpython_compiler_core::source_code::{OneIndexed, SourceLocation};
-
-    #[derive(Debug)]
-    pub struct SourceRange {
-        pub start: SourceLocation,
-        pub end: Option<SourceLocation>,
-    }
-
-    impl SourceRange {
-        pub fn new(start: SourceLocation, end: SourceLocation) -> Self {
-            Self {
-                start,
-                end: Some(end),
-            }
-        }
-        pub fn unwrap_end(&self) -> SourceLocation {
-            self.end.unwrap()
-        }
-    }
-
-    impl From<std::ops::Range<SourceLocation>> for SourceRange {
-        fn from(value: std::ops::Range<SourceLocation>) -> Self {
-            Self {
-                start: value.start,
-                end: Some(value.end),
-            }
-        }
-    }
+#[cfg(feature = "source-code")]
+pub mod located {
+    include!("gen/located.rs");
 }
+
+pub use rustpython_parser_core::source_code;

ast/src/source_locator.rs

@@ -1,11 +1,5 @@
 use crate::attributed::Attributed;
-use crate::fold_helpers::Foldable;
-use crate::location::SourceRange;
-use rustpython_compiler_core::SourceLocator;
-
-pub fn locate<X: Foldable<(), SourceRange>>(locator: &mut SourceLocator, ast: X) -> X::Mapped {
-    ast.fold(locator).unwrap()
-}
+use rustpython_parser_core::source_code::{SourceLocator, SourceRange};
 
 impl crate::fold::Fold<()> for SourceLocator<'_> {
     type TargetU = SourceRange;

ast/src/unparse.rs

@@ -1,7 +1,5 @@
-use crate::{
-    Arg, Arguments, Boolop, Cmpop, Comprehension, Constant, ConversionFlag, Expr, ExprKind,
-    Operator,
-};
+use crate::ConversionFlag;
+use crate::{Arg, Arguments, Boolop, Cmpop, Comprehension, Constant, Expr, ExprKind, Operator};
 use std::fmt;
 
 mod precedence {
@@ -452,7 +450,7 @@ impl<'a> Unparser<'a> {
     fn unparse_formatted<U>(
         &mut self,
         val: &Expr<U>,
-        conversion: usize,
+        conversion: u32,
         spec: Option<&Expr<U>>,
     ) -> fmt::Result {
         let buffered = to_string_fmt(|f| Unparser::new(f).unparse_expr(val, precedence::TEST + 1));
@@ -466,7 +464,7 @@ impl<'a> Unparser<'a> {
         self.p(&buffered)?;
         drop(buffered);
 
-        if conversion != ConversionFlag::None as usize {
+        if conversion != ConversionFlag::None as u32 {
             self.p("!")?;
             let buf = &[conversion as u8];
             let c = std::str::from_utf8(buf).unwrap();