numerous refactoring

- Split parser core and compiler core. Fix #14
- AST int type to `u32`
- Updated asdl_rs.py and update_asdl.sh fix #6
- Use `ruff_python_ast::SourceLocation` for Python source location. Deleted our own Location.
- Renamed ast::Located to ast::Attributed to distinguish terms for TextSize and SourceLocation
- `ast::<Node>`s for TextSize located ast. `ast::located::<Node>` for Python source located ast.
- And also strictly renaming `located` to refer only python location related interfaces.
- `SourceLocator` to convert locations.
- New `source-code` features of to disable python locations when unnecessary.
- Also including fully merging https://github.com/astral-sh/RustPython/pull/4 closes #9
This commit is contained in:
Jeong YunWon 2023-05-10 02:36:52 +09:00
parent 09a6afdd04
commit a3d9d8cb14
29 changed files with 9737 additions and 12000 deletions

View file

@ -4,7 +4,7 @@
//! loosely based on the token definitions found in the [CPython source].
//!
//! [CPython source]: https://github.com/python/cpython/blob/dfc2e065a2e71011017077e549cd2f9bf4944c54/Include/internal/pycore_token.h
use crate::text_size::TextSize;
use crate::{text_size::TextSize, Mode};
use num_bigint::BigInt;
use std::fmt;
@ -196,6 +196,16 @@ pub enum Tok {
StartExpression,
}
impl Tok {
pub fn start_marker(mode: Mode) -> Self {
match mode {
Mode::Module => Tok::StartModule,
Mode::Interactive => Tok::StartInteractive,
Mode::Expression => Tok::StartExpression,
}
}
}
impl fmt::Display for Tok {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
use Tok::*;
@ -404,10 +414,11 @@ impl StringKind {
/// Returns the number of characters in the prefix.
pub fn prefix_len(&self) -> TextSize {
use StringKind::*;
match self {
String => TextSize::from(0),
RawString | FString | Unicode | Bytes => TextSize::from(1),
RawFString | RawBytes => TextSize::from(2),
}
let len = match self {
String => 0,
RawString | FString | Unicode | Bytes => 1,
RawFString | RawBytes => 2,
};
len.into()
}
}