mirror of
https://github.com/RustPython/Parser.git
synced 2025-08-27 22:04:50 +00:00
Separate byteoffset ast and located ast
This commit is contained in:
Jeong YunWon
parent
f47dfca4e3
commit
a14e43e03a
21 changed files with 893 additions and 562 deletions
76
ast/src/attributed.rs
Normal file
76
ast/src/attributed.rs
Normal file
|
|
@ -0,0 +1,76 @@
|
|||
use rustpython_compiler_core::{
|
||||
text_size::{TextRange, TextSize},
|
||||
Location, LocationRange,
|
||||
};
|
||||
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct Attributed<T, U = ()> {
|
||||
pub range: TextRange,
|
||||
pub custom: U,
|
||||
pub node: T,
|
||||
}
|
||||
|
||||
impl<T, U> Attributed<T, U> {
|
||||
/// Returns the node
|
||||
#[inline]
|
||||
pub fn node(&self) -> &T {
|
||||
&self.node
|
||||
}
|
||||
|
||||
/// Returns the `range` of the node. The range offsets are absolute to the start of the document.
|
||||
#[inline]
|
||||
pub const fn range(&self) -> TextRange {
|
||||
self.range
|
||||
}
|
||||
|
||||
/// Returns the absolute start position of the node from the beginning of the document.
|
||||
#[inline]
|
||||
pub const fn start(&self) -> TextSize {
|
||||
self.range.start()
|
||||
}
|
||||
|
||||
/// Returns the absolute position at which the node ends in the source document.
|
||||
#[inline]
|
||||
pub const fn end(&self) -> TextSize {
|
||||
self.range.end()
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> Attributed<T, ()> {
|
||||
/// Creates a new node that spans the position specified by `range`.
|
||||
pub fn new(range: impl Into<TextRange>, node: T) -> Self {
|
||||
Self {
|
||||
range: range.into(),
|
||||
custom: (),
|
||||
node,
|
||||
}
|
||||
}
|
||||
|
||||
/// Consumes self and returns the node.
|
||||
#[inline]
|
||||
pub fn into_node(self) -> T {
|
||||
self.node
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> Attributed<T, LocationRange> {
|
||||
/// Returns the absolute start position of the node from the beginning of the document.
|
||||
#[inline]
|
||||
pub const fn location(&self) -> Location {
|
||||
self.custom.start
|
||||
}
|
||||
|
||||
/// Returns the absolute position at which the node ends in the source document.
|
||||
#[inline]
|
||||
pub const fn end_location(&self) -> Location {
|
||||
self.custom.end
|
||||
}
|
||||
}
|
||||
|
||||
impl<T, U> std::ops::Deref for Attributed<T, U> {
|
||||
type Target = T;
|
||||
|
||||
fn deref(&self) -> &Self::Target {
|
||||
&self.node
|
||||
}
|
||||
}
|
||||
|
|
@ -137,6 +137,7 @@ impl<U> crate::fold::Fold<U> for ConstantOptimizer {
|
|||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use rustpython_compiler_core::text_size::TextRange;
|
||||
|
||||
#[cfg(feature = "constant-optimization")]
|
||||
#[test]
|
||||
|
|
@ -146,13 +147,13 @@ mod tests {
|
|||
let range = TextRange::default();
|
||||
#[allow(clippy::let_unit_value)]
|
||||
let custom = ();
|
||||
let ast = Located {
|
||||
let ast = Attributed {
|
||||
range,
|
||||
custom,
|
||||
node: ExprTuple {
|
||||
ctx: ExprContext::Load,
|
||||
elts: vec![
|
||||
Located {
|
||||
Attributed {
|
||||
range,
|
||||
custom,
|
||||
node: ExprConstant {
|
||||
|
|
@ -161,7 +162,7 @@ mod tests {
|
|||
}
|
||||
.into(),
|
||||
},
|
||||
Located {
|
||||
Attributed {
|
||||
range,
|
||||
custom,
|
||||
node: ExprConstant {
|
||||
|
|
@ -170,13 +171,13 @@ mod tests {
|
|||
}
|
||||
.into(),
|
||||
},
|
||||
Located {
|
||||
Attributed {
|
||||
range,
|
||||
custom,
|
||||
node: ExprTuple {
|
||||
ctx: ExprContext::Load,
|
||||
elts: vec![
|
||||
Located {
|
||||
Attributed {
|
||||
range,
|
||||
custom,
|
||||
node: ExprConstant {
|
||||
|
|
@ -185,7 +186,7 @@ mod tests {
|
|||
}
|
||||
.into(),
|
||||
},
|
||||
Located {
|
||||
Attributed {
|
||||
range,
|
||||
custom,
|
||||
node: ExprConstant {
|
||||
|
|
@ -194,7 +195,7 @@ mod tests {
|
|||
}
|
||||
.into(),
|
||||
},
|
||||
Located {
|
||||
Attributed {
|
||||
range,
|
||||
custom,
|
||||
node: ExprConstant {
|
||||
|
|
@ -216,7 +217,7 @@ mod tests {
|
|||
.unwrap_or_else(|e| match e {});
|
||||
assert_eq!(
|
||||
new_ast,
|
||||
Located {
|
||||
Attributed {
|
||||
range,
|
||||
custom,
|
||||
node: ExprConstant {
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
use crate::{constant, fold::Fold};
|
||||
|
||||
pub(crate) trait Foldable<T, U> {
|
||||
pub trait Foldable<T, U> {
|
||||
type Mapped;
|
||||
fn fold<F: Fold<T, TargetU = U> + ?Sized>(
|
||||
self,
|
||||
|
|
|
|||
|
|
@ -2,75 +2,11 @@
|
|||
|
||||
#![allow(clippy::derive_partial_eq_without_eq)]
|
||||
|
||||
pub use crate::constant::*;
|
||||
pub use rustpython_compiler_core::text_size::{TextRange, TextSize};
|
||||
pub use crate::{constant::*, Attributed};
|
||||
use rustpython_compiler_core::text_size::{TextRange, TextSize};
|
||||
|
||||
type Ident = String;
|
||||
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct Located<T, U = ()> {
|
||||
pub range: TextRange,
|
||||
pub custom: U,
|
||||
pub node: T,
|
||||
}
|
||||
|
||||
impl<T> Located<T> {
|
||||
pub fn new(start: TextSize, end: TextSize, node: T) -> Self {
|
||||
Self {
|
||||
range: TextRange::new(start, end),
|
||||
custom: (),
|
||||
node,
|
||||
}
|
||||
}
|
||||
|
||||
/// Creates a new node that spans the position specified by `range`.
|
||||
pub fn with_range(node: T, range: TextRange) -> Self {
|
||||
Self {
|
||||
range,
|
||||
custom: (),
|
||||
node,
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the absolute start position of the node from the beginning of the document.
|
||||
#[inline]
|
||||
pub const fn start(&self) -> TextSize {
|
||||
self.range.start()
|
||||
}
|
||||
|
||||
/// Returns the node
|
||||
#[inline]
|
||||
pub fn node(&self) -> &T {
|
||||
&self.node
|
||||
}
|
||||
|
||||
/// Consumes self and returns the node.
|
||||
#[inline]
|
||||
pub fn into_node(self) -> T {
|
||||
self.node
|
||||
}
|
||||
|
||||
/// Returns the `range` of the node. The range offsets are absolute to the start of the document.
|
||||
#[inline]
|
||||
pub const fn range(&self) -> TextRange {
|
||||
self.range
|
||||
}
|
||||
|
||||
/// Returns the absolute position at which the node ends in the source document.
|
||||
#[inline]
|
||||
pub const fn end(&self) -> TextSize {
|
||||
self.range.end()
|
||||
}
|
||||
}
|
||||
|
||||
impl<T, U> std::ops::Deref for Located<T, U> {
|
||||
type Target = T;
|
||||
|
||||
fn deref(&self) -> &Self::Target {
|
||||
&self.node
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct ModModule<U = ()> {
|
||||
pub body: Vec<Stmt<U>>,
|
||||
|
|
@ -467,7 +403,7 @@ pub enum StmtKind<U = ()> {
|
|||
Break,
|
||||
Continue,
|
||||
}
|
||||
pub type Stmt<U = ()> = Located<StmtKind<U>, U>;
|
||||
pub type Stmt<U = ()> = Attributed<StmtKind<U>, U>;
|
||||
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct ExprBoolOp<U = ()> {
|
||||
|
|
@ -827,7 +763,7 @@ pub enum ExprKind<U = ()> {
|
|||
Tuple(ExprTuple<U>),
|
||||
Slice(ExprSlice<U>),
|
||||
}
|
||||
pub type Expr<U = ()> = Located<ExprKind<U>, U>;
|
||||
pub type Expr<U = ()> = Attributed<ExprKind<U>, U>;
|
||||
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub enum ExprContext {
|
||||
|
|
@ -906,7 +842,7 @@ impl<U> From<ExcepthandlerExceptHandler<U>> for ExcepthandlerKind<U> {
|
|||
pub enum ExcepthandlerKind<U = ()> {
|
||||
ExceptHandler(ExcepthandlerExceptHandler<U>),
|
||||
}
|
||||
pub type Excepthandler<U = ()> = Located<ExcepthandlerKind<U>, U>;
|
||||
pub type Excepthandler<U = ()> = Attributed<ExcepthandlerKind<U>, U>;
|
||||
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct Arguments<U = ()> {
|
||||
|
|
@ -925,21 +861,21 @@ pub struct ArgData<U = ()> {
|
|||
pub annotation: Option<Box<Expr<U>>>,
|
||||
pub type_comment: Option<String>,
|
||||
}
|
||||
pub type Arg<U = ()> = Located<ArgData<U>, U>;
|
||||
pub type Arg<U = ()> = Attributed<ArgData<U>, U>;
|
||||
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct KeywordData<U = ()> {
|
||||
pub arg: Option<Ident>,
|
||||
pub value: Expr<U>,
|
||||
}
|
||||
pub type Keyword<U = ()> = Located<KeywordData<U>, U>;
|
||||
pub type Keyword<U = ()> = Attributed<KeywordData<U>, U>;
|
||||
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct AliasData {
|
||||
pub name: Ident,
|
||||
pub asname: Option<Ident>,
|
||||
}
|
||||
pub type Alias<U = ()> = Located<AliasData, U>;
|
||||
pub type Alias<U = ()> = Attributed<AliasData, U>;
|
||||
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct Withitem<U = ()> {
|
||||
|
|
@ -1059,7 +995,7 @@ pub enum PatternKind<U = ()> {
|
|||
MatchAs(PatternMatchAs<U>),
|
||||
MatchOr(PatternMatchOr<U>),
|
||||
}
|
||||
pub type Pattern<U = ()> = Located<PatternKind<U>, U>;
|
||||
pub type Pattern<U = ()> = Attributed<PatternKind<U>, U>;
|
||||
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct TypeIgnoreTypeIgnore {
|
||||
|
|
@ -1086,6 +1022,17 @@ pub mod fold {
|
|||
type TargetU;
|
||||
type Error;
|
||||
fn map_user(&mut self, user: U) -> Result<Self::TargetU, Self::Error>;
|
||||
fn map_located<T>(
|
||||
&mut self,
|
||||
located: Attributed<T, U>,
|
||||
) -> Result<Attributed<T, Self::TargetU>, Self::Error> {
|
||||
let custom = self.map_user(located.custom)?;
|
||||
Ok(Attributed {
|
||||
range: located.range,
|
||||
custom,
|
||||
node: located.node,
|
||||
})
|
||||
}
|
||||
fn fold_mod(&mut self, node: Mod<U>) -> Result<Mod<Self::TargetU>, Self::Error> {
|
||||
fold_mod(self, node)
|
||||
}
|
||||
|
|
@ -1164,11 +1111,12 @@ pub mod fold {
|
|||
}
|
||||
fn fold_located<U, F: Fold<U> + ?Sized, T, MT>(
|
||||
folder: &mut F,
|
||||
node: Located<T, U>,
|
||||
node: Attributed<T, U>,
|
||||
f: impl FnOnce(&mut F, T) -> Result<MT, F::Error>,
|
||||
) -> Result<Located<MT, F::TargetU>, F::Error> {
|
||||
Ok(Located {
|
||||
custom: folder.map_user(node.custom)?,
|
||||
) -> Result<Attributed<MT, F::TargetU>, F::Error> {
|
||||
let node = folder.map_located(node)?;
|
||||
Ok(Attributed {
|
||||
custom: node.custom,
|
||||
range: node.range,
|
||||
node: f(folder, node.node)?,
|
||||
})
|
||||
|
|
@ -1,11 +1,21 @@
|
|||
mod ast_gen;
|
||||
mod attributed;
|
||||
mod constant;
|
||||
#[cfg(feature = "fold")]
|
||||
mod fold_helpers;
|
||||
mod generic;
|
||||
mod impls;
|
||||
#[cfg(feature = "location")]
|
||||
pub mod located;
|
||||
#[cfg(feature = "location")]
|
||||
mod locator;
|
||||
|
||||
#[cfg(feature = "unparse")]
|
||||
mod unparse;
|
||||
|
||||
pub use ast_gen::*;
|
||||
pub use attributed::Attributed;
|
||||
pub use constant::{Constant, ConversionFlag};
|
||||
pub use generic::*;
|
||||
#[cfg(feature = "location")]
|
||||
pub use locator::Locator;
|
||||
|
||||
pub type Suite<U = ()> = Vec<Stmt<U>>;
|
||||
|
|
|
|||
95
ast/src/located.rs
Normal file
95
ast/src/located.rs
Normal file
|
|
@ -0,0 +1,95 @@
|
|||
// File automatically generated by ast/asdl_rs.py.
|
||||
|
||||
use rustpython_compiler_core::LocationRange;
|
||||
|
||||
pub type Located<T> = super::generic::Attributed<T, LocationRange>;
|
||||
pub type Mod = super::generic::Mod<LocationRange>;
|
||||
pub type ModModule = super::generic::ModModule<LocationRange>;
|
||||
pub type ModInteractive = super::generic::ModInteractive<LocationRange>;
|
||||
pub type ModExpression = super::generic::ModExpression<LocationRange>;
|
||||
pub type ModFunctionType = super::generic::ModFunctionType<LocationRange>;
|
||||
pub type Stmt = super::generic::Stmt<LocationRange>;
|
||||
pub type StmtKind = super::generic::StmtKind<LocationRange>;
|
||||
pub type StmtFunctionDef = super::generic::StmtFunctionDef<LocationRange>;
|
||||
pub type StmtAsyncFunctionDef = super::generic::StmtAsyncFunctionDef<LocationRange>;
|
||||
pub type StmtClassDef = super::generic::StmtClassDef<LocationRange>;
|
||||
pub type StmtReturn = super::generic::StmtReturn<LocationRange>;
|
||||
pub type StmtDelete = super::generic::StmtDelete<LocationRange>;
|
||||
pub type StmtAssign = super::generic::StmtAssign<LocationRange>;
|
||||
pub type StmtAugAssign = super::generic::StmtAugAssign<LocationRange>;
|
||||
pub type StmtAnnAssign = super::generic::StmtAnnAssign<LocationRange>;
|
||||
pub type StmtFor = super::generic::StmtFor<LocationRange>;
|
||||
pub type StmtAsyncFor = super::generic::StmtAsyncFor<LocationRange>;
|
||||
pub type StmtWhile = super::generic::StmtWhile<LocationRange>;
|
||||
pub type StmtIf = super::generic::StmtIf<LocationRange>;
|
||||
pub type StmtWith = super::generic::StmtWith<LocationRange>;
|
||||
pub type StmtAsyncWith = super::generic::StmtAsyncWith<LocationRange>;
|
||||
pub type StmtMatch = super::generic::StmtMatch<LocationRange>;
|
||||
pub type StmtRaise = super::generic::StmtRaise<LocationRange>;
|
||||
pub type StmtTry = super::generic::StmtTry<LocationRange>;
|
||||
pub type StmtTryStar = super::generic::StmtTryStar<LocationRange>;
|
||||
pub type StmtAssert = super::generic::StmtAssert<LocationRange>;
|
||||
pub type StmtImport = super::generic::StmtImport<LocationRange>;
|
||||
pub type StmtImportFrom = super::generic::StmtImportFrom<LocationRange>;
|
||||
pub type StmtGlobal = super::generic::StmtGlobal;
|
||||
pub type StmtNonlocal = super::generic::StmtNonlocal;
|
||||
pub type StmtExpr = super::generic::StmtExpr<LocationRange>;
|
||||
pub type Expr = super::generic::Expr<LocationRange>;
|
||||
pub type ExprKind = super::generic::ExprKind<LocationRange>;
|
||||
pub type ExprBoolOp = super::generic::ExprBoolOp<LocationRange>;
|
||||
pub type ExprNamedExpr = super::generic::ExprNamedExpr<LocationRange>;
|
||||
pub type ExprBinOp = super::generic::ExprBinOp<LocationRange>;
|
||||
pub type ExprUnaryOp = super::generic::ExprUnaryOp<LocationRange>;
|
||||
pub type ExprLambda = super::generic::ExprLambda<LocationRange>;
|
||||
pub type ExprIfExp = super::generic::ExprIfExp<LocationRange>;
|
||||
pub type ExprDict = super::generic::ExprDict<LocationRange>;
|
||||
pub type ExprSet = super::generic::ExprSet<LocationRange>;
|
||||
pub type ExprListComp = super::generic::ExprListComp<LocationRange>;
|
||||
pub type ExprSetComp = super::generic::ExprSetComp<LocationRange>;
|
||||
pub type ExprDictComp = super::generic::ExprDictComp<LocationRange>;
|
||||
pub type ExprGeneratorExp = super::generic::ExprGeneratorExp<LocationRange>;
|
||||
pub type ExprAwait = super::generic::ExprAwait<LocationRange>;
|
||||
pub type ExprYield = super::generic::ExprYield<LocationRange>;
|
||||
pub type ExprYieldFrom = super::generic::ExprYieldFrom<LocationRange>;
|
||||
pub type ExprCompare = super::generic::ExprCompare<LocationRange>;
|
||||
pub type ExprCall = super::generic::ExprCall<LocationRange>;
|
||||
pub type ExprFormattedValue = super::generic::ExprFormattedValue<LocationRange>;
|
||||
pub type ExprJoinedStr = super::generic::ExprJoinedStr<LocationRange>;
|
||||
pub type ExprConstant = super::generic::ExprConstant;
|
||||
pub type ExprAttribute = super::generic::ExprAttribute<LocationRange>;
|
||||
pub type ExprSubscript = super::generic::ExprSubscript<LocationRange>;
|
||||
pub type ExprStarred = super::generic::ExprStarred<LocationRange>;
|
||||
pub type ExprName = super::generic::ExprName;
|
||||
pub type ExprList = super::generic::ExprList<LocationRange>;
|
||||
pub type ExprTuple = super::generic::ExprTuple<LocationRange>;
|
||||
pub type ExprSlice = super::generic::ExprSlice<LocationRange>;
|
||||
pub type ExprContext = super::generic::ExprContext;
|
||||
pub type Boolop = super::generic::Boolop;
|
||||
pub type Operator = super::generic::Operator;
|
||||
pub type Unaryop = super::generic::Unaryop;
|
||||
pub type Cmpop = super::generic::Cmpop;
|
||||
pub type Comprehension = super::generic::Comprehension<LocationRange>;
|
||||
pub type Excepthandler = super::generic::Excepthandler<LocationRange>;
|
||||
pub type ExcepthandlerKind = super::generic::ExcepthandlerKind<LocationRange>;
|
||||
pub type ExcepthandlerExceptHandler = super::generic::ExcepthandlerExceptHandler<LocationRange>;
|
||||
pub type Arguments = super::generic::Arguments<LocationRange>;
|
||||
pub type Arg = super::generic::Arg<LocationRange>;
|
||||
pub type ArgData = super::generic::ArgData<LocationRange>;
|
||||
pub type Keyword = super::generic::Keyword<LocationRange>;
|
||||
pub type KeywordData = super::generic::KeywordData<LocationRange>;
|
||||
pub type Alias = super::generic::Alias<LocationRange>;
|
||||
pub type AliasData = super::generic::AliasData;
|
||||
pub type Withitem = super::generic::Withitem<LocationRange>;
|
||||
pub type MatchCase = super::generic::MatchCase<LocationRange>;
|
||||
pub type Pattern = super::generic::Pattern<LocationRange>;
|
||||
pub type PatternKind = super::generic::PatternKind<LocationRange>;
|
||||
pub type PatternMatchValue = super::generic::PatternMatchValue<LocationRange>;
|
||||
pub type PatternMatchSingleton = super::generic::PatternMatchSingleton;
|
||||
pub type PatternMatchSequence = super::generic::PatternMatchSequence<LocationRange>;
|
||||
pub type PatternMatchMapping = super::generic::PatternMatchMapping<LocationRange>;
|
||||
pub type PatternMatchClass = super::generic::PatternMatchClass<LocationRange>;
|
||||
pub type PatternMatchStar = super::generic::PatternMatchStar;
|
||||
pub type PatternMatchAs = super::generic::PatternMatchAs<LocationRange>;
|
||||
pub type PatternMatchOr = super::generic::PatternMatchOr<LocationRange>;
|
||||
pub type TypeIgnore = super::generic::TypeIgnore;
|
||||
pub type TypeIgnoreTypeIgnore = super::generic::TypeIgnoreTypeIgnore;
|
||||
57
ast/src/locator.rs
Normal file
57
ast/src/locator.rs
Normal file
|
|
@ -0,0 +1,57 @@
|
|||
use crate::attributed::Attributed;
|
||||
use crate::fold_helpers::Foldable;
|
||||
use rustpython_compiler_core::{
|
||||
text_size::{TextRange, TextSize},
|
||||
Location, LocationRange,
|
||||
};
|
||||
|
||||
/// Converts source code byte-offset to Python convention line and column numbers.
|
||||
#[derive(Default)]
|
||||
pub struct Locator<'a> {
|
||||
source: &'a str,
|
||||
}
|
||||
|
||||
impl<'a> Locator<'a> {
|
||||
#[inline]
|
||||
pub fn new(source: &'a str) -> Self {
|
||||
Self { source }
|
||||
}
|
||||
|
||||
pub fn source(&'a self) -> &'a str {
|
||||
self.source
|
||||
}
|
||||
|
||||
pub fn locate(&mut self, offset: TextSize) -> Location {
|
||||
todo!()
|
||||
}
|
||||
|
||||
pub fn locate_range(&mut self, range: TextRange) -> LocationRange {
|
||||
self.locate(range.start())..self.locate(range.end())
|
||||
}
|
||||
|
||||
pub fn locate_ast<X: Foldable<(), LocationRange>>(&mut self, ast: X) -> X::Mapped {
|
||||
ast.fold(self).unwrap()
|
||||
}
|
||||
}
|
||||
|
||||
impl crate::fold::Fold<()> for Locator<'_> {
|
||||
type TargetU = LocationRange;
|
||||
type Error = std::convert::Infallible;
|
||||
|
||||
#[cold]
|
||||
fn map_user(&mut self, _user: ()) -> Result<Self::TargetU, Self::Error> {
|
||||
unreachable!("implemented map_located");
|
||||
}
|
||||
|
||||
fn map_located<T>(
|
||||
&mut self,
|
||||
node: Attributed<T, ()>,
|
||||
) -> Result<Attributed<T, Self::TargetU>, Self::Error> {
|
||||
let location = self.locate_range(node.range);
|
||||
Ok(Attributed {
|
||||
range: node.range,
|
||||
custom: location,
|
||||
node: node.node,
|
||||
})
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue