diff --git a/ast/asdl_rs.py b/ast/asdl_rs.py index dca65ce..b05f654 100755 --- a/ast/asdl_rs.py +++ b/ast/asdl_rs.py @@ -1951,6 +1951,40 @@ def write_pyo3_wrapper(mod, type_info, namespace, f): f.write("Ok(())\n}") +def write_parse_def(mod, type_info, f): + for info in type_info.values(): + if info.enum_name not in ["expr", "stmt"]: + continue + + type_name = rust_type_name(info.enum_name) + cons_name = rust_type_name(info.name) + + f.write(f""" + impl Parse for ast::{info.rust_sum_name} {{ + fn lex_starts_at( + source: &str, + offset: TextSize, + ) -> SoftKeywordTransformer> {{ + ast::{type_name}::lex_starts_at(source, offset) + }} + fn parse_tokens( + lxr: impl IntoIterator, + source_path: &str, + ) -> Result {{ + let node = ast::{type_name}::parse_tokens(lxr, source_path)?; + match node {{ + ast::{type_name}::{cons_name}(node) => Ok(node), + node => Err(ParseError {{ + error: ParseErrorType::InvalidToken, + offset: node.range().start(), + source_path: source_path.to_owned(), + }}), + }} + }} + }} + """) + + def write_ast_mod(mod, type_info, f): f.write( """ @@ -1972,6 +2006,7 @@ def write_ast_mod(mod, type_info, f): def main( input_filename, ast_dir, + parser_dir, ast_pyo3_dir, module_filename, dump_module=False, @@ -2000,6 +2035,14 @@ def main( f.write(auto_gen_msg) write(f) + for filename, write in [ + ("parse", p(write_parse_def, mod, type_info)), + ]: + with (parser_dir / f"{filename}.rs").open("w") as f: + f.write(auto_gen_msg) + write(f) + + for filename, write in [ ("to_py_ast", p(write_to_pyo3, mod, type_info)), ("wrapper_located", p(write_pyo3_wrapper, mod, type_info, "located")), @@ -2020,6 +2063,7 @@ if __name__ == "__main__": parser = ArgumentParser() parser.add_argument("input_file", type=Path) parser.add_argument("-A", "--ast-dir", type=Path, required=True) + parser.add_argument("-P", "--parser-dir", type=Path, required=True) parser.add_argument("-O", "--ast-pyo3-dir", type=Path, required=True) parser.add_argument("-M", "--module-file", type=Path, required=True) parser.add_argument("-d", "--dump-module", action="store_true") @@ -2028,6 +2072,7 @@ if __name__ == "__main__": main( args.input_file, args.ast_dir, + args.parser_dir, args.ast_pyo3_dir, args.module_file, args.dump_module, diff --git a/parser/src/gen/parse.rs b/parser/src/gen/parse.rs new file mode 100644 index 0000000..01856ac --- /dev/null +++ b/parser/src/gen/parse.rs @@ -0,0 +1,1243 @@ +// File automatically generated by ast/asdl_rs.py. + +impl Parse for ast::StmtFunctionDef { + fn lex_starts_at( + source: &str, + offset: TextSize, + ) -> SoftKeywordTransformer> { + ast::Stmt::lex_starts_at(source, offset) + } + fn parse_tokens( + lxr: impl IntoIterator, + source_path: &str, + ) -> Result { + let node = ast::Stmt::parse_tokens(lxr, source_path)?; + match node { + ast::Stmt::FunctionDef(node) => Ok(node), + node => Err(ParseError { + error: ParseErrorType::InvalidToken, + offset: node.range().start(), + source_path: source_path.to_owned(), + }), + } + } +} + +impl Parse for ast::StmtAsyncFunctionDef { + fn lex_starts_at( + source: &str, + offset: TextSize, + ) -> SoftKeywordTransformer> { + ast::Stmt::lex_starts_at(source, offset) + } + fn parse_tokens( + lxr: impl IntoIterator, + source_path: &str, + ) -> Result { + let node = ast::Stmt::parse_tokens(lxr, source_path)?; + match node { + ast::Stmt::AsyncFunctionDef(node) => Ok(node), + node => Err(ParseError { + error: ParseErrorType::InvalidToken, + offset: node.range().start(), + source_path: source_path.to_owned(), + }), + } + } +} + +impl Parse for ast::StmtClassDef { + fn lex_starts_at( + source: &str, + offset: TextSize, + ) -> SoftKeywordTransformer> { + ast::Stmt::lex_starts_at(source, offset) + } + fn parse_tokens( + lxr: impl IntoIterator, + source_path: &str, + ) -> Result { + let node = ast::Stmt::parse_tokens(lxr, source_path)?; + match node { + ast::Stmt::ClassDef(node) => Ok(node), + node => Err(ParseError { + error: ParseErrorType::InvalidToken, + offset: node.range().start(), + source_path: source_path.to_owned(), + }), + } + } +} + +impl Parse for ast::StmtReturn { + fn lex_starts_at( + source: &str, + offset: TextSize, + ) -> SoftKeywordTransformer> { + ast::Stmt::lex_starts_at(source, offset) + } + fn parse_tokens( + lxr: impl IntoIterator, + source_path: &str, + ) -> Result { + let node = ast::Stmt::parse_tokens(lxr, source_path)?; + match node { + ast::Stmt::Return(node) => Ok(node), + node => Err(ParseError { + error: ParseErrorType::InvalidToken, + offset: node.range().start(), + source_path: source_path.to_owned(), + }), + } + } +} + +impl Parse for ast::StmtDelete { + fn lex_starts_at( + source: &str, + offset: TextSize, + ) -> SoftKeywordTransformer> { + ast::Stmt::lex_starts_at(source, offset) + } + fn parse_tokens( + lxr: impl IntoIterator, + source_path: &str, + ) -> Result { + let node = ast::Stmt::parse_tokens(lxr, source_path)?; + match node { + ast::Stmt::Delete(node) => Ok(node), + node => Err(ParseError { + error: ParseErrorType::InvalidToken, + offset: node.range().start(), + source_path: source_path.to_owned(), + }), + } + } +} + +impl Parse for ast::StmtAssign { + fn lex_starts_at( + source: &str, + offset: TextSize, + ) -> SoftKeywordTransformer> { + ast::Stmt::lex_starts_at(source, offset) + } + fn parse_tokens( + lxr: impl IntoIterator, + source_path: &str, + ) -> Result { + let node = ast::Stmt::parse_tokens(lxr, source_path)?; + match node { + ast::Stmt::Assign(node) => Ok(node), + node => Err(ParseError { + error: ParseErrorType::InvalidToken, + offset: node.range().start(), + source_path: source_path.to_owned(), + }), + } + } +} + +impl Parse for ast::StmtAugAssign { + fn lex_starts_at( + source: &str, + offset: TextSize, + ) -> SoftKeywordTransformer> { + ast::Stmt::lex_starts_at(source, offset) + } + fn parse_tokens( + lxr: impl IntoIterator, + source_path: &str, + ) -> Result { + let node = ast::Stmt::parse_tokens(lxr, source_path)?; + match node { + ast::Stmt::AugAssign(node) => Ok(node), + node => Err(ParseError { + error: ParseErrorType::InvalidToken, + offset: node.range().start(), + source_path: source_path.to_owned(), + }), + } + } +} + +impl Parse for ast::StmtAnnAssign { + fn lex_starts_at( + source: &str, + offset: TextSize, + ) -> SoftKeywordTransformer> { + ast::Stmt::lex_starts_at(source, offset) + } + fn parse_tokens( + lxr: impl IntoIterator, + source_path: &str, + ) -> Result { + let node = ast::Stmt::parse_tokens(lxr, source_path)?; + match node { + ast::Stmt::AnnAssign(node) => Ok(node), + node => Err(ParseError { + error: ParseErrorType::InvalidToken, + offset: node.range().start(), + source_path: source_path.to_owned(), + }), + } + } +} + +impl Parse for ast::StmtFor { + fn lex_starts_at( + source: &str, + offset: TextSize, + ) -> SoftKeywordTransformer> { + ast::Stmt::lex_starts_at(source, offset) + } + fn parse_tokens( + lxr: impl IntoIterator, + source_path: &str, + ) -> Result { + let node = ast::Stmt::parse_tokens(lxr, source_path)?; + match node { + ast::Stmt::For(node) => Ok(node), + node => Err(ParseError { + error: ParseErrorType::InvalidToken, + offset: node.range().start(), + source_path: source_path.to_owned(), + }), + } + } +} + +impl Parse for ast::StmtAsyncFor { + fn lex_starts_at( + source: &str, + offset: TextSize, + ) -> SoftKeywordTransformer> { + ast::Stmt::lex_starts_at(source, offset) + } + fn parse_tokens( + lxr: impl IntoIterator, + source_path: &str, + ) -> Result { + let node = ast::Stmt::parse_tokens(lxr, source_path)?; + match node { + ast::Stmt::AsyncFor(node) => Ok(node), + node => Err(ParseError { + error: ParseErrorType::InvalidToken, + offset: node.range().start(), + source_path: source_path.to_owned(), + }), + } + } +} + +impl Parse for ast::StmtWhile { + fn lex_starts_at( + source: &str, + offset: TextSize, + ) -> SoftKeywordTransformer> { + ast::Stmt::lex_starts_at(source, offset) + } + fn parse_tokens( + lxr: impl IntoIterator, + source_path: &str, + ) -> Result { + let node = ast::Stmt::parse_tokens(lxr, source_path)?; + match node { + ast::Stmt::While(node) => Ok(node), + node => Err(ParseError { + error: ParseErrorType::InvalidToken, + offset: node.range().start(), + source_path: source_path.to_owned(), + }), + } + } +} + +impl Parse for ast::StmtIf { + fn lex_starts_at( + source: &str, + offset: TextSize, + ) -> SoftKeywordTransformer> { + ast::Stmt::lex_starts_at(source, offset) + } + fn parse_tokens( + lxr: impl IntoIterator, + source_path: &str, + ) -> Result { + let node = ast::Stmt::parse_tokens(lxr, source_path)?; + match node { + ast::Stmt::If(node) => Ok(node), + node => Err(ParseError { + error: ParseErrorType::InvalidToken, + offset: node.range().start(), + source_path: source_path.to_owned(), + }), + } + } +} + +impl Parse for ast::StmtWith { + fn lex_starts_at( + source: &str, + offset: TextSize, + ) -> SoftKeywordTransformer> { + ast::Stmt::lex_starts_at(source, offset) + } + fn parse_tokens( + lxr: impl IntoIterator, + source_path: &str, + ) -> Result { + let node = ast::Stmt::parse_tokens(lxr, source_path)?; + match node { + ast::Stmt::With(node) => Ok(node), + node => Err(ParseError { + error: ParseErrorType::InvalidToken, + offset: node.range().start(), + source_path: source_path.to_owned(), + }), + } + } +} + +impl Parse for ast::StmtAsyncWith { + fn lex_starts_at( + source: &str, + offset: TextSize, + ) -> SoftKeywordTransformer> { + ast::Stmt::lex_starts_at(source, offset) + } + fn parse_tokens( + lxr: impl IntoIterator, + source_path: &str, + ) -> Result { + let node = ast::Stmt::parse_tokens(lxr, source_path)?; + match node { + ast::Stmt::AsyncWith(node) => Ok(node), + node => Err(ParseError { + error: ParseErrorType::InvalidToken, + offset: node.range().start(), + source_path: source_path.to_owned(), + }), + } + } +} + +impl Parse for ast::StmtMatch { + fn lex_starts_at( + source: &str, + offset: TextSize, + ) -> SoftKeywordTransformer> { + ast::Stmt::lex_starts_at(source, offset) + } + fn parse_tokens( + lxr: impl IntoIterator, + source_path: &str, + ) -> Result { + let node = ast::Stmt::parse_tokens(lxr, source_path)?; + match node { + ast::Stmt::Match(node) => Ok(node), + node => Err(ParseError { + error: ParseErrorType::InvalidToken, + offset: node.range().start(), + source_path: source_path.to_owned(), + }), + } + } +} + +impl Parse for ast::StmtRaise { + fn lex_starts_at( + source: &str, + offset: TextSize, + ) -> SoftKeywordTransformer> { + ast::Stmt::lex_starts_at(source, offset) + } + fn parse_tokens( + lxr: impl IntoIterator, + source_path: &str, + ) -> Result { + let node = ast::Stmt::parse_tokens(lxr, source_path)?; + match node { + ast::Stmt::Raise(node) => Ok(node), + node => Err(ParseError { + error: ParseErrorType::InvalidToken, + offset: node.range().start(), + source_path: source_path.to_owned(), + }), + } + } +} + +impl Parse for ast::StmtTry { + fn lex_starts_at( + source: &str, + offset: TextSize, + ) -> SoftKeywordTransformer> { + ast::Stmt::lex_starts_at(source, offset) + } + fn parse_tokens( + lxr: impl IntoIterator, + source_path: &str, + ) -> Result { + let node = ast::Stmt::parse_tokens(lxr, source_path)?; + match node { + ast::Stmt::Try(node) => Ok(node), + node => Err(ParseError { + error: ParseErrorType::InvalidToken, + offset: node.range().start(), + source_path: source_path.to_owned(), + }), + } + } +} + +impl Parse for ast::StmtTryStar { + fn lex_starts_at( + source: &str, + offset: TextSize, + ) -> SoftKeywordTransformer> { + ast::Stmt::lex_starts_at(source, offset) + } + fn parse_tokens( + lxr: impl IntoIterator, + source_path: &str, + ) -> Result { + let node = ast::Stmt::parse_tokens(lxr, source_path)?; + match node { + ast::Stmt::TryStar(node) => Ok(node), + node => Err(ParseError { + error: ParseErrorType::InvalidToken, + offset: node.range().start(), + source_path: source_path.to_owned(), + }), + } + } +} + +impl Parse for ast::StmtAssert { + fn lex_starts_at( + source: &str, + offset: TextSize, + ) -> SoftKeywordTransformer> { + ast::Stmt::lex_starts_at(source, offset) + } + fn parse_tokens( + lxr: impl IntoIterator, + source_path: &str, + ) -> Result { + let node = ast::Stmt::parse_tokens(lxr, source_path)?; + match node { + ast::Stmt::Assert(node) => Ok(node), + node => Err(ParseError { + error: ParseErrorType::InvalidToken, + offset: node.range().start(), + source_path: source_path.to_owned(), + }), + } + } +} + +impl Parse for ast::StmtImport { + fn lex_starts_at( + source: &str, + offset: TextSize, + ) -> SoftKeywordTransformer> { + ast::Stmt::lex_starts_at(source, offset) + } + fn parse_tokens( + lxr: impl IntoIterator, + source_path: &str, + ) -> Result { + let node = ast::Stmt::parse_tokens(lxr, source_path)?; + match node { + ast::Stmt::Import(node) => Ok(node), + node => Err(ParseError { + error: ParseErrorType::InvalidToken, + offset: node.range().start(), + source_path: source_path.to_owned(), + }), + } + } +} + +impl Parse for ast::StmtImportFrom { + fn lex_starts_at( + source: &str, + offset: TextSize, + ) -> SoftKeywordTransformer> { + ast::Stmt::lex_starts_at(source, offset) + } + fn parse_tokens( + lxr: impl IntoIterator, + source_path: &str, + ) -> Result { + let node = ast::Stmt::parse_tokens(lxr, source_path)?; + match node { + ast::Stmt::ImportFrom(node) => Ok(node), + node => Err(ParseError { + error: ParseErrorType::InvalidToken, + offset: node.range().start(), + source_path: source_path.to_owned(), + }), + } + } +} + +impl Parse for ast::StmtGlobal { + fn lex_starts_at( + source: &str, + offset: TextSize, + ) -> SoftKeywordTransformer> { + ast::Stmt::lex_starts_at(source, offset) + } + fn parse_tokens( + lxr: impl IntoIterator, + source_path: &str, + ) -> Result { + let node = ast::Stmt::parse_tokens(lxr, source_path)?; + match node { + ast::Stmt::Global(node) => Ok(node), + node => Err(ParseError { + error: ParseErrorType::InvalidToken, + offset: node.range().start(), + source_path: source_path.to_owned(), + }), + } + } +} + +impl Parse for ast::StmtNonlocal { + fn lex_starts_at( + source: &str, + offset: TextSize, + ) -> SoftKeywordTransformer> { + ast::Stmt::lex_starts_at(source, offset) + } + fn parse_tokens( + lxr: impl IntoIterator, + source_path: &str, + ) -> Result { + let node = ast::Stmt::parse_tokens(lxr, source_path)?; + match node { + ast::Stmt::Nonlocal(node) => Ok(node), + node => Err(ParseError { + error: ParseErrorType::InvalidToken, + offset: node.range().start(), + source_path: source_path.to_owned(), + }), + } + } +} + +impl Parse for ast::StmtExpr { + fn lex_starts_at( + source: &str, + offset: TextSize, + ) -> SoftKeywordTransformer> { + ast::Stmt::lex_starts_at(source, offset) + } + fn parse_tokens( + lxr: impl IntoIterator, + source_path: &str, + ) -> Result { + let node = ast::Stmt::parse_tokens(lxr, source_path)?; + match node { + ast::Stmt::Expr(node) => Ok(node), + node => Err(ParseError { + error: ParseErrorType::InvalidToken, + offset: node.range().start(), + source_path: source_path.to_owned(), + }), + } + } +} + +impl Parse for ast::StmtPass { + fn lex_starts_at( + source: &str, + offset: TextSize, + ) -> SoftKeywordTransformer> { + ast::Stmt::lex_starts_at(source, offset) + } + fn parse_tokens( + lxr: impl IntoIterator, + source_path: &str, + ) -> Result { + let node = ast::Stmt::parse_tokens(lxr, source_path)?; + match node { + ast::Stmt::Pass(node) => Ok(node), + node => Err(ParseError { + error: ParseErrorType::InvalidToken, + offset: node.range().start(), + source_path: source_path.to_owned(), + }), + } + } +} + +impl Parse for ast::StmtBreak { + fn lex_starts_at( + source: &str, + offset: TextSize, + ) -> SoftKeywordTransformer> { + ast::Stmt::lex_starts_at(source, offset) + } + fn parse_tokens( + lxr: impl IntoIterator, + source_path: &str, + ) -> Result { + let node = ast::Stmt::parse_tokens(lxr, source_path)?; + match node { + ast::Stmt::Break(node) => Ok(node), + node => Err(ParseError { + error: ParseErrorType::InvalidToken, + offset: node.range().start(), + source_path: source_path.to_owned(), + }), + } + } +} + +impl Parse for ast::StmtContinue { + fn lex_starts_at( + source: &str, + offset: TextSize, + ) -> SoftKeywordTransformer> { + ast::Stmt::lex_starts_at(source, offset) + } + fn parse_tokens( + lxr: impl IntoIterator, + source_path: &str, + ) -> Result { + let node = ast::Stmt::parse_tokens(lxr, source_path)?; + match node { + ast::Stmt::Continue(node) => Ok(node), + node => Err(ParseError { + error: ParseErrorType::InvalidToken, + offset: node.range().start(), + source_path: source_path.to_owned(), + }), + } + } +} + +impl Parse for ast::ExprBoolOp { + fn lex_starts_at( + source: &str, + offset: TextSize, + ) -> SoftKeywordTransformer> { + ast::Expr::lex_starts_at(source, offset) + } + fn parse_tokens( + lxr: impl IntoIterator, + source_path: &str, + ) -> Result { + let node = ast::Expr::parse_tokens(lxr, source_path)?; + match node { + ast::Expr::BoolOp(node) => Ok(node), + node => Err(ParseError { + error: ParseErrorType::InvalidToken, + offset: node.range().start(), + source_path: source_path.to_owned(), + }), + } + } +} + +impl Parse for ast::ExprNamedExpr { + fn lex_starts_at( + source: &str, + offset: TextSize, + ) -> SoftKeywordTransformer> { + ast::Expr::lex_starts_at(source, offset) + } + fn parse_tokens( + lxr: impl IntoIterator, + source_path: &str, + ) -> Result { + let node = ast::Expr::parse_tokens(lxr, source_path)?; + match node { + ast::Expr::NamedExpr(node) => Ok(node), + node => Err(ParseError { + error: ParseErrorType::InvalidToken, + offset: node.range().start(), + source_path: source_path.to_owned(), + }), + } + } +} + +impl Parse for ast::ExprBinOp { + fn lex_starts_at( + source: &str, + offset: TextSize, + ) -> SoftKeywordTransformer> { + ast::Expr::lex_starts_at(source, offset) + } + fn parse_tokens( + lxr: impl IntoIterator, + source_path: &str, + ) -> Result { + let node = ast::Expr::parse_tokens(lxr, source_path)?; + match node { + ast::Expr::BinOp(node) => Ok(node), + node => Err(ParseError { + error: ParseErrorType::InvalidToken, + offset: node.range().start(), + source_path: source_path.to_owned(), + }), + } + } +} + +impl Parse for ast::ExprUnaryOp { + fn lex_starts_at( + source: &str, + offset: TextSize, + ) -> SoftKeywordTransformer> { + ast::Expr::lex_starts_at(source, offset) + } + fn parse_tokens( + lxr: impl IntoIterator, + source_path: &str, + ) -> Result { + let node = ast::Expr::parse_tokens(lxr, source_path)?; + match node { + ast::Expr::UnaryOp(node) => Ok(node), + node => Err(ParseError { + error: ParseErrorType::InvalidToken, + offset: node.range().start(), + source_path: source_path.to_owned(), + }), + } + } +} + +impl Parse for ast::ExprLambda { + fn lex_starts_at( + source: &str, + offset: TextSize, + ) -> SoftKeywordTransformer> { + ast::Expr::lex_starts_at(source, offset) + } + fn parse_tokens( + lxr: impl IntoIterator, + source_path: &str, + ) -> Result { + let node = ast::Expr::parse_tokens(lxr, source_path)?; + match node { + ast::Expr::Lambda(node) => Ok(node), + node => Err(ParseError { + error: ParseErrorType::InvalidToken, + offset: node.range().start(), + source_path: source_path.to_owned(), + }), + } + } +} + +impl Parse for ast::ExprIfExp { + fn lex_starts_at( + source: &str, + offset: TextSize, + ) -> SoftKeywordTransformer> { + ast::Expr::lex_starts_at(source, offset) + } + fn parse_tokens( + lxr: impl IntoIterator, + source_path: &str, + ) -> Result { + let node = ast::Expr::parse_tokens(lxr, source_path)?; + match node { + ast::Expr::IfExp(node) => Ok(node), + node => Err(ParseError { + error: ParseErrorType::InvalidToken, + offset: node.range().start(), + source_path: source_path.to_owned(), + }), + } + } +} + +impl Parse for ast::ExprDict { + fn lex_starts_at( + source: &str, + offset: TextSize, + ) -> SoftKeywordTransformer> { + ast::Expr::lex_starts_at(source, offset) + } + fn parse_tokens( + lxr: impl IntoIterator, + source_path: &str, + ) -> Result { + let node = ast::Expr::parse_tokens(lxr, source_path)?; + match node { + ast::Expr::Dict(node) => Ok(node), + node => Err(ParseError { + error: ParseErrorType::InvalidToken, + offset: node.range().start(), + source_path: source_path.to_owned(), + }), + } + } +} + +impl Parse for ast::ExprSet { + fn lex_starts_at( + source: &str, + offset: TextSize, + ) -> SoftKeywordTransformer> { + ast::Expr::lex_starts_at(source, offset) + } + fn parse_tokens( + lxr: impl IntoIterator, + source_path: &str, + ) -> Result { + let node = ast::Expr::parse_tokens(lxr, source_path)?; + match node { + ast::Expr::Set(node) => Ok(node), + node => Err(ParseError { + error: ParseErrorType::InvalidToken, + offset: node.range().start(), + source_path: source_path.to_owned(), + }), + } + } +} + +impl Parse for ast::ExprListComp { + fn lex_starts_at( + source: &str, + offset: TextSize, + ) -> SoftKeywordTransformer> { + ast::Expr::lex_starts_at(source, offset) + } + fn parse_tokens( + lxr: impl IntoIterator, + source_path: &str, + ) -> Result { + let node = ast::Expr::parse_tokens(lxr, source_path)?; + match node { + ast::Expr::ListComp(node) => Ok(node), + node => Err(ParseError { + error: ParseErrorType::InvalidToken, + offset: node.range().start(), + source_path: source_path.to_owned(), + }), + } + } +} + +impl Parse for ast::ExprSetComp { + fn lex_starts_at( + source: &str, + offset: TextSize, + ) -> SoftKeywordTransformer> { + ast::Expr::lex_starts_at(source, offset) + } + fn parse_tokens( + lxr: impl IntoIterator, + source_path: &str, + ) -> Result { + let node = ast::Expr::parse_tokens(lxr, source_path)?; + match node { + ast::Expr::SetComp(node) => Ok(node), + node => Err(ParseError { + error: ParseErrorType::InvalidToken, + offset: node.range().start(), + source_path: source_path.to_owned(), + }), + } + } +} + +impl Parse for ast::ExprDictComp { + fn lex_starts_at( + source: &str, + offset: TextSize, + ) -> SoftKeywordTransformer> { + ast::Expr::lex_starts_at(source, offset) + } + fn parse_tokens( + lxr: impl IntoIterator, + source_path: &str, + ) -> Result { + let node = ast::Expr::parse_tokens(lxr, source_path)?; + match node { + ast::Expr::DictComp(node) => Ok(node), + node => Err(ParseError { + error: ParseErrorType::InvalidToken, + offset: node.range().start(), + source_path: source_path.to_owned(), + }), + } + } +} + +impl Parse for ast::ExprGeneratorExp { + fn lex_starts_at( + source: &str, + offset: TextSize, + ) -> SoftKeywordTransformer> { + ast::Expr::lex_starts_at(source, offset) + } + fn parse_tokens( + lxr: impl IntoIterator, + source_path: &str, + ) -> Result { + let node = ast::Expr::parse_tokens(lxr, source_path)?; + match node { + ast::Expr::GeneratorExp(node) => Ok(node), + node => Err(ParseError { + error: ParseErrorType::InvalidToken, + offset: node.range().start(), + source_path: source_path.to_owned(), + }), + } + } +} + +impl Parse for ast::ExprAwait { + fn lex_starts_at( + source: &str, + offset: TextSize, + ) -> SoftKeywordTransformer> { + ast::Expr::lex_starts_at(source, offset) + } + fn parse_tokens( + lxr: impl IntoIterator, + source_path: &str, + ) -> Result { + let node = ast::Expr::parse_tokens(lxr, source_path)?; + match node { + ast::Expr::Await(node) => Ok(node), + node => Err(ParseError { + error: ParseErrorType::InvalidToken, + offset: node.range().start(), + source_path: source_path.to_owned(), + }), + } + } +} + +impl Parse for ast::ExprYield { + fn lex_starts_at( + source: &str, + offset: TextSize, + ) -> SoftKeywordTransformer> { + ast::Expr::lex_starts_at(source, offset) + } + fn parse_tokens( + lxr: impl IntoIterator, + source_path: &str, + ) -> Result { + let node = ast::Expr::parse_tokens(lxr, source_path)?; + match node { + ast::Expr::Yield(node) => Ok(node), + node => Err(ParseError { + error: ParseErrorType::InvalidToken, + offset: node.range().start(), + source_path: source_path.to_owned(), + }), + } + } +} + +impl Parse for ast::ExprYieldFrom { + fn lex_starts_at( + source: &str, + offset: TextSize, + ) -> SoftKeywordTransformer> { + ast::Expr::lex_starts_at(source, offset) + } + fn parse_tokens( + lxr: impl IntoIterator, + source_path: &str, + ) -> Result { + let node = ast::Expr::parse_tokens(lxr, source_path)?; + match node { + ast::Expr::YieldFrom(node) => Ok(node), + node => Err(ParseError { + error: ParseErrorType::InvalidToken, + offset: node.range().start(), + source_path: source_path.to_owned(), + }), + } + } +} + +impl Parse for ast::ExprCompare { + fn lex_starts_at( + source: &str, + offset: TextSize, + ) -> SoftKeywordTransformer> { + ast::Expr::lex_starts_at(source, offset) + } + fn parse_tokens( + lxr: impl IntoIterator, + source_path: &str, + ) -> Result { + let node = ast::Expr::parse_tokens(lxr, source_path)?; + match node { + ast::Expr::Compare(node) => Ok(node), + node => Err(ParseError { + error: ParseErrorType::InvalidToken, + offset: node.range().start(), + source_path: source_path.to_owned(), + }), + } + } +} + +impl Parse for ast::ExprCall { + fn lex_starts_at( + source: &str, + offset: TextSize, + ) -> SoftKeywordTransformer> { + ast::Expr::lex_starts_at(source, offset) + } + fn parse_tokens( + lxr: impl IntoIterator, + source_path: &str, + ) -> Result { + let node = ast::Expr::parse_tokens(lxr, source_path)?; + match node { + ast::Expr::Call(node) => Ok(node), + node => Err(ParseError { + error: ParseErrorType::InvalidToken, + offset: node.range().start(), + source_path: source_path.to_owned(), + }), + } + } +} + +impl Parse for ast::ExprFormattedValue { + fn lex_starts_at( + source: &str, + offset: TextSize, + ) -> SoftKeywordTransformer> { + ast::Expr::lex_starts_at(source, offset) + } + fn parse_tokens( + lxr: impl IntoIterator, + source_path: &str, + ) -> Result { + let node = ast::Expr::parse_tokens(lxr, source_path)?; + match node { + ast::Expr::FormattedValue(node) => Ok(node), + node => Err(ParseError { + error: ParseErrorType::InvalidToken, + offset: node.range().start(), + source_path: source_path.to_owned(), + }), + } + } +} + +impl Parse for ast::ExprJoinedStr { + fn lex_starts_at( + source: &str, + offset: TextSize, + ) -> SoftKeywordTransformer> { + ast::Expr::lex_starts_at(source, offset) + } + fn parse_tokens( + lxr: impl IntoIterator, + source_path: &str, + ) -> Result { + let node = ast::Expr::parse_tokens(lxr, source_path)?; + match node { + ast::Expr::JoinedStr(node) => Ok(node), + node => Err(ParseError { + error: ParseErrorType::InvalidToken, + offset: node.range().start(), + source_path: source_path.to_owned(), + }), + } + } +} + +impl Parse for ast::ExprConstant { + fn lex_starts_at( + source: &str, + offset: TextSize, + ) -> SoftKeywordTransformer> { + ast::Expr::lex_starts_at(source, offset) + } + fn parse_tokens( + lxr: impl IntoIterator, + source_path: &str, + ) -> Result { + let node = ast::Expr::parse_tokens(lxr, source_path)?; + match node { + ast::Expr::Constant(node) => Ok(node), + node => Err(ParseError { + error: ParseErrorType::InvalidToken, + offset: node.range().start(), + source_path: source_path.to_owned(), + }), + } + } +} + +impl Parse for ast::ExprAttribute { + fn lex_starts_at( + source: &str, + offset: TextSize, + ) -> SoftKeywordTransformer> { + ast::Expr::lex_starts_at(source, offset) + } + fn parse_tokens( + lxr: impl IntoIterator, + source_path: &str, + ) -> Result { + let node = ast::Expr::parse_tokens(lxr, source_path)?; + match node { + ast::Expr::Attribute(node) => Ok(node), + node => Err(ParseError { + error: ParseErrorType::InvalidToken, + offset: node.range().start(), + source_path: source_path.to_owned(), + }), + } + } +} + +impl Parse for ast::ExprSubscript { + fn lex_starts_at( + source: &str, + offset: TextSize, + ) -> SoftKeywordTransformer> { + ast::Expr::lex_starts_at(source, offset) + } + fn parse_tokens( + lxr: impl IntoIterator, + source_path: &str, + ) -> Result { + let node = ast::Expr::parse_tokens(lxr, source_path)?; + match node { + ast::Expr::Subscript(node) => Ok(node), + node => Err(ParseError { + error: ParseErrorType::InvalidToken, + offset: node.range().start(), + source_path: source_path.to_owned(), + }), + } + } +} + +impl Parse for ast::ExprStarred { + fn lex_starts_at( + source: &str, + offset: TextSize, + ) -> SoftKeywordTransformer> { + ast::Expr::lex_starts_at(source, offset) + } + fn parse_tokens( + lxr: impl IntoIterator, + source_path: &str, + ) -> Result { + let node = ast::Expr::parse_tokens(lxr, source_path)?; + match node { + ast::Expr::Starred(node) => Ok(node), + node => Err(ParseError { + error: ParseErrorType::InvalidToken, + offset: node.range().start(), + source_path: source_path.to_owned(), + }), + } + } +} + +impl Parse for ast::ExprName { + fn lex_starts_at( + source: &str, + offset: TextSize, + ) -> SoftKeywordTransformer> { + ast::Expr::lex_starts_at(source, offset) + } + fn parse_tokens( + lxr: impl IntoIterator, + source_path: &str, + ) -> Result { + let node = ast::Expr::parse_tokens(lxr, source_path)?; + match node { + ast::Expr::Name(node) => Ok(node), + node => Err(ParseError { + error: ParseErrorType::InvalidToken, + offset: node.range().start(), + source_path: source_path.to_owned(), + }), + } + } +} + +impl Parse for ast::ExprList { + fn lex_starts_at( + source: &str, + offset: TextSize, + ) -> SoftKeywordTransformer> { + ast::Expr::lex_starts_at(source, offset) + } + fn parse_tokens( + lxr: impl IntoIterator, + source_path: &str, + ) -> Result { + let node = ast::Expr::parse_tokens(lxr, source_path)?; + match node { + ast::Expr::List(node) => Ok(node), + node => Err(ParseError { + error: ParseErrorType::InvalidToken, + offset: node.range().start(), + source_path: source_path.to_owned(), + }), + } + } +} + +impl Parse for ast::ExprTuple { + fn lex_starts_at( + source: &str, + offset: TextSize, + ) -> SoftKeywordTransformer> { + ast::Expr::lex_starts_at(source, offset) + } + fn parse_tokens( + lxr: impl IntoIterator, + source_path: &str, + ) -> Result { + let node = ast::Expr::parse_tokens(lxr, source_path)?; + match node { + ast::Expr::Tuple(node) => Ok(node), + node => Err(ParseError { + error: ParseErrorType::InvalidToken, + offset: node.range().start(), + source_path: source_path.to_owned(), + }), + } + } +} + +impl Parse for ast::ExprSlice { + fn lex_starts_at( + source: &str, + offset: TextSize, + ) -> SoftKeywordTransformer> { + ast::Expr::lex_starts_at(source, offset) + } + fn parse_tokens( + lxr: impl IntoIterator, + source_path: &str, + ) -> Result { + let node = ast::Expr::parse_tokens(lxr, source_path)?; + match node { + ast::Expr::Slice(node) => Ok(node), + node => Err(ParseError { + error: ParseErrorType::InvalidToken, + offset: node.range().start(), + source_path: source_path.to_owned(), + }), + } + } +} diff --git a/parser/src/parser.rs b/parser/src/parser.rs index 60001e0..ca579cf 100644 --- a/parser/src/parser.rs +++ b/parser/src/parser.rs @@ -59,6 +59,9 @@ where fn parse(source: &str, source_path: &str) -> Result { Self::parse_starts_at(source, source_path, TextSize::default()) } + fn parse_without_path(source: &str) -> Result { + Self::parse(source, "") + } fn parse_starts_at( source: &str, source_path: &str, @@ -197,6 +200,52 @@ impl Parse for ast::Expr { } } +impl Parse for ast::Identifier { + fn lex_starts_at( + source: &str, + offset: TextSize, + ) -> SoftKeywordTransformer> { + ast::Expr::lex_starts_at(source, offset) + } + fn parse_tokens( + lxr: impl IntoIterator, + source_path: &str, + ) -> Result { + let expr = ast::Expr::parse_tokens(lxr, source_path)?; + match expr { + ast::Expr::Name(name) => Ok(name.id), + expr => Err(ParseError { + error: ParseErrorType::InvalidToken, + offset: expr.range().start(), + source_path: source_path.to_owned(), + }), + } + } +} + +impl Parse for ast::Constant { + fn lex_starts_at( + source: &str, + offset: TextSize, + ) -> SoftKeywordTransformer> { + ast::Expr::lex_starts_at(source, offset) + } + fn parse_tokens( + lxr: impl IntoIterator, + source_path: &str, + ) -> Result { + let expr = ast::Expr::parse_tokens(lxr, source_path)?; + match expr { + ast::Expr::Constant(c) => Ok(c.value), + expr => Err(ParseError { + error: ParseErrorType::InvalidToken, + offset: expr.range().start(), + source_path: source_path.to_owned(), + }), + } + } +} + /// Parse a full Python program usually consisting of multiple lines. /// /// This is a convenience function that can be used to parse a full Python program without having to @@ -505,6 +554,8 @@ pub(super) fn optional_range(start: TextSize, end: TextSize) -> OptionalRange::new(start, end) } +include!("gen/parse.rs"); + #[cfg(test)] mod tests { use super::*; @@ -1033,4 +1084,21 @@ def args_to_tuple(*args: *Ts) -> Tuple[*Ts]: ... .unwrap(); insta::assert_debug_snapshot!(parse_ast); } + + #[test] + fn test_parse_constant() { + use num_traits::ToPrimitive; + + let c = ast::Constant::parse_without_path("'string'").unwrap(); + assert_eq!(c.str().unwrap(), "string"); + + let c = ast::Constant::parse_without_path("10").unwrap(); + assert_eq!(c.int().unwrap().to_i32().unwrap(), 10); + } + + #[test] + fn test_parse_identifier() { + let i = ast::Identifier::parse_without_path("test").unwrap(); + assert_eq!(i.as_str(), "test"); + } } diff --git a/scripts/update_asdl.sh b/scripts/update_asdl.sh index af34cd2..c7115a5 100755 --- a/scripts/update_asdl.sh +++ b/scripts/update_asdl.sh @@ -4,5 +4,5 @@ set -e cd "$(dirname "$(dirname "$0")")" # rm ast/src/gen/*.rs -python ast/asdl_rs.py --ast-dir ast/src/gen/ --ast-pyo3-dir ast-pyo3/src/gen/ --module-file ../RustPython/vm/src/stdlib/ast/gen.rs ast/Python.asdl -rustfmt ast/src/gen/*.rs ast-pyo3/src/gen/*.rs ../RustPython/vm/src/stdlib/ast/gen.rs +python ast/asdl_rs.py --ast-dir ast/src/gen/ --parser-dir parser/src/gen/ --ast-pyo3-dir ast-pyo3/src/gen/ --module-file ../RustPython/vm/src/stdlib/ast/gen.rs ast/Python.asdl +rustfmt ast/src/gen/*.rs parser/src/gen/*.rs ast-pyo3/src/gen/*.rs ../RustPython/vm/src/stdlib/ast/gen.rs