Merge pull request #18790 from ChayimFriedman2/proper-make

internal: Create a quoting mechanism instead of textual AST make
This commit is contained in:
Lukas Wirth 2024-12-30 11:39:29 +00:00 committed by GitHub
commit 2e13684be1
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
8 changed files with 1672 additions and 60 deletions

View file

@ -203,6 +203,8 @@ new_ret_no_self = "allow"
useless_asref = "allow"
# Has false positives
assigning_clones = "allow"
# Does not work with macros
vec_init_then_push = "allow"
## Following lints should be tackled at some point
too_many_arguments = "allow"

View file

@ -85,7 +85,6 @@ pub(crate) fn generate_fn_type_alias(acc: &mut Assists, ctx: &AssistContext<'_>)
let is_unsafe = func_node.unsafe_token().is_some();
let ty = make::ty_fn_ptr(
None,
is_unsafe,
func_node.abi(),
fn_params_vec.into_iter(),

View file

@ -331,6 +331,331 @@ pub enum SyntaxKind {
}
use self::SyntaxKind::*;
impl SyntaxKind {
#[allow(unreachable_patterns)]
pub const fn text(self) -> &'static str {
match self {
TOMBSTONE
| EOF
| __LAST
| BYTE
| BYTE_STRING
| CHAR
| C_STRING
| FLOAT_NUMBER
| INT_NUMBER
| RAW_BYTE_STRING
| RAW_C_STRING
| RAW_STRING
| STRING
| ABI
| ADT
| ARG_LIST
| ARRAY_EXPR
| ARRAY_TYPE
| ASM_CLOBBER_ABI
| ASM_CONST
| ASM_DIR_SPEC
| ASM_EXPR
| ASM_LABEL
| ASM_OPERAND
| ASM_OPERAND_EXPR
| ASM_OPERAND_NAMED
| ASM_OPTION
| ASM_OPTIONS
| ASM_PIECE
| ASM_REG_OPERAND
| ASM_REG_SPEC
| ASM_SYM
| ASSOC_ITEM
| ASSOC_ITEM_LIST
| ASSOC_TYPE_ARG
| ATTR
| AWAIT_EXPR
| BECOME_EXPR
| BIN_EXPR
| BLOCK_EXPR
| BOX_PAT
| BREAK_EXPR
| CALL_EXPR
| CAST_EXPR
| CLOSURE_BINDER
| CLOSURE_EXPR
| CONST
| CONST_ARG
| CONST_BLOCK_PAT
| CONST_PARAM
| CONTINUE_EXPR
| DYN_TRAIT_TYPE
| ENUM
| EXPR
| EXPR_STMT
| EXTERN_BLOCK
| EXTERN_CRATE
| EXTERN_ITEM
| EXTERN_ITEM_LIST
| FIELD_EXPR
| FIELD_LIST
| FN
| FN_PTR_TYPE
| FORMAT_ARGS_ARG
| FORMAT_ARGS_EXPR
| FOR_EXPR
| FOR_TYPE
| GENERIC_ARG
| GENERIC_ARG_LIST
| GENERIC_PARAM
| GENERIC_PARAM_LIST
| IDENT_PAT
| IF_EXPR
| IMPL
| IMPL_TRAIT_TYPE
| INDEX_EXPR
| INFER_TYPE
| ITEM
| ITEM_LIST
| LABEL
| LET_ELSE
| LET_EXPR
| LET_STMT
| LIFETIME
| LIFETIME_ARG
| LIFETIME_PARAM
| LITERAL
| LITERAL_PAT
| LOOP_EXPR
| MACRO_CALL
| MACRO_DEF
| MACRO_EXPR
| MACRO_ITEMS
| MACRO_PAT
| MACRO_RULES
| MACRO_STMTS
| MACRO_TYPE
| MATCH_ARM
| MATCH_ARM_LIST
| MATCH_EXPR
| MATCH_GUARD
| META
| METHOD_CALL_EXPR
| MODULE
| NAME
| NAME_REF
| NEVER_TYPE
| OFFSET_OF_EXPR
| OR_PAT
| PARAM
| PARAM_LIST
| PARENTHESIZED_ARG_LIST
| PAREN_EXPR
| PAREN_PAT
| PAREN_TYPE
| PAT
| PATH
| PATH_EXPR
| PATH_PAT
| PATH_SEGMENT
| PATH_TYPE
| PREFIX_EXPR
| PTR_TYPE
| RANGE_EXPR
| RANGE_PAT
| RECORD_EXPR
| RECORD_EXPR_FIELD
| RECORD_EXPR_FIELD_LIST
| RECORD_FIELD
| RECORD_FIELD_LIST
| RECORD_PAT
| RECORD_PAT_FIELD
| RECORD_PAT_FIELD_LIST
| REF_EXPR
| REF_PAT
| REF_TYPE
| RENAME
| REST_PAT
| RETURN_EXPR
| RETURN_TYPE_SYNTAX
| RET_TYPE
| SELF_PARAM
| SLICE_PAT
| SLICE_TYPE
| SOURCE_FILE
| STATIC
| STMT
| STMT_LIST
| STRUCT
| TOKEN_TREE
| TRAIT
| TRAIT_ALIAS
| TRY_EXPR
| TUPLE_EXPR
| TUPLE_FIELD
| TUPLE_FIELD_LIST
| TUPLE_PAT
| TUPLE_STRUCT_PAT
| TUPLE_TYPE
| TYPE
| TYPE_ALIAS
| TYPE_ARG
| TYPE_BOUND
| TYPE_BOUND_LIST
| TYPE_PARAM
| UNDERSCORE_EXPR
| UNION
| USE
| USE_BOUND_GENERIC_ARG
| USE_BOUND_GENERIC_ARGS
| USE_TREE
| USE_TREE_LIST
| VARIANT
| VARIANT_LIST
| VISIBILITY
| WHERE_CLAUSE
| WHERE_PRED
| WHILE_EXPR
| WILDCARD_PAT
| YEET_EXPR
| YIELD_EXPR
| COMMENT
| ERROR
| IDENT
| LIFETIME_IDENT
| NEWLINE
| SHEBANG
| WHITESPACE => panic!("no text for these `SyntaxKind`s"),
DOLLAR => "$",
SEMICOLON => ";",
COMMA => ",",
L_PAREN => "(",
R_PAREN => ")",
L_CURLY => "{",
R_CURLY => "}",
L_BRACK => "[",
R_BRACK => "]",
L_ANGLE => "<",
R_ANGLE => ">",
AT => "@",
POUND => "#",
TILDE => "~",
QUESTION => "?",
AMP => "&",
PIPE => "|",
PLUS => "+",
STAR => "*",
SLASH => "/",
CARET => "^",
PERCENT => "%",
UNDERSCORE => "_",
DOT => ".",
DOT2 => "..",
DOT3 => "...",
DOT2EQ => "..=",
COLON => ":",
COLON2 => "::",
EQ => "=",
EQ2 => "==",
FAT_ARROW => "=>",
BANG => "!",
NEQ => "!=",
MINUS => "-",
THIN_ARROW => "->",
LTEQ => "<=",
GTEQ => ">=",
PLUSEQ => "+=",
MINUSEQ => "-=",
PIPEEQ => "|=",
AMPEQ => "&=",
CARETEQ => "^=",
SLASHEQ => "/=",
STAREQ => "*=",
PERCENTEQ => "%=",
AMP2 => "&&",
PIPE2 => "||",
SHL => "<<",
SHR => ">>",
SHLEQ => "<<=",
SHREQ => ">>=",
SELF_TYPE_KW => "Self",
ABSTRACT_KW => "abstract",
AS_KW => "as",
BECOME_KW => "become",
BOX_KW => "box",
BREAK_KW => "break",
CONST_KW => "const",
CONTINUE_KW => "continue",
CRATE_KW => "crate",
DO_KW => "do",
ELSE_KW => "else",
ENUM_KW => "enum",
EXTERN_KW => "extern",
FALSE_KW => "false",
FINAL_KW => "final",
FN_KW => "fn",
FOR_KW => "for",
IF_KW => "if",
IMPL_KW => "impl",
IN_KW => "in",
LET_KW => "let",
LOOP_KW => "loop",
MACRO_KW => "macro",
MATCH_KW => "match",
MOD_KW => "mod",
MOVE_KW => "move",
MUT_KW => "mut",
OVERRIDE_KW => "override",
PRIV_KW => "priv",
PUB_KW => "pub",
REF_KW => "ref",
RETURN_KW => "return",
SELF_KW => "self",
STATIC_KW => "static",
STRUCT_KW => "struct",
SUPER_KW => "super",
TRAIT_KW => "trait",
TRUE_KW => "true",
TYPE_KW => "type",
TYPEOF_KW => "typeof",
UNSAFE_KW => "unsafe",
UNSIZED_KW => "unsized",
USE_KW => "use",
VIRTUAL_KW => "virtual",
WHERE_KW => "where",
WHILE_KW => "while",
YIELD_KW => "yield",
ASM_KW => "asm",
ATT_SYNTAX_KW => "att_syntax",
AUTO_KW => "auto",
BUILTIN_KW => "builtin",
CLOBBER_ABI_KW => "clobber_abi",
DEFAULT_KW => "default",
DYN_KW => "dyn",
FORMAT_ARGS_KW => "format_args",
INLATEOUT_KW => "inlateout",
INOUT_KW => "inout",
LABEL_KW => "label",
LATEOUT_KW => "lateout",
MACRO_RULES_KW => "macro_rules",
MAY_UNWIND_KW => "may_unwind",
NOMEM_KW => "nomem",
NORETURN_KW => "noreturn",
NOSTACK_KW => "nostack",
OFFSET_OF_KW => "offset_of",
OPTIONS_KW => "options",
OUT_KW => "out",
PRESERVES_FLAGS_KW => "preserves_flags",
PURE_KW => "pure",
RAW_KW => "raw",
READONLY_KW => "readonly",
SAFE_KW => "safe",
SYM_KW => "sym",
UNION_KW => "union",
YEET_KW => "yeet",
ASYNC_KW => "async",
AWAIT_KW => "await",
DYN_KW => "dyn",
GEN_KW => "gen",
TRY_KW => "try",
}
}
#[doc = r" Checks whether this syntax kind is a strict keyword for the given edition."]
#[doc = r" Strict keywords are identifiers that are always considered keywords."]
pub fn is_strict_keyword(self, edition: Edition) -> bool {

View file

@ -42,6 +42,14 @@ pub use self::{
/// the same representation: a pointer to the tree root and a pointer to the
/// node itself.
pub trait AstNode {
/// This panics if the `SyntaxKind` is not statically known.
fn kind() -> SyntaxKind
where
Self: Sized,
{
panic!("dynamic `SyntaxKind` for `AstNode::kind()`")
}
fn can_cast(kind: SyntaxKind) -> bool
where
Self: Sized;

File diff suppressed because it is too large Load diff

View file

@ -8,7 +8,10 @@
//! Keep in mind that `from_text` functions should be kept private. The public
//! API should require to assemble every node piecewise. The trick of
//! `parse(format!())` we use internally is an implementation detail -- long
//! term, it will be replaced with direct tree manipulation.
//! term, it will be replaced with `quote!`. Do not add more usages to `from_text` -
//! use `quote!` instead.
mod quote;
use itertools::Itertools;
use parser::{Edition, T};
@ -16,7 +19,7 @@ use rowan::NodeOrToken;
use stdx::{format_to, format_to_acc, never};
use crate::{
ast::{self, Param},
ast::{self, make::quote::quote, Param},
utils::is_raw_identifier,
AstNode, SourceFile, SyntaxKind, SyntaxToken,
};
@ -118,7 +121,11 @@ pub fn name(name: &str) -> ast::Name {
}
pub fn name_ref(name_ref: &str) -> ast::NameRef {
let raw_escape = raw_ident_esc(name_ref);
ast_from_text(&format!("fn f() {{ {raw_escape}{name_ref}; }}"))
quote! {
NameRef {
[IDENT format!("{raw_escape}{name_ref}")]
}
}
}
fn raw_ident_esc(ident: &str) -> &'static str {
if is_raw_identifier(ident, Edition::CURRENT) {
@ -135,7 +142,11 @@ pub fn lifetime(text: &str) -> ast::Lifetime {
tmp = format!("'{text}");
text = &tmp;
}
ast_from_text(&format!("fn f<{text}>() {{ }}"))
quote! {
Lifetime {
[LIFETIME_IDENT text]
}
}
}
// FIXME: replace stringly-typed constructor with a family of typed ctors, a-la
@ -175,63 +186,37 @@ pub fn ty_alias(
where_clause: Option<ast::WhereClause>,
assignment: Option<(ast::Type, Option<ast::WhereClause>)>,
) -> ast::TypeAlias {
let mut s = String::new();
s.push_str(&format!("type {ident}"));
if let Some(list) = generic_param_list {
s.push_str(&list.to_string());
}
if let Some(list) = type_param_bounds {
s.push_str(&format!(" : {list}"));
}
if let Some(cl) = where_clause {
s.push_str(&format!(" {cl}"));
}
if let Some(exp) = assignment {
if let Some(cl) = exp.1 {
s.push_str(&format!(" = {} {cl}", exp.0));
} else {
s.push_str(&format!(" = {}", exp.0));
let (assignment_ty, assignment_where) = assignment.unzip();
let assignment_where = assignment_where.flatten();
quote! {
TypeAlias {
[type] " "
Name { [IDENT ident] }
#generic_param_list
#(" " [:] " " #type_param_bounds)*
#(" " #where_clause)*
#(" " [=] " " #assignment_ty)*
#(" " #assignment_where)*
[;]
}
}
s.push(';');
ast_from_text(&s)
}
pub fn ty_fn_ptr<I: Iterator<Item = Param>>(
for_lifetime_list: Option<ast::GenericParamList>,
is_unsafe: bool,
abi: Option<ast::Abi>,
params: I,
mut params: I,
ret_type: Option<ast::RetType>,
) -> ast::FnPtrType {
let mut s = String::from("type __ = ");
if let Some(list) = for_lifetime_list {
format_to!(s, "for{} ", list);
let is_unsafe = is_unsafe.then_some(());
let first_param = params.next();
quote! {
FnPtrType {
#(#is_unsafe [unsafe] " ")* #(#abi " ")* [fn]
['('] #first_param #([,] " " #params)* [')']
#(" " #ret_type)*
}
}
if is_unsafe {
s.push_str("unsafe ");
}
if let Some(abi) = abi {
format_to!(s, "{} ", abi)
}
s.push_str("fn");
format_to!(s, "({})", params.map(|p| p.to_string()).join(", "));
if let Some(ret_type) = ret_type {
format_to!(s, " {}", ret_type);
}
ast_from_text(&s)
}
pub fn assoc_item_list() -> ast::AssocItemList {
@ -480,15 +465,16 @@ pub fn block_expr(
stmts: impl IntoIterator<Item = ast::Stmt>,
tail_expr: Option<ast::Expr>,
) -> ast::BlockExpr {
let mut buf = "{\n".to_owned();
for stmt in stmts.into_iter() {
format_to!(buf, " {stmt}\n");
quote! {
BlockExpr {
StmtList {
['{'] "\n"
#(" " #stmts "\n")*
#(" " #tail_expr "\n")*
['}']
}
}
}
if let Some(tail_expr) = tail_expr {
format_to!(buf, " {tail_expr}\n");
}
buf += "}";
ast_from_text(&format!("fn f() {buf}"))
}
pub fn async_move_block_expr(

View file

@ -0,0 +1,191 @@
//! A `quote!`-like API for crafting AST nodes.
pub(crate) use rowan::{GreenNode, GreenToken, NodeOrToken, SyntaxKind as RSyntaxKind};
macro_rules! quote_impl_ {
( @append $children:ident ) => {}; // Base case.
( @append $children:ident
$node:ident {
$($tree:tt)*
}
$($rest:tt)*
) => {
{
#[allow(unused_mut)]
let mut inner_children = ::std::vec::Vec::<$crate::ast::make::quote::NodeOrToken<
$crate::ast::make::quote::GreenNode,
$crate::ast::make::quote::GreenToken,
>>::new();
$crate::ast::make::quote::quote_impl!( @append inner_children
$($tree)*
);
let kind = <$crate::ast::$node as $crate::ast::AstNode>::kind();
let node = $crate::ast::make::quote::GreenNode::new($crate::ast::make::quote::RSyntaxKind(kind as u16), inner_children);
$children.push($crate::ast::make::quote::NodeOrToken::Node(node));
}
$crate::ast::make::quote::quote_impl!( @append $children $($rest)* );
};
( @append $children:ident
[ $token_kind:ident $token_text:expr ]
$($rest:tt)*
) => {
$children.push($crate::ast::make::quote::NodeOrToken::Token(
$crate::ast::make::quote::GreenToken::new(
$crate::ast::make::quote::RSyntaxKind($crate::SyntaxKind::$token_kind as u16),
&$token_text,
),
));
$crate::ast::make::quote::quote_impl!( @append $children $($rest)* );
};
( @append $children:ident
[$($token:tt)+]
$($rest:tt)*
) => {
$children.push($crate::ast::make::quote::NodeOrToken::Token(
$crate::ast::make::quote::GreenToken::new(
$crate::ast::make::quote::RSyntaxKind($crate::T![ $($token)+ ] as u16),
const { $crate::T![ $($token)+ ].text() },
),
));
$crate::ast::make::quote::quote_impl!( @append $children $($rest)* );
};
( @append $children:ident
$whitespace:literal
$($rest:tt)*
) => {
const { $crate::ast::make::quote::verify_only_whitespaces($whitespace) };
$children.push($crate::ast::make::quote::NodeOrToken::Token(
$crate::ast::make::quote::GreenToken::new(
$crate::ast::make::quote::RSyntaxKind($crate::SyntaxKind::WHITESPACE as u16),
$whitespace,
),
));
$crate::ast::make::quote::quote_impl!( @append $children $($rest)* );
};
( @append $children:ident
# $var:ident
$($rest:tt)*
) => {
$crate::ast::make::quote::ToNodeChild::append_node_child($var, &mut $children);
$crate::ast::make::quote::quote_impl!( @append $children $($rest)* );
};
( @append $children:ident
#( $($repetition:tt)+ )*
$($rest:tt)*
) => {
$crate::ast::make::quote::quote_impl!( @extract_pounded_in_repetition $children
[] [] $($repetition)*
);
$crate::ast::make::quote::quote_impl!( @append $children $($rest)* );
};
// Base case - no repetition var.
( @extract_pounded_in_repetition $children:ident
[ $($repetition:tt)* ] [ ]
) => {
::std::compile_error!("repetition in `ast::make::quote!()` without variable");
};
// Base case - repetition var found.
( @extract_pounded_in_repetition $children:ident
[ $($repetition:tt)* ] [ $repetition_var:ident ]
) => {
::std::iter::IntoIterator::into_iter($repetition_var).for_each(|$repetition_var| {
$crate::ast::make::quote::quote_impl!( @append $children $($repetition)* );
});
};
( @extract_pounded_in_repetition $children:ident
[ $($repetition:tt)* ] [ $repetition_var1:ident ] # $repetition_var2:ident $($rest:tt)*
) => {
::std::compile_error!("repetition in `ast::make::quote!()` with more than one variable");
};
( @extract_pounded_in_repetition $children:ident
[ $($repetition:tt)* ] [ ] # $repetition_var:ident $($rest:tt)*
) => {
$crate::ast::make::quote::quote_impl!( @extract_pounded_in_repetition $children
[ $($repetition)* # $repetition_var ] [ $repetition_var ] $($rest)*
);
};
( @extract_pounded_in_repetition $children:ident
[ $($repetition:tt)* ] [ $($repetition_var:tt)* ] $non_repetition_var:tt $($rest:tt)*
) => {
$crate::ast::make::quote::quote_impl!( @extract_pounded_in_repetition $children
[ $($repetition)* $non_repetition_var ] [ $($repetition_var)* ] $($rest)*
);
};
}
pub(crate) use quote_impl_ as quote_impl;
/// A `quote!`-like API for crafting AST nodes.
///
/// Syntax: AST nodes are created with `Node { children }`, where `Node` is the node name in `ast` (`ast::Node`).
/// Tokens are creates with their syntax enclosed by brackets, e.g. `[::]` or `['{']`. Alternatively, tokens can
/// be created with the syntax `[token_kind token_text]`, where `token_kind` is a variant of `SyntaxKind` (e.g.
/// `IDENT`) and `token_text` is an expression producing `String` or `&str`. Whitespaces can be added
/// as string literals (i.e. `"\n "` is a whitespace token). Interpolation is allowed with `#` (`#variable`),
/// from `AstNode`s and `Option`s of them. Repetition is also supported, with only one repeating variable
/// and no separator (`#("\n" #variable [>])*`), for any `IntoIterator`. Note that `Option`s are also `IntoIterator`,
/// which can help when you want to conditionally include something along with an optional node.
///
/// There needs to be one root node, and its type is returned.
///
/// Be careful to closely match the Ungrammar AST, there is no validation for this!
macro_rules! quote_ {
( $root:ident { $($tree:tt)* } ) => {{
#[allow(unused_mut)]
let mut root = ::std::vec::Vec::<$crate::ast::make::quote::NodeOrToken<
$crate::ast::make::quote::GreenNode,
$crate::ast::make::quote::GreenToken,
>>::with_capacity(1);
$crate::ast::make::quote::quote_impl!( @append root $root { $($tree)* } );
let root = root.into_iter().next().unwrap();
let root = $crate::SyntaxNode::new_root(root.into_node().unwrap());
<$crate::ast::$root as $crate::ast::AstNode>::cast(root).unwrap()
}};
}
pub(crate) use quote_ as quote;
use crate::AstNode;
pub(crate) trait ToNodeChild {
fn append_node_child(self, children: &mut Vec<NodeOrToken<GreenNode, GreenToken>>);
}
impl<N: AstNode> ToNodeChild for N {
fn append_node_child(self, children: &mut Vec<NodeOrToken<GreenNode, GreenToken>>) {
children.push((*self.syntax().clone_subtree().green()).to_owned().into());
}
}
impl<C: ToNodeChild> ToNodeChild for Option<C> {
fn append_node_child(self, children: &mut Vec<NodeOrToken<GreenNode, GreenToken>>) {
if let Some(child) = self {
child.append_node_child(children);
}
}
}
// This is useful when you want conditionally, based on some `bool`, to emit some code.
impl ToNodeChild for () {
fn append_node_child(self, _children: &mut Vec<NodeOrToken<GreenNode, GreenToken>>) {}
}
pub(crate) const fn verify_only_whitespaces(text: &str) {
let text = text.as_bytes();
let mut i = 0;
while i < text.len() {
if !text[i].is_ascii_whitespace() {
panic!("non-whitespace found in whitespace token");
}
i += 1;
}
}

View file

@ -162,6 +162,13 @@ fn generate_nodes(kinds: KindsSrc, grammar: &AstSrc) -> String {
},
quote! {
impl AstNode for #name {
#[inline]
fn kind() -> SyntaxKind
where
Self: Sized
{
#kind
}
#[inline]
fn can_cast(kind: SyntaxKind) -> bool {
kind == #kind
@ -397,6 +404,7 @@ fn generate_syntax_kinds(grammar: KindsSrc) -> String {
});
let punctuation =
grammar.punct.iter().map(|(_token, name)| format_ident!("{}", name)).collect::<Vec<_>>();
let punctuation_texts = grammar.punct.iter().map(|&(text, _name)| text);
let fmt_kw_as_variant = |&name| match name {
"Self" => format_ident!("SELF_TYPE_KW"),
@ -422,6 +430,7 @@ fn generate_syntax_kinds(grammar: KindsSrc) -> String {
quote! { #kw if #ed <= edition }
})
.collect::<Vec<_>>();
let edition_dependent_keywords = grammar.edition_dependent_keywords.iter().map(|&(it, _)| it);
let edition_dependent_keywords_variants = grammar
.edition_dependent_keywords
.iter()
@ -495,6 +504,20 @@ fn generate_syntax_kinds(grammar: KindsSrc) -> String {
use self::SyntaxKind::*;
impl SyntaxKind {
#[allow(unreachable_patterns)]
pub const fn text(self) -> &'static str {
match self {
TOMBSTONE | EOF | __LAST
#( | #literals )*
#( | #nodes )*
#( | #tokens )* => panic!("no text for these `SyntaxKind`s"),
#( #punctuation => #punctuation_texts ,)*
#( #strict_keywords_variants => #strict_keywords ,)*
#( #contextual_keywords_variants => #contextual_keywords ,)*
#( #edition_dependent_keywords_variants => #edition_dependent_keywords ,)*
}
}
/// Checks whether this syntax kind is a strict keyword for the given edition.
/// Strict keywords are identifiers that are always considered keywords.
pub fn is_strict_keyword(self, edition: Edition) -> bool {