internal: Move grammar codegen into xtask

crates/syntax/src/ast/generated/tokens.rs

@@ -7,58 +7,16 @@ use crate::{
 };
 
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
-pub struct Whitespace {
+pub struct Byte {
     pub(crate) syntax: SyntaxToken,
 }
-impl std::fmt::Display for Whitespace {
+impl std::fmt::Display for Byte {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         std::fmt::Display::fmt(&self.syntax, f)
     }
 }
-impl AstToken for Whitespace {
-    fn can_cast(kind: SyntaxKind) -> bool { kind == WHITESPACE }
-    fn cast(syntax: SyntaxToken) -> Option<Self> {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
-    }
-    fn syntax(&self) -> &SyntaxToken { &self.syntax }
-}
-
-#[derive(Debug, Clone, PartialEq, Eq, Hash)]
-pub struct Comment {
-    pub(crate) syntax: SyntaxToken,
-}
-impl std::fmt::Display for Comment {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        std::fmt::Display::fmt(&self.syntax, f)
-    }
-}
-impl AstToken for Comment {
-    fn can_cast(kind: SyntaxKind) -> bool { kind == COMMENT }
-    fn cast(syntax: SyntaxToken) -> Option<Self> {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
-    }
-    fn syntax(&self) -> &SyntaxToken { &self.syntax }
-}
-
-#[derive(Debug, Clone, PartialEq, Eq, Hash)]
-pub struct String {
-    pub(crate) syntax: SyntaxToken,
-}
-impl std::fmt::Display for String {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        std::fmt::Display::fmt(&self.syntax, f)
-    }
-}
-impl AstToken for String {
-    fn can_cast(kind: SyntaxKind) -> bool { kind == STRING }
+impl AstToken for Byte {
+    fn can_cast(kind: SyntaxKind) -> bool { kind == BYTE }
     fn cast(syntax: SyntaxToken) -> Option<Self> {
         if Self::can_cast(syntax.kind()) {
             Some(Self { syntax })
@@ -112,16 +70,37 @@ impl AstToken for CString {
 }
 
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
-pub struct IntNumber {
+pub struct Char {
     pub(crate) syntax: SyntaxToken,
 }
-impl std::fmt::Display for IntNumber {
+impl std::fmt::Display for Char {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         std::fmt::Display::fmt(&self.syntax, f)
     }
 }
-impl AstToken for IntNumber {
-    fn can_cast(kind: SyntaxKind) -> bool { kind == INT_NUMBER }
+impl AstToken for Char {
+    fn can_cast(kind: SyntaxKind) -> bool { kind == CHAR }
+    fn cast(syntax: SyntaxToken) -> Option<Self> {
+        if Self::can_cast(syntax.kind()) {
+            Some(Self { syntax })
+        } else {
+            None
+        }
+    }
+    fn syntax(&self) -> &SyntaxToken { &self.syntax }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct Comment {
+    pub(crate) syntax: SyntaxToken,
+}
+impl std::fmt::Display for Comment {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        std::fmt::Display::fmt(&self.syntax, f)
+    }
+}
+impl AstToken for Comment {
+    fn can_cast(kind: SyntaxKind) -> bool { kind == COMMENT }
     fn cast(syntax: SyntaxToken) -> Option<Self> {
         if Self::can_cast(syntax.kind()) {
             Some(Self { syntax })
@@ -153,48 +132,6 @@ impl AstToken for FloatNumber {
     fn syntax(&self) -> &SyntaxToken { &self.syntax }
 }
 
-#[derive(Debug, Clone, PartialEq, Eq, Hash)]
-pub struct Char {
-    pub(crate) syntax: SyntaxToken,
-}
-impl std::fmt::Display for Char {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        std::fmt::Display::fmt(&self.syntax, f)
-    }
-}
-impl AstToken for Char {
-    fn can_cast(kind: SyntaxKind) -> bool { kind == CHAR }
-    fn cast(syntax: SyntaxToken) -> Option<Self> {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
-    }
-    fn syntax(&self) -> &SyntaxToken { &self.syntax }
-}
-
-#[derive(Debug, Clone, PartialEq, Eq, Hash)]
-pub struct Byte {
-    pub(crate) syntax: SyntaxToken,
-}
-impl std::fmt::Display for Byte {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        std::fmt::Display::fmt(&self.syntax, f)
-    }
-}
-impl AstToken for Byte {
-    fn can_cast(kind: SyntaxKind) -> bool { kind == BYTE }
-    fn cast(syntax: SyntaxToken) -> Option<Self> {
-        if Self::can_cast(syntax.kind()) {
-            Some(Self { syntax })
-        } else {
-            None
-        }
-    }
-    fn syntax(&self) -> &SyntaxToken { &self.syntax }
-}
-
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
 pub struct Ident {
     pub(crate) syntax: SyntaxToken,
@@ -215,3 +152,66 @@ impl AstToken for Ident {
     }
     fn syntax(&self) -> &SyntaxToken { &self.syntax }
 }
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct IntNumber {
+    pub(crate) syntax: SyntaxToken,
+}
+impl std::fmt::Display for IntNumber {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        std::fmt::Display::fmt(&self.syntax, f)
+    }
+}
+impl AstToken for IntNumber {
+    fn can_cast(kind: SyntaxKind) -> bool { kind == INT_NUMBER }
+    fn cast(syntax: SyntaxToken) -> Option<Self> {
+        if Self::can_cast(syntax.kind()) {
+            Some(Self { syntax })
+        } else {
+            None
+        }
+    }
+    fn syntax(&self) -> &SyntaxToken { &self.syntax }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct String {
+    pub(crate) syntax: SyntaxToken,
+}
+impl std::fmt::Display for String {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        std::fmt::Display::fmt(&self.syntax, f)
+    }
+}
+impl AstToken for String {
+    fn can_cast(kind: SyntaxKind) -> bool { kind == STRING }
+    fn cast(syntax: SyntaxToken) -> Option<Self> {
+        if Self::can_cast(syntax.kind()) {
+            Some(Self { syntax })
+        } else {
+            None
+        }
+    }
+    fn syntax(&self) -> &SyntaxToken { &self.syntax }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct Whitespace {
+    pub(crate) syntax: SyntaxToken,
+}
+impl std::fmt::Display for Whitespace {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        std::fmt::Display::fmt(&self.syntax, f)
+    }
+}
+impl AstToken for Whitespace {
+    fn can_cast(kind: SyntaxKind) -> bool { kind == WHITESPACE }
+    fn cast(syntax: SyntaxToken) -> Option<Self> {
+        if Self::can_cast(syntax.kind()) {
+            Some(Self { syntax })
+        } else {
+            None
+        }
+    }
+    fn syntax(&self) -> &SyntaxToken { &self.syntax }
+}

crates/syntax/src/tests.rs

@@ -1,8 +1,3 @@
-#[cfg(not(feature = "in-rust-tree"))]
-mod ast_src;
-#[cfg(not(feature = "in-rust-tree"))]
-mod sourcegen_ast;
-
 use std::{
     fs,
     path::{Path, PathBuf},
@@ -82,7 +77,25 @@ fn reparse_fuzz_tests() {
 fn self_hosting_parsing() {
     let crates_dir = project_root().join("crates");
 
-    let mut files = ::sourcegen::list_rust_files(&crates_dir);
+    let mut files = Vec::new();
+    let mut work = vec![crates_dir.to_path_buf()];
+    while let Some(dir) = work.pop() {
+        for entry in dir.read_dir().unwrap() {
+            let entry = entry.unwrap();
+            let file_type = entry.file_type().unwrap();
+            let path = entry.path();
+            let file_name = &path.file_name().unwrap_or_default().to_str().unwrap_or_default();
+            let is_hidden = file_name.starts_with('.');
+            if !is_hidden {
+                if file_type.is_dir() {
+                    work.push(path);
+                } else if file_type.is_file() && file_name.ends_with(".rs") {
+                    files.push(path);
+                }
+            }
+        }
+    }
+
     files.retain(|path| {
         // Get all files which are not in the crates/syntax/test_data folder
         !path.components().any(|component| component.as_os_str() == "test_data")

crates/syntax/src/tests/ast_src.rs

@@ -1,273 +0,0 @@
-//! Defines input for code generation process.
-
-pub(crate) struct KindsSrc<'a> {
-    pub(crate) punct: &'a [(&'a str, &'a str)],
-    pub(crate) keywords: &'a [&'a str],
-    pub(crate) contextual_keywords: &'a [&'a str],
-    pub(crate) literals: &'a [&'a str],
-    pub(crate) tokens: &'a [&'a str],
-    pub(crate) nodes: &'a [&'a str],
-}
-
-pub(crate) const KINDS_SRC: KindsSrc<'_> = KindsSrc {
-    punct: &[
-        (";", "SEMICOLON"),
-        (",", "COMMA"),
-        ("(", "L_PAREN"),
-        (")", "R_PAREN"),
-        ("{", "L_CURLY"),
-        ("}", "R_CURLY"),
-        ("[", "L_BRACK"),
-        ("]", "R_BRACK"),
-        ("<", "L_ANGLE"),
-        (">", "R_ANGLE"),
-        ("@", "AT"),
-        ("#", "POUND"),
-        ("~", "TILDE"),
-        ("?", "QUESTION"),
-        ("$", "DOLLAR"),
-        ("&", "AMP"),
-        ("|", "PIPE"),
-        ("+", "PLUS"),
-        ("*", "STAR"),
-        ("/", "SLASH"),
-        ("^", "CARET"),
-        ("%", "PERCENT"),
-        ("_", "UNDERSCORE"),
-        (".", "DOT"),
-        ("..", "DOT2"),
-        ("...", "DOT3"),
-        ("..=", "DOT2EQ"),
-        (":", "COLON"),
-        ("::", "COLON2"),
-        ("=", "EQ"),
-        ("==", "EQ2"),
-        ("=>", "FAT_ARROW"),
-        ("!", "BANG"),
-        ("!=", "NEQ"),
-        ("-", "MINUS"),
-        ("->", "THIN_ARROW"),
-        ("<=", "LTEQ"),
-        (">=", "GTEQ"),
-        ("+=", "PLUSEQ"),
-        ("-=", "MINUSEQ"),
-        ("|=", "PIPEEQ"),
-        ("&=", "AMPEQ"),
-        ("^=", "CARETEQ"),
-        ("/=", "SLASHEQ"),
-        ("*=", "STAREQ"),
-        ("%=", "PERCENTEQ"),
-        ("&&", "AMP2"),
-        ("||", "PIPE2"),
-        ("<<", "SHL"),
-        (">>", "SHR"),
-        ("<<=", "SHLEQ"),
-        (">>=", "SHREQ"),
-    ],
-    keywords: &[
-        "as", "async", "await", "box", "break", "const", "continue", "crate", "do", "dyn", "else",
-        "enum", "extern", "false", "fn", "for", "if", "impl", "in", "let", "loop", "macro",
-        "match", "mod", "move", "mut", "pub", "ref", "return", "become", "self", "Self", "static",
-        "struct", "super", "trait", "true", "try", "type", "unsafe", "use", "where", "while",
-        "yield",
-    ],
-    contextual_keywords: &[
-        "auto",
-        "builtin",
-        "default",
-        "existential",
-        "union",
-        "raw",
-        "macro_rules",
-        "yeet",
-        "offset_of",
-        "asm",
-        "format_args",
-    ],
-    literals: &["INT_NUMBER", "FLOAT_NUMBER", "CHAR", "BYTE", "STRING", "BYTE_STRING", "C_STRING"],
-    tokens: &["ERROR", "IDENT", "WHITESPACE", "LIFETIME_IDENT", "COMMENT", "SHEBANG"],
-    nodes: &[
-        "SOURCE_FILE",
-        "STRUCT",
-        "UNION",
-        "ENUM",
-        "FN",
-        "RET_TYPE",
-        "EXTERN_CRATE",
-        "MODULE",
-        "USE",
-        "STATIC",
-        "CONST",
-        "TRAIT",
-        "TRAIT_ALIAS",
-        "IMPL",
-        "TYPE_ALIAS",
-        "MACRO_CALL",
-        "MACRO_RULES",
-        "MACRO_ARM",
-        "TOKEN_TREE",
-        "MACRO_DEF",
-        "PAREN_TYPE",
-        "TUPLE_TYPE",
-        "MACRO_TYPE",
-        "NEVER_TYPE",
-        "PATH_TYPE",
-        "PTR_TYPE",
-        "ARRAY_TYPE",
-        "SLICE_TYPE",
-        "REF_TYPE",
-        "INFER_TYPE",
-        "FN_PTR_TYPE",
-        "FOR_TYPE",
-        "IMPL_TRAIT_TYPE",
-        "DYN_TRAIT_TYPE",
-        "OR_PAT",
-        "PAREN_PAT",
-        "REF_PAT",
-        "BOX_PAT",
-        "IDENT_PAT",
-        "WILDCARD_PAT",
-        "REST_PAT",
-        "PATH_PAT",
-        "RECORD_PAT",
-        "RECORD_PAT_FIELD_LIST",
-        "RECORD_PAT_FIELD",
-        "TUPLE_STRUCT_PAT",
-        "TUPLE_PAT",
-        "SLICE_PAT",
-        "RANGE_PAT",
-        "LITERAL_PAT",
-        "MACRO_PAT",
-        "CONST_BLOCK_PAT",
-        // atoms
-        "TUPLE_EXPR",
-        "ARRAY_EXPR",
-        "PAREN_EXPR",
-        "PATH_EXPR",
-        "CLOSURE_EXPR",
-        "IF_EXPR",
-        "WHILE_EXPR",
-        "LOOP_EXPR",
-        "FOR_EXPR",
-        "CONTINUE_EXPR",
-        "BREAK_EXPR",
-        "LABEL",
-        "BLOCK_EXPR",
-        "STMT_LIST",
-        "RETURN_EXPR",
-        "BECOME_EXPR",
-        "YIELD_EXPR",
-        "YEET_EXPR",
-        "LET_EXPR",
-        "UNDERSCORE_EXPR",
-        "MACRO_EXPR",
-        "MATCH_EXPR",
-        "MATCH_ARM_LIST",
-        "MATCH_ARM",
-        "MATCH_GUARD",
-        "RECORD_EXPR",
-        "RECORD_EXPR_FIELD_LIST",
-        "RECORD_EXPR_FIELD",
-        "OFFSET_OF_EXPR",
-        "ASM_EXPR",
-        "FORMAT_ARGS_EXPR",
-        "FORMAT_ARGS_ARG",
-        // postfix
-        "CALL_EXPR",
-        "INDEX_EXPR",
-        "METHOD_CALL_EXPR",
-        "FIELD_EXPR",
-        "AWAIT_EXPR",
-        "TRY_EXPR",
-        "CAST_EXPR",
-        // unary
-        "REF_EXPR",
-        "PREFIX_EXPR",
-        "RANGE_EXPR", // just weird
-        "BIN_EXPR",
-        "EXTERN_BLOCK",
-        "EXTERN_ITEM_LIST",
-        "VARIANT",
-        "RECORD_FIELD_LIST",
-        "RECORD_FIELD",
-        "TUPLE_FIELD_LIST",
-        "TUPLE_FIELD",
-        "VARIANT_LIST",
-        "ITEM_LIST",
-        "ASSOC_ITEM_LIST",
-        "ATTR",
-        "META",
-        "USE_TREE",
-        "USE_TREE_LIST",
-        "PATH",
-        "PATH_SEGMENT",
-        "LITERAL",
-        "RENAME",
-        "VISIBILITY",
-        "WHERE_CLAUSE",
-        "WHERE_PRED",
-        "ABI",
-        "NAME",
-        "NAME_REF",
-        "LET_STMT",
-        "LET_ELSE",
-        "EXPR_STMT",
-        "GENERIC_PARAM_LIST",
-        "GENERIC_PARAM",
-        "LIFETIME_PARAM",
-        "TYPE_PARAM",
-        "RETURN_TYPE_ARG",
-        "CONST_PARAM",
-        "GENERIC_ARG_LIST",
-        "LIFETIME",
-        "LIFETIME_ARG",
-        "TYPE_ARG",
-        "ASSOC_TYPE_ARG",
-        "CONST_ARG",
-        "PARAM_LIST",
-        "PARAM",
-        "SELF_PARAM",
-        "ARG_LIST",
-        "TYPE_BOUND",
-        "TYPE_BOUND_LIST",
-        // macro related
-        "MACRO_ITEMS",
-        "MACRO_STMTS",
-        "MACRO_EAGER_INPUT",
-    ],
-};
-
-#[derive(Default, Debug)]
-pub(crate) struct AstSrc {
-    pub(crate) tokens: Vec<String>,
-    pub(crate) nodes: Vec<AstNodeSrc>,
-    pub(crate) enums: Vec<AstEnumSrc>,
-}
-
-#[derive(Debug)]
-pub(crate) struct AstNodeSrc {
-    pub(crate) doc: Vec<String>,
-    pub(crate) name: String,
-    pub(crate) traits: Vec<String>,
-    pub(crate) fields: Vec<Field>,
-}
-
-#[derive(Debug, Eq, PartialEq)]
-pub(crate) enum Field {
-    Token(String),
-    Node { name: String, ty: String, cardinality: Cardinality },
-}
-
-#[derive(Debug, Eq, PartialEq)]
-pub(crate) enum Cardinality {
-    Optional,
-    Many,
-}
-
-#[derive(Debug)]
-pub(crate) struct AstEnumSrc {
-    pub(crate) doc: Vec<String>,
-    pub(crate) name: String,
-    pub(crate) traits: Vec<String>,
-    pub(crate) variants: Vec<String>,
-}

crates/syntax/src/tests/sourcegen_ast.rs

@@ -1,859 +0,0 @@
-//! This module generates AST datatype used by rust-analyzer.
-//!
-//! Specifically, it generates the `SyntaxKind` enum and a number of newtype
-//! wrappers around `SyntaxNode` which implement `syntax::AstNode`.
-
-use std::{collections::BTreeSet, fmt::Write};
-
-use itertools::Itertools;
-use proc_macro2::{Punct, Spacing};
-use quote::{format_ident, quote};
-use rustc_hash::FxHashSet;
-use ungrammar::{Grammar, Rule};
-
-use crate::tests::ast_src::{
-    AstEnumSrc, AstNodeSrc, AstSrc, Cardinality, Field, KindsSrc, KINDS_SRC,
-};
-
-#[test]
-fn sourcegen_ast() {
-    let syntax_kinds = generate_syntax_kinds(KINDS_SRC);
-    let syntax_kinds_file =
-        sourcegen::project_root().join("crates/parser/src/syntax_kind/generated.rs");
-    sourcegen::ensure_file_contents(syntax_kinds_file.as_path(), &syntax_kinds);
-
-    let grammar =
-        include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/rust.ungram")).parse().unwrap();
-    let ast = lower(&grammar);
-
-    let ast_tokens = generate_tokens(&ast);
-    let ast_tokens_file =
-        sourcegen::project_root().join("crates/syntax/src/ast/generated/tokens.rs");
-    sourcegen::ensure_file_contents(ast_tokens_file.as_path(), &ast_tokens);
-
-    let ast_nodes = generate_nodes(KINDS_SRC, &ast);
-    let ast_nodes_file = sourcegen::project_root().join("crates/syntax/src/ast/generated/nodes.rs");
-    sourcegen::ensure_file_contents(ast_nodes_file.as_path(), &ast_nodes);
-}
-
-fn generate_tokens(grammar: &AstSrc) -> String {
-    let tokens = grammar.tokens.iter().map(|token| {
-        let name = format_ident!("{}", token);
-        let kind = format_ident!("{}", to_upper_snake_case(token));
-        quote! {
-            #[derive(Debug, Clone, PartialEq, Eq, Hash)]
-            pub struct #name {
-                pub(crate) syntax: SyntaxToken,
-            }
-            impl std::fmt::Display for #name {
-                fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-                    std::fmt::Display::fmt(&self.syntax, f)
-                }
-            }
-            impl AstToken for #name {
-                fn can_cast(kind: SyntaxKind) -> bool { kind == #kind }
-                fn cast(syntax: SyntaxToken) -> Option<Self> {
-                    if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
-                }
-                fn syntax(&self) -> &SyntaxToken { &self.syntax }
-            }
-        }
-    });
-
-    sourcegen::add_preamble(
-        "sourcegen_ast",
-        sourcegen::reformat(
-            quote! {
-                use crate::{SyntaxKind::{self, *}, SyntaxToken, ast::AstToken};
-                #(#tokens)*
-            }
-            .to_string(),
-        ),
-    )
-    .replace("#[derive", "\n#[derive")
-}
-
-fn generate_nodes(kinds: KindsSrc<'_>, grammar: &AstSrc) -> String {
-    let (node_defs, node_boilerplate_impls): (Vec<_>, Vec<_>) = grammar
-        .nodes
-        .iter()
-        .sorted_by_key(|it| it.name.clone())
-        .map(|node| {
-            let name = format_ident!("{}", node.name);
-            let kind = format_ident!("{}", to_upper_snake_case(&node.name));
-            let traits = node
-                .traits
-                .iter()
-                .filter(|trait_name| {
-                    // Loops have two expressions so this might collide, therefore manual impl it
-                    node.name != "ForExpr" && node.name != "WhileExpr"
-                        || trait_name.as_str() != "HasLoopBody"
-                })
-                .sorted()
-                .map(|trait_name| {
-                    let trait_name = format_ident!("{}", trait_name);
-                    quote!(impl ast::#trait_name for #name {})
-                });
-
-            let methods = node.fields.iter().sorted_by_key(|it| it.method_name()).map(|field| {
-                let method_name = field.method_name();
-                let ty = field.ty();
-
-                if field.is_many() {
-                    quote! {
-                        pub fn #method_name(&self) -> AstChildren<#ty> {
-                            support::children(&self.syntax)
-                        }
-                    }
-                } else if let Some(token_kind) = field.token_kind() {
-                    quote! {
-                        pub fn #method_name(&self) -> Option<#ty> {
-                            support::token(&self.syntax, #token_kind)
-                        }
-                    }
-                } else {
-                    quote! {
-                        pub fn #method_name(&self) -> Option<#ty> {
-                            support::child(&self.syntax)
-                        }
-                    }
-                }
-            });
-            (
-                quote! {
-                    #[pretty_doc_comment_placeholder_workaround]
-                    #[derive(Debug, Clone, PartialEq, Eq, Hash)]
-                    pub struct #name {
-                        pub(crate) syntax: SyntaxNode,
-                    }
-
-                    #(#traits)*
-
-                    impl #name {
-                        #(#methods)*
-                    }
-                },
-                quote! {
-                    impl AstNode for #name {
-                        fn can_cast(kind: SyntaxKind) -> bool {
-                            kind == #kind
-                        }
-                        fn cast(syntax: SyntaxNode) -> Option<Self> {
-                            if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
-                        }
-                        fn syntax(&self) -> &SyntaxNode { &self.syntax }
-                    }
-                },
-            )
-        })
-        .unzip();
-
-    let (enum_defs, enum_boilerplate_impls): (Vec<_>, Vec<_>) = grammar
-        .enums
-        .iter()
-        .sorted_by_key(|it| it.name.clone())
-        .map(|en| {
-            let variants: Vec<_> =
-                en.variants.iter().map(|var| format_ident!("{}", var)).sorted().collect();
-            let name = format_ident!("{}", en.name);
-            let kinds: Vec<_> = variants
-                .iter()
-                .map(|name| format_ident!("{}", to_upper_snake_case(&name.to_string())))
-                .collect();
-            let traits = en.traits.iter().sorted().map(|trait_name| {
-                let trait_name = format_ident!("{}", trait_name);
-                quote!(impl ast::#trait_name for #name {})
-            });
-
-            let ast_node = if en.name == "Stmt" {
-                quote! {}
-            } else {
-                quote! {
-                    impl AstNode for #name {
-                        fn can_cast(kind: SyntaxKind) -> bool {
-                            matches!(kind, #(#kinds)|*)
-                        }
-                        fn cast(syntax: SyntaxNode) -> Option<Self> {
-                            let res = match syntax.kind() {
-                                #(
-                                #kinds => #name::#variants(#variants { syntax }),
-                                )*
-                                _ => return None,
-                            };
-                            Some(res)
-                        }
-                        fn syntax(&self) -> &SyntaxNode {
-                            match self {
-                                #(
-                                #name::#variants(it) => &it.syntax,
-                                )*
-                            }
-                        }
-                    }
-                }
-            };
-
-            (
-                quote! {
-                    #[pretty_doc_comment_placeholder_workaround]
-                    #[derive(Debug, Clone, PartialEq, Eq, Hash)]
-                    pub enum #name {
-                        #(#variants(#variants),)*
-                    }
-
-                    #(#traits)*
-                },
-                quote! {
-                    #(
-                        impl From<#variants> for #name {
-                            fn from(node: #variants) -> #name {
-                                #name::#variants(node)
-                            }
-                        }
-                    )*
-                    #ast_node
-                },
-            )
-        })
-        .unzip();
-
-    let (any_node_defs, any_node_boilerplate_impls): (Vec<_>, Vec<_>) = grammar
-        .nodes
-        .iter()
-        .flat_map(|node| node.traits.iter().map(move |t| (t, node)))
-        .into_group_map()
-        .into_iter()
-        .sorted_by_key(|(k, _)| *k)
-        .map(|(trait_name, nodes)| {
-            let name = format_ident!("Any{}", trait_name);
-            let trait_name = format_ident!("{}", trait_name);
-            let kinds: Vec<_> = nodes
-                .iter()
-                .map(|name| format_ident!("{}", to_upper_snake_case(&name.name.to_string())))
-                .collect();
-
-            (
-                quote! {
-                    #[pretty_doc_comment_placeholder_workaround]
-                    #[derive(Debug, Clone, PartialEq, Eq, Hash)]
-                    pub struct #name {
-                        pub(crate) syntax: SyntaxNode,
-                    }
-                    impl ast::#trait_name for #name {}
-                },
-                quote! {
-                    impl #name {
-                        #[inline]
-                        pub fn new<T: ast::#trait_name>(node: T) -> #name {
-                            #name {
-                                syntax: node.syntax().clone()
-                            }
-                        }
-                    }
-                    impl AstNode for #name {
-                        fn can_cast(kind: SyntaxKind) -> bool {
-                            matches!(kind, #(#kinds)|*)
-                        }
-                        fn cast(syntax: SyntaxNode) -> Option<Self> {
-                            Self::can_cast(syntax.kind()).then_some(#name { syntax })
-                        }
-                        fn syntax(&self) -> &SyntaxNode {
-                            &self.syntax
-                        }
-                    }
-                },
-            )
-        })
-        .unzip();
-
-    let enum_names = grammar.enums.iter().map(|it| &it.name);
-    let node_names = grammar.nodes.iter().map(|it| &it.name);
-
-    let display_impls =
-        enum_names.chain(node_names.clone()).map(|it| format_ident!("{}", it)).sorted().map(
-            |name| {
-                quote! {
-                    impl std::fmt::Display for #name {
-                        fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-                            std::fmt::Display::fmt(self.syntax(), f)
-                        }
-                    }
-                }
-            },
-        );
-
-    let defined_nodes: FxHashSet<_> = node_names.collect();
-
-    for node in kinds
-        .nodes
-        .iter()
-        .map(|kind| to_pascal_case(kind))
-        .filter(|name| !defined_nodes.iter().any(|&it| it == name))
-    {
-        drop(node)
-        // FIXME: restore this
-        // eprintln!("Warning: node {} not defined in ast source", node);
-    }
-
-    let ast = quote! {
-        #![allow(non_snake_case)]
-        use crate::{
-            SyntaxNode, SyntaxToken, SyntaxKind::{self, *},
-            ast::{self, AstNode, AstChildren, support},
-            T,
-        };
-
-        #(#node_defs)*
-        #(#enum_defs)*
-        #(#any_node_defs)*
-        #(#node_boilerplate_impls)*
-        #(#enum_boilerplate_impls)*
-        #(#any_node_boilerplate_impls)*
-        #(#display_impls)*
-    };
-
-    let ast = ast.to_string().replace("T ! [", "T![");
-
-    let mut res = String::with_capacity(ast.len() * 2);
-
-    let mut docs =
-        grammar.nodes.iter().map(|it| &it.doc).chain(grammar.enums.iter().map(|it| &it.doc));
-
-    for chunk in ast.split("# [pretty_doc_comment_placeholder_workaround] ") {
-        res.push_str(chunk);
-        if let Some(doc) = docs.next() {
-            write_doc_comment(doc, &mut res);
-        }
-    }
-
-    let res = sourcegen::add_preamble("sourcegen_ast", sourcegen::reformat(res));
-    res.replace("#[derive", "\n#[derive")
-}
-
-fn write_doc_comment(contents: &[String], dest: &mut String) {
-    for line in contents {
-        writeln!(dest, "///{line}").unwrap();
-    }
-}
-
-fn generate_syntax_kinds(grammar: KindsSrc<'_>) -> String {
-    let (single_byte_tokens_values, single_byte_tokens): (Vec<_>, Vec<_>) = grammar
-        .punct
-        .iter()
-        .filter(|(token, _name)| token.len() == 1)
-        .map(|(token, name)| (token.chars().next().unwrap(), format_ident!("{}", name)))
-        .unzip();
-
-    let punctuation_values = grammar.punct.iter().map(|(token, _name)| {
-        if "{}[]()".contains(token) {
-            let c = token.chars().next().unwrap();
-            quote! { #c }
-        } else {
-            let cs = token.chars().map(|c| Punct::new(c, Spacing::Joint));
-            quote! { #(#cs)* }
-        }
-    });
-    let punctuation =
-        grammar.punct.iter().map(|(_token, name)| format_ident!("{}", name)).collect::<Vec<_>>();
-
-    let x = |&name| match name {
-        "Self" => format_ident!("SELF_TYPE_KW"),
-        name => format_ident!("{}_KW", to_upper_snake_case(name)),
-    };
-    let full_keywords_values = grammar.keywords;
-    let full_keywords = full_keywords_values.iter().map(x);
-
-    let contextual_keywords_values = &grammar.contextual_keywords;
-    let contextual_keywords = contextual_keywords_values.iter().map(x);
-
-    let all_keywords_values = grammar
-        .keywords
-        .iter()
-        .chain(grammar.contextual_keywords.iter())
-        .copied()
-        .collect::<Vec<_>>();
-    let all_keywords_idents = all_keywords_values.iter().map(|kw| format_ident!("{}", kw));
-    let all_keywords = all_keywords_values.iter().map(x).collect::<Vec<_>>();
-
-    let literals =
-        grammar.literals.iter().map(|name| format_ident!("{}", name)).collect::<Vec<_>>();
-
-    let tokens = grammar.tokens.iter().map(|name| format_ident!("{}", name)).collect::<Vec<_>>();
-
-    let nodes = grammar.nodes.iter().map(|name| format_ident!("{}", name)).collect::<Vec<_>>();
-
-    let ast = quote! {
-        #![allow(bad_style, missing_docs, unreachable_pub)]
-        /// The kind of syntax node, e.g. `IDENT`, `USE_KW`, or `STRUCT`.
-        #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)]
-        #[repr(u16)]
-        pub enum SyntaxKind {
-            // Technical SyntaxKinds: they appear temporally during parsing,
-            // but never end up in the final tree
-            #[doc(hidden)]
-            TOMBSTONE,
-            #[doc(hidden)]
-            EOF,
-            #(#punctuation,)*
-            #(#all_keywords,)*
-            #(#literals,)*
-            #(#tokens,)*
-            #(#nodes,)*
-
-            // Technical kind so that we can cast from u16 safely
-            #[doc(hidden)]
-            __LAST,
-        }
-        use self::SyntaxKind::*;
-
-        impl SyntaxKind {
-            pub fn is_keyword(self) -> bool {
-                matches!(self, #(#all_keywords)|*)
-            }
-
-            pub fn is_punct(self) -> bool {
-
-                matches!(self, #(#punctuation)|*)
-
-            }
-
-            pub fn is_literal(self) -> bool {
-                matches!(self, #(#literals)|*)
-            }
-
-            pub fn from_keyword(ident: &str) -> Option<SyntaxKind> {
-                let kw = match ident {
-                    #(#full_keywords_values => #full_keywords,)*
-                    _ => return None,
-                };
-                Some(kw)
-            }
-
-            pub fn from_contextual_keyword(ident: &str) -> Option<SyntaxKind> {
-                let kw = match ident {
-                    #(#contextual_keywords_values => #contextual_keywords,)*
-                    _ => return None,
-                };
-                Some(kw)
-            }
-
-            pub fn from_char(c: char) -> Option<SyntaxKind> {
-                let tok = match c {
-                    #(#single_byte_tokens_values => #single_byte_tokens,)*
-                    _ => return None,
-                };
-                Some(tok)
-            }
-        }
-
-        #[macro_export]
-        macro_rules! T {
-            #([#punctuation_values] => { $crate::SyntaxKind::#punctuation };)*
-            #([#all_keywords_idents] => { $crate::SyntaxKind::#all_keywords };)*
-            [lifetime_ident] => { $crate::SyntaxKind::LIFETIME_IDENT };
-            [ident] => { $crate::SyntaxKind::IDENT };
-            [shebang] => { $crate::SyntaxKind::SHEBANG };
-        }
-    };
-
-    sourcegen::add_preamble("sourcegen_ast", sourcegen::reformat(ast.to_string()))
-}
-
-fn to_upper_snake_case(s: &str) -> String {
-    let mut buf = String::with_capacity(s.len());
-    let mut prev = false;
-    for c in s.chars() {
-        if c.is_ascii_uppercase() && prev {
-            buf.push('_')
-        }
-        prev = true;
-
-        buf.push(c.to_ascii_uppercase());
-    }
-    buf
-}
-
-fn to_lower_snake_case(s: &str) -> String {
-    let mut buf = String::with_capacity(s.len());
-    let mut prev = false;
-    for c in s.chars() {
-        if c.is_ascii_uppercase() && prev {
-            buf.push('_')
-        }
-        prev = true;
-
-        buf.push(c.to_ascii_lowercase());
-    }
-    buf
-}
-
-fn to_pascal_case(s: &str) -> String {
-    let mut buf = String::with_capacity(s.len());
-    let mut prev_is_underscore = true;
-    for c in s.chars() {
-        if c == '_' {
-            prev_is_underscore = true;
-        } else if prev_is_underscore {
-            buf.push(c.to_ascii_uppercase());
-            prev_is_underscore = false;
-        } else {
-            buf.push(c.to_ascii_lowercase());
-        }
-    }
-    buf
-}
-
-fn pluralize(s: &str) -> String {
-    format!("{s}s")
-}
-
-impl Field {
-    fn is_many(&self) -> bool {
-        matches!(self, Field::Node { cardinality: Cardinality::Many, .. })
-    }
-    fn token_kind(&self) -> Option<proc_macro2::TokenStream> {
-        match self {
-            Field::Token(token) => {
-                let token: proc_macro2::TokenStream = token.parse().unwrap();
-                Some(quote! { T![#token] })
-            }
-            _ => None,
-        }
-    }
-    fn method_name(&self) -> proc_macro2::Ident {
-        match self {
-            Field::Token(name) => {
-                let name = match name.as_str() {
-                    ";" => "semicolon",
-                    "->" => "thin_arrow",
-                    "'{'" => "l_curly",
-                    "'}'" => "r_curly",
-                    "'('" => "l_paren",
-                    "')'" => "r_paren",
-                    "'['" => "l_brack",
-                    "']'" => "r_brack",
-                    "<" => "l_angle",
-                    ">" => "r_angle",
-                    "=" => "eq",
-                    "!" => "excl",
-                    "*" => "star",
-                    "&" => "amp",
-                    "-" => "minus",
-                    "_" => "underscore",
-                    "." => "dot",
-                    ".." => "dotdot",
-                    "..." => "dotdotdot",
-                    "..=" => "dotdoteq",
-                    "=>" => "fat_arrow",
-                    "@" => "at",
-                    ":" => "colon",
-                    "::" => "coloncolon",
-                    "#" => "pound",
-                    "?" => "question_mark",
-                    "," => "comma",
-                    "|" => "pipe",
-                    "~" => "tilde",
-                    _ => name,
-                };
-                format_ident!("{}_token", name)
-            }
-            Field::Node { name, .. } => {
-                if name == "type" {
-                    format_ident!("ty")
-                } else {
-                    format_ident!("{}", name)
-                }
-            }
-        }
-    }
-    fn ty(&self) -> proc_macro2::Ident {
-        match self {
-            Field::Token(_) => format_ident!("SyntaxToken"),
-            Field::Node { ty, .. } => format_ident!("{}", ty),
-        }
-    }
-}
-
-fn lower(grammar: &Grammar) -> AstSrc {
-    let mut res = AstSrc {
-        tokens:
-            "Whitespace Comment String ByteString CString IntNumber FloatNumber Char Byte Ident"
-                .split_ascii_whitespace()
-                .map(|it| it.to_owned())
-                .collect::<Vec<_>>(),
-        ..Default::default()
-    };
-
-    let nodes = grammar.iter().collect::<Vec<_>>();
-
-    for &node in &nodes {
-        let name = grammar[node].name.clone();
-        let rule = &grammar[node].rule;
-        match lower_enum(grammar, rule) {
-            Some(variants) => {
-                let enum_src = AstEnumSrc { doc: Vec::new(), name, traits: Vec::new(), variants };
-                res.enums.push(enum_src);
-            }
-            None => {
-                let mut fields = Vec::new();
-                lower_rule(&mut fields, grammar, None, rule);
-                res.nodes.push(AstNodeSrc { doc: Vec::new(), name, traits: Vec::new(), fields });
-            }
-        }
-    }
-
-    deduplicate_fields(&mut res);
-    extract_enums(&mut res);
-    extract_struct_traits(&mut res);
-    extract_enum_traits(&mut res);
-    res
-}
-
-fn lower_enum(grammar: &Grammar, rule: &Rule) -> Option<Vec<String>> {
-    let alternatives = match rule {
-        Rule::Alt(it) => it,
-        _ => return None,
-    };
-    let mut variants = Vec::new();
-    for alternative in alternatives {
-        match alternative {
-            Rule::Node(it) => variants.push(grammar[*it].name.clone()),
-            Rule::Token(it) if grammar[*it].name == ";" => (),
-            _ => return None,
-        }
-    }
-    Some(variants)
-}
-
-fn lower_rule(acc: &mut Vec<Field>, grammar: &Grammar, label: Option<&String>, rule: &Rule) {
-    if lower_separated_list(acc, grammar, label, rule) {
-        return;
-    }
-
-    match rule {
-        Rule::Node(node) => {
-            let ty = grammar[*node].name.clone();
-            let name = label.cloned().unwrap_or_else(|| to_lower_snake_case(&ty));
-            let field = Field::Node { name, ty, cardinality: Cardinality::Optional };
-            acc.push(field);
-        }
-        Rule::Token(token) => {
-            assert!(label.is_none());
-            let mut name = grammar[*token].name.clone();
-            if name != "int_number" && name != "string" {
-                if "[]{}()".contains(&name) {
-                    name = format!("'{name}'");
-                }
-                let field = Field::Token(name);
-                acc.push(field);
-            }
-        }
-        Rule::Rep(inner) => {
-            if let Rule::Node(node) = &**inner {
-                let ty = grammar[*node].name.clone();
-                let name = label.cloned().unwrap_or_else(|| pluralize(&to_lower_snake_case(&ty)));
-                let field = Field::Node { name, ty, cardinality: Cardinality::Many };
-                acc.push(field);
-                return;
-            }
-            panic!("unhandled rule: {rule:?}")
-        }
-        Rule::Labeled { label: l, rule } => {
-            assert!(label.is_none());
-            let manually_implemented = matches!(
-                l.as_str(),
-                "lhs"
-                    | "rhs"
-                    | "then_branch"
-                    | "else_branch"
-                    | "start"
-                    | "end"
-                    | "op"
-                    | "index"
-                    | "base"
-                    | "value"
-                    | "trait"
-                    | "self_ty"
-                    | "iterable"
-                    | "condition"
-            );
-            if manually_implemented {
-                return;
-            }
-            lower_rule(acc, grammar, Some(l), rule);
-        }
-        Rule::Seq(rules) | Rule::Alt(rules) => {
-            for rule in rules {
-                lower_rule(acc, grammar, label, rule)
-            }
-        }
-        Rule::Opt(rule) => lower_rule(acc, grammar, label, rule),
-    }
-}
-
-// (T (',' T)* ','?)
-fn lower_separated_list(
-    acc: &mut Vec<Field>,
-    grammar: &Grammar,
-    label: Option<&String>,
-    rule: &Rule,
-) -> bool {
-    let rule = match rule {
-        Rule::Seq(it) => it,
-        _ => return false,
-    };
-    let (node, repeat, trailing_sep) = match rule.as_slice() {
-        [Rule::Node(node), Rule::Rep(repeat), Rule::Opt(trailing_sep)] => {
-            (node, repeat, Some(trailing_sep))
-        }
-        [Rule::Node(node), Rule::Rep(repeat)] => (node, repeat, None),
-        _ => return false,
-    };
-    let repeat = match &**repeat {
-        Rule::Seq(it) => it,
-        _ => return false,
-    };
-    if !matches!(
-        repeat.as_slice(),
-        [comma, Rule::Node(n)]
-            if trailing_sep.map_or(true, |it| comma == &**it) && n == node
-    ) {
-        return false;
-    }
-    let ty = grammar[*node].name.clone();
-    let name = label.cloned().unwrap_or_else(|| pluralize(&to_lower_snake_case(&ty)));
-    let field = Field::Node { name, ty, cardinality: Cardinality::Many };
-    acc.push(field);
-    true
-}
-
-fn deduplicate_fields(ast: &mut AstSrc) {
-    for node in &mut ast.nodes {
-        let mut i = 0;
-        'outer: while i < node.fields.len() {
-            for j in 0..i {
-                let f1 = &node.fields[i];
-                let f2 = &node.fields[j];
-                if f1 == f2 {
-                    node.fields.remove(i);
-                    continue 'outer;
-                }
-            }
-            i += 1;
-        }
-    }
-}
-
-fn extract_enums(ast: &mut AstSrc) {
-    for node in &mut ast.nodes {
-        for enm in &ast.enums {
-            let mut to_remove = Vec::new();
-            for (i, field) in node.fields.iter().enumerate() {
-                let ty = field.ty().to_string();
-                if enm.variants.iter().any(|it| it == &ty) {
-                    to_remove.push(i);
-                }
-            }
-            if to_remove.len() == enm.variants.len() {
-                node.remove_field(to_remove);
-                let ty = enm.name.clone();
-                let name = to_lower_snake_case(&ty);
-                node.fields.push(Field::Node { name, ty, cardinality: Cardinality::Optional });
-            }
-        }
-    }
-}
-
-fn extract_struct_traits(ast: &mut AstSrc) {
-    let traits: &[(&str, &[&str])] = &[
-        ("HasAttrs", &["attrs"]),
-        ("HasName", &["name"]),
-        ("HasVisibility", &["visibility"]),
-        ("HasGenericParams", &["generic_param_list", "where_clause"]),
-        ("HasTypeBounds", &["type_bound_list", "colon_token"]),
-        ("HasModuleItem", &["items"]),
-        ("HasLoopBody", &["label", "loop_body"]),
-        ("HasArgList", &["arg_list"]),
-    ];
-
-    for node in &mut ast.nodes {
-        for (name, methods) in traits {
-            extract_struct_trait(node, name, methods);
-        }
-    }
-
-    let nodes_with_doc_comments = [
-        "SourceFile",
-        "Fn",
-        "Struct",
-        "Union",
-        "RecordField",
-        "TupleField",
-        "Enum",
-        "Variant",
-        "Trait",
-        "TraitAlias",
-        "Module",
-        "Static",
-        "Const",
-        "TypeAlias",
-        "Impl",
-        "ExternBlock",
-        "ExternCrate",
-        "MacroCall",
-        "MacroRules",
-        "MacroDef",
-        "Use",
-    ];
-
-    for node in &mut ast.nodes {
-        if nodes_with_doc_comments.contains(&&*node.name) {
-            node.traits.push("HasDocComments".into());
-        }
-    }
-}
-
-fn extract_struct_trait(node: &mut AstNodeSrc, trait_name: &str, methods: &[&str]) {
-    let mut to_remove = Vec::new();
-    for (i, field) in node.fields.iter().enumerate() {
-        let method_name = field.method_name().to_string();
-        if methods.iter().any(|&it| it == method_name) {
-            to_remove.push(i);
-        }
-    }
-    if to_remove.len() == methods.len() {
-        node.traits.push(trait_name.to_owned());
-        node.remove_field(to_remove);
-    }
-}
-
-fn extract_enum_traits(ast: &mut AstSrc) {
-    for enm in &mut ast.enums {
-        if enm.name == "Stmt" {
-            continue;
-        }
-        let nodes = &ast.nodes;
-        let mut variant_traits = enm
-            .variants
-            .iter()
-            .map(|var| nodes.iter().find(|it| &it.name == var).unwrap())
-            .map(|node| node.traits.iter().cloned().collect::<BTreeSet<_>>());
-
-        let mut enum_traits = match variant_traits.next() {
-            Some(it) => it,
-            None => continue,
-        };
-        for traits in variant_traits {
-            enum_traits = enum_traits.intersection(&traits).cloned().collect();
-        }
-        enm.traits = enum_traits.into_iter().collect();
-    }
-}
-
-impl AstNodeSrc {
-    fn remove_field(&mut self, to_remove: Vec<usize>) {
-        to_remove.into_iter().rev().for_each(|idx| {
-            self.fields.remove(idx);
-        });
-    }
-}