Generate only minimal set of ineresting tokens

crates/ra_syntax/src/ast.rs

@@ -80,7 +80,7 @@ impl<N: AstNode> Iterator for AstChildren<N> {
 }
 
 mod support {
-    use super::{AstChildren, AstNode, AstToken, SyntaxKind, SyntaxNode, SyntaxToken};
+    use super::{AstChildren, AstNode, SyntaxKind, SyntaxNode, SyntaxToken};
 
     pub(super) fn child<N: AstNode>(parent: &SyntaxNode) -> Option<N> {
         parent.children().find_map(N::cast)
@@ -90,10 +90,6 @@ mod support {
         AstChildren::new(parent)
     }
 
-    pub(super) fn token<T: AstToken>(parent: &SyntaxNode) -> Option<T> {
-        parent.children_with_tokens().filter_map(|it| it.into_token()).find_map(T::cast)
-    }
-
     pub(super) fn token2(parent: &SyntaxNode, kind: SyntaxKind) -> Option<SyntaxToken> {
         parent.children_with_tokens().filter_map(|it| it.into_token()).find(|it| it.kind() == kind)
     }

crates/ra_syntax/src/ast/edit.rs

@@ -189,15 +189,15 @@ impl ast::RecordFieldList {
 impl ast::TypeParam {
     #[must_use]
     pub fn remove_bounds(&self) -> ast::TypeParam {
-        let colon = match self.colon() {
+        let colon = match self.colon_token() {
             Some(it) => it,
             None => return self.clone(),
         };
         let end = match self.type_bound_list() {
             Some(it) => it.syntax().clone().into(),
-            None => colon.syntax().clone().into(),
+            None => colon.clone().into(),
         };
-        self.replace_children(colon.syntax().clone().into()..=end, iter::empty())
+        self.replace_children(colon.into()..=end, iter::empty())
     }
 }
 

crates/ra_syntax/src/ast/extensions.rs

@@ -5,7 +5,7 @@ use itertools::Itertools;
 use ra_parser::SyntaxKind;
 
 use crate::{
-    ast::{self, support, AstNode, AstToken, AttrInput, NameOwner, SyntaxNode},
+    ast::{self, support, AstNode, AttrInput, NameOwner, SyntaxNode},
     SmolStr, SyntaxElement, SyntaxToken, T,
 };
 
@@ -327,23 +327,23 @@ impl ast::TypeBound {
         }
     }
 
-    pub fn const_question_token(&self) -> Option<ast::Question> {
+    pub fn const_question_token(&self) -> Option<SyntaxToken> {
         self.syntax()
             .children_with_tokens()
             .filter_map(|it| it.into_token())
             .take_while(|it| it.kind() != T![const])
-            .find_map(ast::Question::cast)
+            .find(|it| it.kind() == T![?])
     }
 
-    pub fn question_token(&self) -> Option<ast::Question> {
+    pub fn question_token(&self) -> Option<SyntaxToken> {
         if self.const_token().is_some() {
             self.syntax()
                 .children_with_tokens()
                 .filter_map(|it| it.into_token())
                 .skip_while(|it| it.kind() != T![const])
-                .find_map(ast::Question::cast)
+                .find(|it| it.kind() == T![?])
         } else {
-            support::token(&self.syntax)
+            support::token2(&self.syntax, T![?])
         }
     }
 }
@@ -384,12 +384,12 @@ impl ast::MacroCall {
 }
 
 impl ast::LifetimeParam {
-    pub fn lifetime_bounds(&self) -> impl Iterator<Item = ast::Lifetime> {
+    pub fn lifetime_bounds(&self) -> impl Iterator<Item = SyntaxToken> {
         self.syntax()
             .children_with_tokens()
             .filter_map(|it| it.into_token())
             .skip_while(|x| x.kind() != T![:])
-            .filter_map(ast::Lifetime::cast)
+            .filter(|it| it.kind() == T![lifetime])
     }
 }
 

crates/ra_syntax/src/ast/traits.rs

@@ -6,6 +6,7 @@ use stdx::SepBy;
 use crate::{
     ast::{self, support, AstChildren, AstNode, AstToken},
     syntax_node::SyntaxElementChildren,
+    SyntaxToken, T,
 };
 
 pub trait TypeAscriptionOwner: AstNode {
@@ -63,8 +64,8 @@ pub trait TypeBoundsOwner: AstNode {
         support::child(self.syntax())
     }
 
-    fn colon(&self) -> Option<ast::Colon> {
+    fn colon_token(&self) -> Option<SyntaxToken> {
-        support::token(self.syntax())
+        support::token2(self.syntax(), T![:])
     }
 }
 

xtask/src/ast_src.rs

@@ -225,6 +225,7 @@ pub(crate) const KINDS_SRC: KindsSrc = KindsSrc {
 };
 
 pub(crate) struct AstSrc<'a> {
+    pub(crate) tokens: &'a [&'a str],
     pub(crate) nodes: &'a [AstNodeSrc<'a>],
     pub(crate) enums: &'a [AstEnumSrc<'a>],
 }
@@ -303,6 +304,7 @@ macro_rules! ast_enums {
 }
 
 pub(crate) const AST_SRC: AstSrc = AstSrc {
+    tokens: &["Whitespace", "Comment", "String", "RawString"],
     nodes: &ast_nodes! {
         struct SourceFile: ModuleItemOwner, AttrsOwner {
             modules: [Module],

xtask/src/codegen.rs

@@ -24,6 +24,7 @@ const ERR_INLINE_TESTS_DIR: &str = "crates/ra_syntax/test_data/parser/inline/err
 
 const SYNTAX_KINDS: &str = "crates/ra_parser/src/syntax_kind/generated.rs";
 const AST_NODES: &str = "crates/ra_syntax/src/ast/generated/nodes.rs";
+const AST_TOKENS: &str = "crates/ra_syntax/src/ast/generated/tokens.rs";
 
 const ASSISTS_DIR: &str = "crates/ra_assists/src/handlers";
 const ASSISTS_TESTS: &str = "crates/ra_assists/src/doc_tests/generated.rs";

xtask/src/codegen/gen_syntax.rs

@@ -19,6 +19,10 @@ pub fn generate_syntax(mode: Mode) -> Result<()> {
     let syntax_kinds = generate_syntax_kinds(KINDS_SRC)?;
     update(syntax_kinds_file.as_path(), &syntax_kinds, mode)?;
 
+    let ast_tokens_file = project_root().join(codegen::AST_TOKENS);
+    let contents = generate_tokens(KINDS_SRC, AST_SRC)?;
+    update(ast_tokens_file.as_path(), &contents, mode)?;
+
     let ast_nodes_file = project_root().join(codegen::AST_NODES);
     let contents = generate_nodes(KINDS_SRC, AST_SRC)?;
     update(ast_nodes_file.as_path(), &contents, mode)?;
@@ -33,6 +37,37 @@ struct ElementKinds {
     has_tokens: bool,
 }
 
+fn generate_tokens(kinds: KindsSrc<'_>, grammar: AstSrc<'_>) -> Result<String> {
+    let tokens = grammar.tokens.iter().map(|token| {
+        let name = format_ident!("{}", token);
+        let kind = format_ident!("{}", to_upper_snake_case(token));
+        quote! {
+            #[derive(Debug, Clone, PartialEq, Eq, Hash)]
+            pub struct #name {
+                pub(crate) syntax: SyntaxToken,
+            }
+            impl std::fmt::Display for #name {
+                fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+                    std::fmt::Display::fmt(&self.syntax, f)
+                }
+            }
+            impl AstToken for #name {
+                fn can_cast(kind: SyntaxKind) -> bool { kind == #kind }
+                fn cast(syntax: SyntaxToken) -> Option<Self> {
+                    if Self::can_cast(syntax.kind()) { Some(Self { syntax }) } else { None }
+                }
+                fn syntax(&self) -> &SyntaxToken { &self.syntax }
+            }
+        }
+    });
+
+    let pretty = crate::reformat(quote! {
+        use crate::{SyntaxKind::{self, *}, SyntaxToken, ast::AstToken};
+        #(#tokens)*
+    })?;
+    Ok(pretty)
+}
+
 fn generate_nodes(kinds: KindsSrc<'_>, grammar: AstSrc<'_>) -> Result<String> {
     let nodes = grammar.nodes.iter().map(|node| {
         let name = format_ident!("{}", node.name);