Rename ra_syntax -> syntax

crates/syntax/src/ast/edit.rs

@@ -0,0 +1,642 @@
+//! This module contains functions for editing syntax trees. As the trees are
+//! immutable, all function here return a fresh copy of the tree, instead of
+//! doing an in-place modification.
+use std::{
+    fmt, iter,
+    ops::{self, RangeInclusive},
+};
+
+use arrayvec::ArrayVec;
+
+use crate::{
+    algo::{self, neighbor, SyntaxRewriter},
+    ast::{
+        self,
+        make::{self, tokens},
+        AstNode, TypeBoundsOwner,
+    },
+    AstToken, Direction, InsertPosition, SmolStr, SyntaxElement, SyntaxKind,
+    SyntaxKind::{ATTR, COMMENT, WHITESPACE},
+    SyntaxNode, SyntaxToken, T,
+};
+
+impl ast::BinExpr {
+    #[must_use]
+    pub fn replace_op(&self, op: SyntaxKind) -> Option<ast::BinExpr> {
+        let op_node: SyntaxElement = self.op_details()?.0.into();
+        let to_insert: Option<SyntaxElement> = Some(make::token(op).into());
+        Some(self.replace_children(single_node(op_node), to_insert))
+    }
+}
+
+impl ast::Fn {
+    #[must_use]
+    pub fn with_body(&self, body: ast::BlockExpr) -> ast::Fn {
+        let mut to_insert: ArrayVec<[SyntaxElement; 2]> = ArrayVec::new();
+        let old_body_or_semi: SyntaxElement = if let Some(old_body) = self.body() {
+            old_body.syntax().clone().into()
+        } else if let Some(semi) = self.semicolon_token() {
+            to_insert.push(make::tokens::single_space().into());
+            semi.into()
+        } else {
+            to_insert.push(make::tokens::single_space().into());
+            to_insert.push(body.syntax().clone().into());
+            return self.insert_children(InsertPosition::Last, to_insert);
+        };
+        to_insert.push(body.syntax().clone().into());
+        self.replace_children(single_node(old_body_or_semi), to_insert)
+    }
+}
+
+fn make_multiline<N>(node: N) -> N
+where
+    N: AstNode + Clone,
+{
+    let l_curly = match node.syntax().children_with_tokens().find(|it| it.kind() == T!['{']) {
+        Some(it) => it,
+        None => return node,
+    };
+    let sibling = match l_curly.next_sibling_or_token() {
+        Some(it) => it,
+        None => return node,
+    };
+    let existing_ws = match sibling.as_token() {
+        None => None,
+        Some(tok) if tok.kind() != WHITESPACE => None,
+        Some(ws) => {
+            if ws.text().contains('\n') {
+                return node;
+            }
+            Some(ws.clone())
+        }
+    };
+
+    let indent = leading_indent(node.syntax()).unwrap_or_default();
+    let ws = tokens::WsBuilder::new(&format!("\n{}", indent));
+    let to_insert = iter::once(ws.ws().into());
+    match existing_ws {
+        None => node.insert_children(InsertPosition::After(l_curly), to_insert),
+        Some(ws) => node.replace_children(single_node(ws), to_insert),
+    }
+}
+
+impl ast::AssocItemList {
+    #[must_use]
+    pub fn append_items(
+        &self,
+        items: impl IntoIterator<Item = ast::AssocItem>,
+    ) -> ast::AssocItemList {
+        let mut res = self.clone();
+        if !self.syntax().text().contains_char('\n') {
+            res = make_multiline(res);
+        }
+        items.into_iter().for_each(|it| res = res.append_item(it));
+        res
+    }
+
+    #[must_use]
+    pub fn append_item(&self, item: ast::AssocItem) -> ast::AssocItemList {
+        let (indent, position) = match self.assoc_items().last() {
+            Some(it) => (
+                leading_indent(it.syntax()).unwrap_or_default().to_string(),
+                InsertPosition::After(it.syntax().clone().into()),
+            ),
+            None => match self.l_curly_token() {
+                Some(it) => (
+                    "    ".to_string() + &leading_indent(self.syntax()).unwrap_or_default(),
+                    InsertPosition::After(it.into()),
+                ),
+                None => return self.clone(),
+            },
+        };
+        let ws = tokens::WsBuilder::new(&format!("\n{}", indent));
+        let to_insert: ArrayVec<[SyntaxElement; 2]> =
+            [ws.ws().into(), item.syntax().clone().into()].into();
+        self.insert_children(position, to_insert)
+    }
+}
+
+impl ast::RecordExprFieldList {
+    #[must_use]
+    pub fn append_field(&self, field: &ast::RecordExprField) -> ast::RecordExprFieldList {
+        self.insert_field(InsertPosition::Last, field)
+    }
+
+    #[must_use]
+    pub fn insert_field(
+        &self,
+        position: InsertPosition<&'_ ast::RecordExprField>,
+        field: &ast::RecordExprField,
+    ) -> ast::RecordExprFieldList {
+        let is_multiline = self.syntax().text().contains_char('\n');
+        let ws;
+        let space = if is_multiline {
+            ws = tokens::WsBuilder::new(&format!(
+                "\n{}    ",
+                leading_indent(self.syntax()).unwrap_or_default()
+            ));
+            ws.ws()
+        } else {
+            tokens::single_space()
+        };
+
+        let mut to_insert: ArrayVec<[SyntaxElement; 4]> = ArrayVec::new();
+        to_insert.push(space.into());
+        to_insert.push(field.syntax().clone().into());
+        to_insert.push(make::token(T![,]).into());
+
+        macro_rules! after_l_curly {
+            () => {{
+                let anchor = match self.l_curly_token() {
+                    Some(it) => it.into(),
+                    None => return self.clone(),
+                };
+                InsertPosition::After(anchor)
+            }};
+        }
+
+        macro_rules! after_field {
+            ($anchor:expr) => {
+                if let Some(comma) = $anchor
+                    .syntax()
+                    .siblings_with_tokens(Direction::Next)
+                    .find(|it| it.kind() == T![,])
+                {
+                    InsertPosition::After(comma)
+                } else {
+                    to_insert.insert(0, make::token(T![,]).into());
+                    InsertPosition::After($anchor.syntax().clone().into())
+                }
+            };
+        };
+
+        let position = match position {
+            InsertPosition::First => after_l_curly!(),
+            InsertPosition::Last => {
+                if !is_multiline {
+                    // don't insert comma before curly
+                    to_insert.pop();
+                }
+                match self.fields().last() {
+                    Some(it) => after_field!(it),
+                    None => after_l_curly!(),
+                }
+            }
+            InsertPosition::Before(anchor) => {
+                InsertPosition::Before(anchor.syntax().clone().into())
+            }
+            InsertPosition::After(anchor) => after_field!(anchor),
+        };
+
+        self.insert_children(position, to_insert)
+    }
+}
+
+impl ast::TypeAlias {
+    #[must_use]
+    pub fn remove_bounds(&self) -> ast::TypeAlias {
+        let colon = match self.colon_token() {
+            Some(it) => it,
+            None => return self.clone(),
+        };
+        let end = match self.type_bound_list() {
+            Some(it) => it.syntax().clone().into(),
+            None => colon.clone().into(),
+        };
+        self.replace_children(colon.into()..=end, iter::empty())
+    }
+}
+
+impl ast::TypeParam {
+    #[must_use]
+    pub fn remove_bounds(&self) -> ast::TypeParam {
+        let colon = match self.colon_token() {
+            Some(it) => it,
+            None => return self.clone(),
+        };
+        let end = match self.type_bound_list() {
+            Some(it) => it.syntax().clone().into(),
+            None => colon.clone().into(),
+        };
+        self.replace_children(colon.into()..=end, iter::empty())
+    }
+}
+
+impl ast::Path {
+    #[must_use]
+    pub fn with_segment(&self, segment: ast::PathSegment) -> ast::Path {
+        if let Some(old) = self.segment() {
+            return self.replace_children(
+                single_node(old.syntax().clone()),
+                iter::once(segment.syntax().clone().into()),
+            );
+        }
+        self.clone()
+    }
+}
+
+impl ast::PathSegment {
+    #[must_use]
+    pub fn with_type_args(&self, type_args: ast::GenericArgList) -> ast::PathSegment {
+        self._with_type_args(type_args, false)
+    }
+
+    #[must_use]
+    pub fn with_turbo_fish(&self, type_args: ast::GenericArgList) -> ast::PathSegment {
+        self._with_type_args(type_args, true)
+    }
+
+    fn _with_type_args(&self, type_args: ast::GenericArgList, turbo: bool) -> ast::PathSegment {
+        if let Some(old) = self.generic_arg_list() {
+            return self.replace_children(
+                single_node(old.syntax().clone()),
+                iter::once(type_args.syntax().clone().into()),
+            );
+        }
+        let mut to_insert: ArrayVec<[SyntaxElement; 2]> = ArrayVec::new();
+        if turbo {
+            to_insert.push(make::token(T![::]).into());
+        }
+        to_insert.push(type_args.syntax().clone().into());
+        self.insert_children(InsertPosition::Last, to_insert)
+    }
+}
+
+impl ast::Use {
+    #[must_use]
+    pub fn with_use_tree(&self, use_tree: ast::UseTree) -> ast::Use {
+        if let Some(old) = self.use_tree() {
+            return self.replace_descendant(old, use_tree);
+        }
+        self.clone()
+    }
+
+    pub fn remove(&self) -> SyntaxRewriter<'static> {
+        let mut res = SyntaxRewriter::default();
+        res.delete(self.syntax());
+        let next_ws = self
+            .syntax()
+            .next_sibling_or_token()
+            .and_then(|it| it.into_token())
+            .and_then(ast::Whitespace::cast);
+        if let Some(next_ws) = next_ws {
+            let ws_text = next_ws.syntax().text();
+            if ws_text.starts_with('\n') {
+                let rest = &ws_text[1..];
+                if rest.is_empty() {
+                    res.delete(next_ws.syntax())
+                } else {
+                    res.replace(next_ws.syntax(), &make::tokens::whitespace(rest));
+                }
+            }
+        }
+        res
+    }
+}
+
+impl ast::UseTree {
+    #[must_use]
+    pub fn with_path(&self, path: ast::Path) -> ast::UseTree {
+        if let Some(old) = self.path() {
+            return self.replace_descendant(old, path);
+        }
+        self.clone()
+    }
+
+    #[must_use]
+    pub fn with_use_tree_list(&self, use_tree_list: ast::UseTreeList) -> ast::UseTree {
+        if let Some(old) = self.use_tree_list() {
+            return self.replace_descendant(old, use_tree_list);
+        }
+        self.clone()
+    }
+
+    #[must_use]
+    pub fn split_prefix(&self, prefix: &ast::Path) -> ast::UseTree {
+        let suffix = match split_path_prefix(&prefix) {
+            Some(it) => it,
+            None => return self.clone(),
+        };
+        let use_tree = make::use_tree(
+            suffix,
+            self.use_tree_list(),
+            self.rename(),
+            self.star_token().is_some(),
+        );
+        let nested = make::use_tree_list(iter::once(use_tree));
+        return make::use_tree(prefix.clone(), Some(nested), None, false);
+
+        fn split_path_prefix(prefix: &ast::Path) -> Option<ast::Path> {
+            let parent = prefix.parent_path()?;
+            let segment = parent.segment()?;
+            if algo::has_errors(segment.syntax()) {
+                return None;
+            }
+            let mut res = make::path_unqualified(segment);
+            for p in iter::successors(parent.parent_path(), |it| it.parent_path()) {
+                res = make::path_qualified(res, p.segment()?);
+            }
+            Some(res)
+        }
+    }
+
+    pub fn remove(&self) -> SyntaxRewriter<'static> {
+        let mut res = SyntaxRewriter::default();
+        res.delete(self.syntax());
+        for &dir in [Direction::Next, Direction::Prev].iter() {
+            if let Some(nb) = neighbor(self, dir) {
+                self.syntax()
+                    .siblings_with_tokens(dir)
+                    .skip(1)
+                    .take_while(|it| it.as_node() != Some(nb.syntax()))
+                    .for_each(|el| res.delete(&el));
+                return res;
+            }
+        }
+        res
+    }
+}
+
+impl ast::MatchArmList {
+    #[must_use]
+    pub fn append_arms(&self, items: impl IntoIterator<Item = ast::MatchArm>) -> ast::MatchArmList {
+        let mut res = self.clone();
+        res = res.strip_if_only_whitespace();
+        if !res.syntax().text().contains_char('\n') {
+            res = make_multiline(res);
+        }
+        items.into_iter().for_each(|it| res = res.append_arm(it));
+        res
+    }
+
+    fn strip_if_only_whitespace(&self) -> ast::MatchArmList {
+        let mut iter = self.syntax().children_with_tokens().skip_while(|it| it.kind() != T!['{']);
+        iter.next(); // Eat the curly
+        let mut inner = iter.take_while(|it| it.kind() != T!['}']);
+        if !inner.clone().all(|it| it.kind() == WHITESPACE) {
+            return self.clone();
+        }
+        let start = match inner.next() {
+            Some(s) => s,
+            None => return self.clone(),
+        };
+        let end = match inner.last() {
+            Some(s) => s,
+            None => start.clone(),
+        };
+        self.replace_children(start..=end, &mut iter::empty())
+    }
+
+    #[must_use]
+    pub fn remove_placeholder(&self) -> ast::MatchArmList {
+        let placeholder =
+            self.arms().find(|arm| matches!(arm.pat(), Some(ast::Pat::WildcardPat(_))));
+        if let Some(placeholder) = placeholder {
+            self.remove_arm(&placeholder)
+        } else {
+            self.clone()
+        }
+    }
+
+    #[must_use]
+    fn remove_arm(&self, arm: &ast::MatchArm) -> ast::MatchArmList {
+        let start = arm.syntax().clone();
+        let end = if let Some(comma) = start
+            .siblings_with_tokens(Direction::Next)
+            .skip(1)
+            .skip_while(|it| it.kind().is_trivia())
+            .next()
+            .filter(|it| it.kind() == T![,])
+        {
+            comma
+        } else {
+            start.clone().into()
+        };
+        self.replace_children(start.into()..=end, None)
+    }
+
+    #[must_use]
+    pub fn append_arm(&self, item: ast::MatchArm) -> ast::MatchArmList {
+        let r_curly = match self.syntax().children_with_tokens().find(|it| it.kind() == T!['}']) {
+            Some(t) => t,
+            None => return self.clone(),
+        };
+        let position = InsertPosition::Before(r_curly.into());
+        let arm_ws = tokens::WsBuilder::new("    ");
+        let match_indent = &leading_indent(self.syntax()).unwrap_or_default();
+        let match_ws = tokens::WsBuilder::new(&format!("\n{}", match_indent));
+        let to_insert: ArrayVec<[SyntaxElement; 3]> =
+            [arm_ws.ws().into(), item.syntax().clone().into(), match_ws.ws().into()].into();
+        self.insert_children(position, to_insert)
+    }
+}
+
+#[must_use]
+pub fn remove_attrs_and_docs<N: ast::AttrsOwner>(node: &N) -> N {
+    N::cast(remove_attrs_and_docs_inner(node.syntax().clone())).unwrap()
+}
+
+fn remove_attrs_and_docs_inner(mut node: SyntaxNode) -> SyntaxNode {
+    while let Some(start) =
+        node.children_with_tokens().find(|it| it.kind() == ATTR || it.kind() == COMMENT)
+    {
+        let end = match &start.next_sibling_or_token() {
+            Some(el) if el.kind() == WHITESPACE => el.clone(),
+            Some(_) | None => start.clone(),
+        };
+        node = algo::replace_children(&node, start..=end, &mut iter::empty());
+    }
+    node
+}
+
+#[derive(Debug, Clone, Copy)]
+pub struct IndentLevel(pub u8);
+
+impl From<u8> for IndentLevel {
+    fn from(level: u8) -> IndentLevel {
+        IndentLevel(level)
+    }
+}
+
+impl fmt::Display for IndentLevel {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let spaces = "                                        ";
+        let buf;
+        let len = self.0 as usize * 4;
+        let indent = if len <= spaces.len() {
+            &spaces[..len]
+        } else {
+            buf = iter::repeat(' ').take(len).collect::<String>();
+            &buf
+        };
+        fmt::Display::fmt(indent, f)
+    }
+}
+
+impl ops::Add<u8> for IndentLevel {
+    type Output = IndentLevel;
+    fn add(self, rhs: u8) -> IndentLevel {
+        IndentLevel(self.0 + rhs)
+    }
+}
+
+impl IndentLevel {
+    pub fn from_node(node: &SyntaxNode) -> IndentLevel {
+        let first_token = match node.first_token() {
+            Some(it) => it,
+            None => return IndentLevel(0),
+        };
+        for ws in prev_tokens(first_token).filter_map(ast::Whitespace::cast) {
+            let text = ws.syntax().text();
+            if let Some(pos) = text.rfind('\n') {
+                let level = text[pos + 1..].chars().count() / 4;
+                return IndentLevel(level as u8);
+            }
+        }
+        IndentLevel(0)
+    }
+
+    /// XXX: this intentionally doesn't change the indent of the very first token.
+    /// Ie, in something like
+    /// ```
+    /// fn foo() {
+    ///    92
+    /// }
+    /// ```
+    /// if you indent the block, the `{` token would stay put.
+    fn increase_indent(self, node: SyntaxNode) -> SyntaxNode {
+        let mut rewriter = SyntaxRewriter::default();
+        node.descendants_with_tokens()
+            .filter_map(|el| el.into_token())
+            .filter_map(ast::Whitespace::cast)
+            .filter(|ws| {
+                let text = ws.syntax().text();
+                text.contains('\n')
+            })
+            .for_each(|ws| {
+                let new_ws = make::tokens::whitespace(&format!("{}{}", ws.syntax(), self,));
+                rewriter.replace(ws.syntax(), &new_ws)
+            });
+        rewriter.rewrite(&node)
+    }
+
+    fn decrease_indent(self, node: SyntaxNode) -> SyntaxNode {
+        let mut rewriter = SyntaxRewriter::default();
+        node.descendants_with_tokens()
+            .filter_map(|el| el.into_token())
+            .filter_map(ast::Whitespace::cast)
+            .filter(|ws| {
+                let text = ws.syntax().text();
+                text.contains('\n')
+            })
+            .for_each(|ws| {
+                let new_ws = make::tokens::whitespace(
+                    &ws.syntax().text().replace(&format!("\n{}", self), "\n"),
+                );
+                rewriter.replace(ws.syntax(), &new_ws)
+            });
+        rewriter.rewrite(&node)
+    }
+}
+
+// FIXME: replace usages with IndentLevel above
+fn leading_indent(node: &SyntaxNode) -> Option<SmolStr> {
+    for token in prev_tokens(node.first_token()?) {
+        if let Some(ws) = ast::Whitespace::cast(token.clone()) {
+            let ws_text = ws.text();
+            if let Some(pos) = ws_text.rfind('\n') {
+                return Some(ws_text[pos + 1..].into());
+            }
+        }
+        if token.text().contains('\n') {
+            break;
+        }
+    }
+    None
+}
+
+fn prev_tokens(token: SyntaxToken) -> impl Iterator<Item = SyntaxToken> {
+    iter::successors(Some(token), |token| token.prev_token())
+}
+
+pub trait AstNodeEdit: AstNode + Clone + Sized {
+    #[must_use]
+    fn insert_children(
+        &self,
+        position: InsertPosition<SyntaxElement>,
+        to_insert: impl IntoIterator<Item = SyntaxElement>,
+    ) -> Self {
+        let new_syntax = algo::insert_children(self.syntax(), position, to_insert);
+        Self::cast(new_syntax).unwrap()
+    }
+
+    #[must_use]
+    fn replace_children(
+        &self,
+        to_replace: RangeInclusive<SyntaxElement>,
+        to_insert: impl IntoIterator<Item = SyntaxElement>,
+    ) -> Self {
+        let new_syntax = algo::replace_children(self.syntax(), to_replace, to_insert);
+        Self::cast(new_syntax).unwrap()
+    }
+
+    #[must_use]
+    fn replace_descendant<D: AstNode>(&self, old: D, new: D) -> Self {
+        self.replace_descendants(iter::once((old, new)))
+    }
+
+    #[must_use]
+    fn replace_descendants<D: AstNode>(
+        &self,
+        replacement_map: impl IntoIterator<Item = (D, D)>,
+    ) -> Self {
+        let mut rewriter = SyntaxRewriter::default();
+        for (from, to) in replacement_map {
+            rewriter.replace(from.syntax(), to.syntax())
+        }
+        rewriter.rewrite_ast(self)
+    }
+    #[must_use]
+    fn indent(&self, level: IndentLevel) -> Self {
+        Self::cast(level.increase_indent(self.syntax().clone())).unwrap()
+    }
+    #[must_use]
+    fn dedent(&self, level: IndentLevel) -> Self {
+        Self::cast(level.decrease_indent(self.syntax().clone())).unwrap()
+    }
+    #[must_use]
+    fn reset_indent(&self) -> Self {
+        let level = IndentLevel::from_node(self.syntax());
+        self.dedent(level)
+    }
+}
+
+impl<N: AstNode + Clone> AstNodeEdit for N {}
+
+fn single_node(element: impl Into<SyntaxElement>) -> RangeInclusive<SyntaxElement> {
+    let element = element.into();
+    element.clone()..=element
+}
+
+#[test]
+fn test_increase_indent() {
+    let arm_list = {
+        let arm = make::match_arm(iter::once(make::wildcard_pat().into()), make::expr_unit());
+        make::match_arm_list(vec![arm.clone(), arm])
+    };
+    assert_eq!(
+        arm_list.syntax().to_string(),
+        "{
+    _ => (),
+    _ => (),
+}"
+    );
+    let indented = arm_list.indent(IndentLevel(2));
+    assert_eq!(
+        indented.syntax().to_string(),
+        "{
+            _ => (),
+            _ => (),
+        }"
+    );
+}

crates/syntax/src/ast/expr_ext.rs

@@ -0,0 +1,418 @@
+//! Various extension methods to ast Expr Nodes, which are hard to code-generate.
+
+use crate::{
+    ast::{self, support, AstChildren, AstNode},
+    SmolStr,
+    SyntaxKind::*,
+    SyntaxToken, T,
+};
+
+impl ast::AttrsOwner for ast::Expr {}
+
+impl ast::Expr {
+    pub fn is_block_like(&self) -> bool {
+        match self {
+            ast::Expr::IfExpr(_)
+            | ast::Expr::LoopExpr(_)
+            | ast::Expr::ForExpr(_)
+            | ast::Expr::WhileExpr(_)
+            | ast::Expr::BlockExpr(_)
+            | ast::Expr::MatchExpr(_)
+            | ast::Expr::EffectExpr(_) => true,
+            _ => false,
+        }
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum ElseBranch {
+    Block(ast::BlockExpr),
+    IfExpr(ast::IfExpr),
+}
+
+impl ast::IfExpr {
+    pub fn then_branch(&self) -> Option<ast::BlockExpr> {
+        self.blocks().next()
+    }
+    pub fn else_branch(&self) -> Option<ElseBranch> {
+        let res = match self.blocks().nth(1) {
+            Some(block) => ElseBranch::Block(block),
+            None => {
+                let elif: ast::IfExpr = support::child(self.syntax())?;
+                ElseBranch::IfExpr(elif)
+            }
+        };
+        Some(res)
+    }
+
+    pub fn blocks(&self) -> AstChildren<ast::BlockExpr> {
+        support::children(self.syntax())
+    }
+}
+
+#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
+pub enum PrefixOp {
+    /// The `*` operator for dereferencing
+    Deref,
+    /// The `!` operator for logical inversion
+    Not,
+    /// The `-` operator for negation
+    Neg,
+}
+
+impl ast::PrefixExpr {
+    pub fn op_kind(&self) -> Option<PrefixOp> {
+        match self.op_token()?.kind() {
+            T![*] => Some(PrefixOp::Deref),
+            T![!] => Some(PrefixOp::Not),
+            T![-] => Some(PrefixOp::Neg),
+            _ => None,
+        }
+    }
+
+    pub fn op_token(&self) -> Option<SyntaxToken> {
+        self.syntax().first_child_or_token()?.into_token()
+    }
+}
+
+#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
+pub enum BinOp {
+    /// The `||` operator for boolean OR
+    BooleanOr,
+    /// The `&&` operator for boolean AND
+    BooleanAnd,
+    /// The `==` operator for equality testing
+    EqualityTest,
+    /// The `!=` operator for equality testing
+    NegatedEqualityTest,
+    /// The `<=` operator for lesser-equal testing
+    LesserEqualTest,
+    /// The `>=` operator for greater-equal testing
+    GreaterEqualTest,
+    /// The `<` operator for comparison
+    LesserTest,
+    /// The `>` operator for comparison
+    GreaterTest,
+    /// The `+` operator for addition
+    Addition,
+    /// The `*` operator for multiplication
+    Multiplication,
+    /// The `-` operator for subtraction
+    Subtraction,
+    /// The `/` operator for division
+    Division,
+    /// The `%` operator for remainder after division
+    Remainder,
+    /// The `<<` operator for left shift
+    LeftShift,
+    /// The `>>` operator for right shift
+    RightShift,
+    /// The `^` operator for bitwise XOR
+    BitwiseXor,
+    /// The `|` operator for bitwise OR
+    BitwiseOr,
+    /// The `&` operator for bitwise AND
+    BitwiseAnd,
+    /// The `=` operator for assignment
+    Assignment,
+    /// The `+=` operator for assignment after addition
+    AddAssign,
+    /// The `/=` operator for assignment after division
+    DivAssign,
+    /// The `*=` operator for assignment after multiplication
+    MulAssign,
+    /// The `%=` operator for assignment after remainders
+    RemAssign,
+    /// The `>>=` operator for assignment after shifting right
+    ShrAssign,
+    /// The `<<=` operator for assignment after shifting left
+    ShlAssign,
+    /// The `-=` operator for assignment after subtraction
+    SubAssign,
+    /// The `|=` operator for assignment after bitwise OR
+    BitOrAssign,
+    /// The `&=` operator for assignment after bitwise AND
+    BitAndAssign,
+    /// The `^=` operator for assignment after bitwise XOR
+    BitXorAssign,
+}
+
+impl BinOp {
+    pub fn is_assignment(self) -> bool {
+        match self {
+            BinOp::Assignment
+            | BinOp::AddAssign
+            | BinOp::DivAssign
+            | BinOp::MulAssign
+            | BinOp::RemAssign
+            | BinOp::ShrAssign
+            | BinOp::ShlAssign
+            | BinOp::SubAssign
+            | BinOp::BitOrAssign
+            | BinOp::BitAndAssign
+            | BinOp::BitXorAssign => true,
+            _ => false,
+        }
+    }
+}
+
+impl ast::BinExpr {
+    pub fn op_details(&self) -> Option<(SyntaxToken, BinOp)> {
+        self.syntax().children_with_tokens().filter_map(|it| it.into_token()).find_map(|c| {
+            let bin_op = match c.kind() {
+                T![||] => BinOp::BooleanOr,
+                T![&&] => BinOp::BooleanAnd,
+                T![==] => BinOp::EqualityTest,
+                T![!=] => BinOp::NegatedEqualityTest,
+                T![<=] => BinOp::LesserEqualTest,
+                T![>=] => BinOp::GreaterEqualTest,
+                T![<] => BinOp::LesserTest,
+                T![>] => BinOp::GreaterTest,
+                T![+] => BinOp::Addition,
+                T![*] => BinOp::Multiplication,
+                T![-] => BinOp::Subtraction,
+                T![/] => BinOp::Division,
+                T![%] => BinOp::Remainder,
+                T![<<] => BinOp::LeftShift,
+                T![>>] => BinOp::RightShift,
+                T![^] => BinOp::BitwiseXor,
+                T![|] => BinOp::BitwiseOr,
+                T![&] => BinOp::BitwiseAnd,
+                T![=] => BinOp::Assignment,
+                T![+=] => BinOp::AddAssign,
+                T![/=] => BinOp::DivAssign,
+                T![*=] => BinOp::MulAssign,
+                T![%=] => BinOp::RemAssign,
+                T![>>=] => BinOp::ShrAssign,
+                T![<<=] => BinOp::ShlAssign,
+                T![-=] => BinOp::SubAssign,
+                T![|=] => BinOp::BitOrAssign,
+                T![&=] => BinOp::BitAndAssign,
+                T![^=] => BinOp::BitXorAssign,
+                _ => return None,
+            };
+            Some((c, bin_op))
+        })
+    }
+
+    pub fn op_kind(&self) -> Option<BinOp> {
+        self.op_details().map(|t| t.1)
+    }
+
+    pub fn op_token(&self) -> Option<SyntaxToken> {
+        self.op_details().map(|t| t.0)
+    }
+
+    pub fn lhs(&self) -> Option<ast::Expr> {
+        support::children(self.syntax()).next()
+    }
+
+    pub fn rhs(&self) -> Option<ast::Expr> {
+        support::children(self.syntax()).nth(1)
+    }
+
+    pub fn sub_exprs(&self) -> (Option<ast::Expr>, Option<ast::Expr>) {
+        let mut children = support::children(self.syntax());
+        let first = children.next();
+        let second = children.next();
+        (first, second)
+    }
+}
+
+#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
+pub enum RangeOp {
+    /// `..`
+    Exclusive,
+    /// `..=`
+    Inclusive,
+}
+
+impl ast::RangeExpr {
+    fn op_details(&self) -> Option<(usize, SyntaxToken, RangeOp)> {
+        self.syntax().children_with_tokens().enumerate().find_map(|(ix, child)| {
+            let token = child.into_token()?;
+            let bin_op = match token.kind() {
+                T![..] => RangeOp::Exclusive,
+                T![..=] => RangeOp::Inclusive,
+                _ => return None,
+            };
+            Some((ix, token, bin_op))
+        })
+    }
+
+    pub fn op_kind(&self) -> Option<RangeOp> {
+        self.op_details().map(|t| t.2)
+    }
+
+    pub fn op_token(&self) -> Option<SyntaxToken> {
+        self.op_details().map(|t| t.1)
+    }
+
+    pub fn start(&self) -> Option<ast::Expr> {
+        let op_ix = self.op_details()?.0;
+        self.syntax()
+            .children_with_tokens()
+            .take(op_ix)
+            .find_map(|it| ast::Expr::cast(it.into_node()?))
+    }
+
+    pub fn end(&self) -> Option<ast::Expr> {
+        let op_ix = self.op_details()?.0;
+        self.syntax()
+            .children_with_tokens()
+            .skip(op_ix + 1)
+            .find_map(|it| ast::Expr::cast(it.into_node()?))
+    }
+}
+
+impl ast::IndexExpr {
+    pub fn base(&self) -> Option<ast::Expr> {
+        support::children(self.syntax()).next()
+    }
+    pub fn index(&self) -> Option<ast::Expr> {
+        support::children(self.syntax()).nth(1)
+    }
+}
+
+pub enum ArrayExprKind {
+    Repeat { initializer: Option<ast::Expr>, repeat: Option<ast::Expr> },
+    ElementList(AstChildren<ast::Expr>),
+}
+
+impl ast::ArrayExpr {
+    pub fn kind(&self) -> ArrayExprKind {
+        if self.is_repeat() {
+            ArrayExprKind::Repeat {
+                initializer: support::children(self.syntax()).next(),
+                repeat: support::children(self.syntax()).nth(1),
+            }
+        } else {
+            ArrayExprKind::ElementList(support::children(self.syntax()))
+        }
+    }
+
+    fn is_repeat(&self) -> bool {
+        self.syntax().children_with_tokens().any(|it| it.kind() == T![;])
+    }
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Hash)]
+pub enum LiteralKind {
+    String,
+    ByteString,
+    Char,
+    Byte,
+    IntNumber { suffix: Option<SmolStr> },
+    FloatNumber { suffix: Option<SmolStr> },
+    Bool(bool),
+}
+
+impl ast::Literal {
+    pub fn token(&self) -> SyntaxToken {
+        self.syntax()
+            .children_with_tokens()
+            .find(|e| e.kind() != ATTR && !e.kind().is_trivia())
+            .and_then(|e| e.into_token())
+            .unwrap()
+    }
+
+    fn find_suffix(text: &str, possible_suffixes: &[&str]) -> Option<SmolStr> {
+        possible_suffixes
+            .iter()
+            .find(|&suffix| text.ends_with(suffix))
+            .map(|&suffix| SmolStr::new(suffix))
+    }
+
+    pub fn kind(&self) -> LiteralKind {
+        const INT_SUFFIXES: [&str; 12] = [
+            "u64", "u32", "u16", "u8", "usize", "isize", "i64", "i32", "i16", "i8", "u128", "i128",
+        ];
+        const FLOAT_SUFFIXES: [&str; 2] = ["f32", "f64"];
+
+        let token = self.token();
+
+        match token.kind() {
+            INT_NUMBER => {
+                // FYI: there was a bug here previously, thus the if statement below is necessary.
+                // The lexer treats e.g. `1f64` as an integer literal. See
+                // https://github.com/rust-analyzer/rust-analyzer/issues/1592
+                // and the comments on the linked PR.
+
+                let text = token.text();
+                if let suffix @ Some(_) = Self::find_suffix(&text, &FLOAT_SUFFIXES) {
+                    LiteralKind::FloatNumber { suffix }
+                } else {
+                    LiteralKind::IntNumber { suffix: Self::find_suffix(&text, &INT_SUFFIXES) }
+                }
+            }
+            FLOAT_NUMBER => {
+                let text = token.text();
+                LiteralKind::FloatNumber { suffix: Self::find_suffix(&text, &FLOAT_SUFFIXES) }
+            }
+            STRING | RAW_STRING => LiteralKind::String,
+            T![true] => LiteralKind::Bool(true),
+            T![false] => LiteralKind::Bool(false),
+            BYTE_STRING | RAW_BYTE_STRING => LiteralKind::ByteString,
+            CHAR => LiteralKind::Char,
+            BYTE => LiteralKind::Byte,
+            _ => unreachable!(),
+        }
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum Effect {
+    Async(SyntaxToken),
+    Unsafe(SyntaxToken),
+    Try(SyntaxToken),
+    // Very much not an effect, but we stuff it into this node anyway
+    Label(ast::Label),
+}
+
+impl ast::EffectExpr {
+    pub fn effect(&self) -> Effect {
+        if let Some(token) = self.async_token() {
+            return Effect::Async(token);
+        }
+        if let Some(token) = self.unsafe_token() {
+            return Effect::Unsafe(token);
+        }
+        if let Some(token) = self.try_token() {
+            return Effect::Try(token);
+        }
+        if let Some(label) = self.label() {
+            return Effect::Label(label);
+        }
+        unreachable!("ast::EffectExpr without Effect")
+    }
+}
+
+impl ast::BlockExpr {
+    /// false if the block is an intrinsic part of the syntax and can't be
+    /// replaced with arbitrary expression.
+    ///
+    /// ```not_rust
+    /// fn foo() { not_stand_alone }
+    /// const FOO: () = { stand_alone };
+    /// ```
+    pub fn is_standalone(&self) -> bool {
+        let parent = match self.syntax().parent() {
+            Some(it) => it,
+            None => return true,
+        };
+        !matches!(parent.kind(), FN | IF_EXPR | WHILE_EXPR | LOOP_EXPR | EFFECT_EXPR)
+    }
+}
+
+#[test]
+fn test_literal_with_attr() {
+    let parse = ast::SourceFile::parse(r#"const _: &str = { #[attr] "Hello" };"#);
+    let lit = parse.tree().syntax().descendants().find_map(ast::Literal::cast).unwrap();
+    assert_eq!(lit.token().text(), r#""Hello""#);
+}
+
+impl ast::RecordExprField {
+    pub fn parent_record_lit(&self) -> ast::RecordExpr {
+        self.syntax().ancestors().find_map(ast::RecordExpr::cast).unwrap()
+    }
+}

crates/syntax/src/ast/generated.rs

@@ -0,0 +1,41 @@
+//! This file is actually hand-written, but the submodules are indeed generated.
+#[rustfmt::skip]
+mod nodes;
+#[rustfmt::skip]
+mod tokens;
+
+use crate::{
+    AstNode,
+    SyntaxKind::{self, *},
+    SyntaxNode,
+};
+
+pub use {nodes::*, tokens::*};
+
+// Stmt is the only nested enum, so it's easier to just hand-write it
+impl AstNode for Stmt {
+    fn can_cast(kind: SyntaxKind) -> bool {
+        match kind {
+            LET_STMT | EXPR_STMT => true,
+            _ => Item::can_cast(kind),
+        }
+    }
+    fn cast(syntax: SyntaxNode) -> Option<Self> {
+        let res = match syntax.kind() {
+            LET_STMT => Stmt::LetStmt(LetStmt { syntax }),
+            EXPR_STMT => Stmt::ExprStmt(ExprStmt { syntax }),
+            _ => {
+                let item = Item::cast(syntax)?;
+                Stmt::Item(item)
+            }
+        };
+        Some(res)
+    }
+    fn syntax(&self) -> &SyntaxNode {
+        match self {
+            Stmt::LetStmt(it) => &it.syntax,
+            Stmt::ExprStmt(it) => &it.syntax,
+            Stmt::Item(it) => it.syntax(),
+        }
+    }
+}

crates/syntax/src/ast/generated/tokens.rs

@@ -0,0 +1,91 @@
+//! Generated file, do not edit by hand, see `xtask/src/codegen`
+
+use crate::{
+    ast::AstToken,
+    SyntaxKind::{self, *},
+    SyntaxToken,
+};
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct Whitespace {
+    pub(crate) syntax: SyntaxToken,
+}
+impl std::fmt::Display for Whitespace {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        std::fmt::Display::fmt(&self.syntax, f)
+    }
+}
+impl AstToken for Whitespace {
+    fn can_cast(kind: SyntaxKind) -> bool { kind == WHITESPACE }
+    fn cast(syntax: SyntaxToken) -> Option<Self> {
+        if Self::can_cast(syntax.kind()) {
+            Some(Self { syntax })
+        } else {
+            None
+        }
+    }
+    fn syntax(&self) -> &SyntaxToken { &self.syntax }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct Comment {
+    pub(crate) syntax: SyntaxToken,
+}
+impl std::fmt::Display for Comment {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        std::fmt::Display::fmt(&self.syntax, f)
+    }
+}
+impl AstToken for Comment {
+    fn can_cast(kind: SyntaxKind) -> bool { kind == COMMENT }
+    fn cast(syntax: SyntaxToken) -> Option<Self> {
+        if Self::can_cast(syntax.kind()) {
+            Some(Self { syntax })
+        } else {
+            None
+        }
+    }
+    fn syntax(&self) -> &SyntaxToken { &self.syntax }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct String {
+    pub(crate) syntax: SyntaxToken,
+}
+impl std::fmt::Display for String {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        std::fmt::Display::fmt(&self.syntax, f)
+    }
+}
+impl AstToken for String {
+    fn can_cast(kind: SyntaxKind) -> bool { kind == STRING }
+    fn cast(syntax: SyntaxToken) -> Option<Self> {
+        if Self::can_cast(syntax.kind()) {
+            Some(Self { syntax })
+        } else {
+            None
+        }
+    }
+    fn syntax(&self) -> &SyntaxToken { &self.syntax }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct RawString {
+    pub(crate) syntax: SyntaxToken,
+}
+impl std::fmt::Display for RawString {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        std::fmt::Display::fmt(&self.syntax, f)
+    }
+}
+impl AstToken for RawString {
+    fn can_cast(kind: SyntaxKind) -> bool { kind == RAW_STRING }
+    fn cast(syntax: SyntaxToken) -> Option<Self> {
+        if Self::can_cast(syntax.kind()) {
+            Some(Self { syntax })
+        } else {
+            None
+        }
+    }
+    fn syntax(&self) -> &SyntaxToken { &self.syntax }
+}

crates/syntax/src/ast/make.rs

@@ -0,0 +1,392 @@
+//! This module contains free-standing functions for creating AST fragments out
+//! of smaller pieces.
+//!
+//! Note that all functions here intended to be stupid constructors, which just
+//! assemble a finish node from immediate children. If you want to do something
+//! smarter than that, it probably doesn't belong in this module.
+use itertools::Itertools;
+use stdx::format_to;
+
+use crate::{ast, AstNode, SourceFile, SyntaxKind, SyntaxNode, SyntaxToken};
+
+pub fn name(text: &str) -> ast::Name {
+    ast_from_text(&format!("mod {};", text))
+}
+
+pub fn name_ref(text: &str) -> ast::NameRef {
+    ast_from_text(&format!("fn f() {{ {}; }}", text))
+}
+
+pub fn ty(text: &str) -> ast::Type {
+    ast_from_text(&format!("impl {} for D {{}};", text))
+}
+
+pub fn path_segment(name_ref: ast::NameRef) -> ast::PathSegment {
+    ast_from_text(&format!("use {};", name_ref))
+}
+pub fn path_unqualified(segment: ast::PathSegment) -> ast::Path {
+    path_from_text(&format!("use {}", segment))
+}
+pub fn path_qualified(qual: ast::Path, segment: ast::PathSegment) -> ast::Path {
+    path_from_text(&format!("{}::{}", qual, segment))
+}
+pub fn path_from_text(text: &str) -> ast::Path {
+    ast_from_text(text)
+}
+
+pub fn use_tree(
+    path: ast::Path,
+    use_tree_list: Option<ast::UseTreeList>,
+    alias: Option<ast::Rename>,
+    add_star: bool,
+) -> ast::UseTree {
+    let mut buf = "use ".to_string();
+    buf += &path.syntax().to_string();
+    if let Some(use_tree_list) = use_tree_list {
+        format_to!(buf, "::{}", use_tree_list);
+    }
+    if add_star {
+        buf += "::*";
+    }
+
+    if let Some(alias) = alias {
+        format_to!(buf, " {}", alias);
+    }
+    ast_from_text(&buf)
+}
+
+pub fn use_tree_list(use_trees: impl IntoIterator<Item = ast::UseTree>) -> ast::UseTreeList {
+    let use_trees = use_trees.into_iter().map(|it| it.syntax().clone()).join(", ");
+    ast_from_text(&format!("use {{{}}};", use_trees))
+}
+
+pub fn use_(use_tree: ast::UseTree) -> ast::Use {
+    ast_from_text(&format!("use {};", use_tree))
+}
+
+pub fn record_expr_field(name: ast::NameRef, expr: Option<ast::Expr>) -> ast::RecordExprField {
+    return match expr {
+        Some(expr) => from_text(&format!("{}: {}", name, expr)),
+        None => from_text(&name.to_string()),
+    };
+
+    fn from_text(text: &str) -> ast::RecordExprField {
+        ast_from_text(&format!("fn f() {{ S {{ {}, }} }}", text))
+    }
+}
+
+pub fn record_field(name: ast::NameRef, ty: ast::Type) -> ast::RecordField {
+    ast_from_text(&format!("struct S {{ {}: {}, }}", name, ty))
+}
+
+pub fn block_expr(
+    stmts: impl IntoIterator<Item = ast::Stmt>,
+    tail_expr: Option<ast::Expr>,
+) -> ast::BlockExpr {
+    let mut buf = "{\n".to_string();
+    for stmt in stmts.into_iter() {
+        format_to!(buf, "    {}\n", stmt);
+    }
+    if let Some(tail_expr) = tail_expr {
+        format_to!(buf, "    {}\n", tail_expr)
+    }
+    buf += "}";
+    ast_from_text(&format!("fn f() {}", buf))
+}
+
+pub fn expr_unit() -> ast::Expr {
+    expr_from_text("()")
+}
+pub fn expr_empty_block() -> ast::Expr {
+    expr_from_text("{}")
+}
+pub fn expr_unimplemented() -> ast::Expr {
+    expr_from_text("unimplemented!()")
+}
+pub fn expr_unreachable() -> ast::Expr {
+    expr_from_text("unreachable!()")
+}
+pub fn expr_todo() -> ast::Expr {
+    expr_from_text("todo!()")
+}
+pub fn expr_path(path: ast::Path) -> ast::Expr {
+    expr_from_text(&path.to_string())
+}
+pub fn expr_continue() -> ast::Expr {
+    expr_from_text("continue")
+}
+pub fn expr_break() -> ast::Expr {
+    expr_from_text("break")
+}
+pub fn expr_return() -> ast::Expr {
+    expr_from_text("return")
+}
+pub fn expr_match(expr: ast::Expr, match_arm_list: ast::MatchArmList) -> ast::Expr {
+    expr_from_text(&format!("match {} {}", expr, match_arm_list))
+}
+pub fn expr_if(condition: ast::Condition, then_branch: ast::BlockExpr) -> ast::Expr {
+    expr_from_text(&format!("if {} {}", condition, then_branch))
+}
+pub fn expr_prefix(op: SyntaxKind, expr: ast::Expr) -> ast::Expr {
+    let token = token(op);
+    expr_from_text(&format!("{}{}", token, expr))
+}
+fn expr_from_text(text: &str) -> ast::Expr {
+    ast_from_text(&format!("const C: () = {};", text))
+}
+
+pub fn try_expr_from_text(text: &str) -> Option<ast::Expr> {
+    try_ast_from_text(&format!("const C: () = {};", text))
+}
+
+pub fn condition(expr: ast::Expr, pattern: Option<ast::Pat>) -> ast::Condition {
+    match pattern {
+        None => ast_from_text(&format!("const _: () = while {} {{}};", expr)),
+        Some(pattern) => {
+            ast_from_text(&format!("const _: () = while let {} = {} {{}};", pattern, expr))
+        }
+    }
+}
+
+pub fn ident_pat(name: ast::Name) -> ast::IdentPat {
+    return from_text(name.text());
+
+    fn from_text(text: &str) -> ast::IdentPat {
+        ast_from_text(&format!("fn f({}: ())", text))
+    }
+}
+
+pub fn wildcard_pat() -> ast::WildcardPat {
+    return from_text("_");
+
+    fn from_text(text: &str) -> ast::WildcardPat {
+        ast_from_text(&format!("fn f({}: ())", text))
+    }
+}
+
+/// Creates a tuple of patterns from an interator of patterns.
+///
+/// Invariant: `pats` must be length > 1
+///
+/// FIXME handle `pats` length == 1
+pub fn tuple_pat(pats: impl IntoIterator<Item = ast::Pat>) -> ast::TuplePat {
+    let pats_str = pats.into_iter().map(|p| p.to_string()).join(", ");
+    return from_text(&format!("({})", pats_str));
+
+    fn from_text(text: &str) -> ast::TuplePat {
+        ast_from_text(&format!("fn f({}: ())", text))
+    }
+}
+
+pub fn tuple_struct_pat(
+    path: ast::Path,
+    pats: impl IntoIterator<Item = ast::Pat>,
+) -> ast::TupleStructPat {
+    let pats_str = pats.into_iter().join(", ");
+    return from_text(&format!("{}({})", path, pats_str));
+
+    fn from_text(text: &str) -> ast::TupleStructPat {
+        ast_from_text(&format!("fn f({}: ())", text))
+    }
+}
+
+pub fn record_pat(path: ast::Path, pats: impl IntoIterator<Item = ast::Pat>) -> ast::RecordPat {
+    let pats_str = pats.into_iter().join(", ");
+    return from_text(&format!("{} {{ {} }}", path, pats_str));
+
+    fn from_text(text: &str) -> ast::RecordPat {
+        ast_from_text(&format!("fn f({}: ())", text))
+    }
+}
+
+/// Returns a `BindPat` if the path has just one segment, a `PathPat` otherwise.
+pub fn path_pat(path: ast::Path) -> ast::Pat {
+    return from_text(&path.to_string());
+    fn from_text(text: &str) -> ast::Pat {
+        ast_from_text(&format!("fn f({}: ())", text))
+    }
+}
+
+pub fn match_arm(pats: impl IntoIterator<Item = ast::Pat>, expr: ast::Expr) -> ast::MatchArm {
+    let pats_str = pats.into_iter().join(" | ");
+    return from_text(&format!("{} => {}", pats_str, expr));
+
+    fn from_text(text: &str) -> ast::MatchArm {
+        ast_from_text(&format!("fn f() {{ match () {{{}}} }}", text))
+    }
+}
+
+pub fn match_arm_list(arms: impl IntoIterator<Item = ast::MatchArm>) -> ast::MatchArmList {
+    let arms_str = arms
+        .into_iter()
+        .map(|arm| {
+            let needs_comma = arm.expr().map_or(true, |it| !it.is_block_like());
+            let comma = if needs_comma { "," } else { "" };
+            format!("    {}{}\n", arm.syntax(), comma)
+        })
+        .collect::<String>();
+    return from_text(&arms_str);
+
+    fn from_text(text: &str) -> ast::MatchArmList {
+        ast_from_text(&format!("fn f() {{ match () {{\n{}}} }}", text))
+    }
+}
+
+pub fn where_pred(
+    path: ast::Path,
+    bounds: impl IntoIterator<Item = ast::TypeBound>,
+) -> ast::WherePred {
+    let bounds = bounds.into_iter().join(" + ");
+    return from_text(&format!("{}: {}", path, bounds));
+
+    fn from_text(text: &str) -> ast::WherePred {
+        ast_from_text(&format!("fn f() where {} {{ }}", text))
+    }
+}
+
+pub fn where_clause(preds: impl IntoIterator<Item = ast::WherePred>) -> ast::WhereClause {
+    let preds = preds.into_iter().join(", ");
+    return from_text(preds.as_str());
+
+    fn from_text(text: &str) -> ast::WhereClause {
+        ast_from_text(&format!("fn f() where {} {{ }}", text))
+    }
+}
+
+pub fn let_stmt(pattern: ast::Pat, initializer: Option<ast::Expr>) -> ast::LetStmt {
+    let text = match initializer {
+        Some(it) => format!("let {} = {};", pattern, it),
+        None => format!("let {};", pattern),
+    };
+    ast_from_text(&format!("fn f() {{ {} }}", text))
+}
+pub fn expr_stmt(expr: ast::Expr) -> ast::ExprStmt {
+    let semi = if expr.is_block_like() { "" } else { ";" };
+    ast_from_text(&format!("fn f() {{ {}{} (); }}", expr, semi))
+}
+
+pub fn token(kind: SyntaxKind) -> SyntaxToken {
+    tokens::SOURCE_FILE
+        .tree()
+        .syntax()
+        .descendants_with_tokens()
+        .filter_map(|it| it.into_token())
+        .find(|it| it.kind() == kind)
+        .unwrap_or_else(|| panic!("unhandled token: {:?}", kind))
+}
+
+pub fn param(name: String, ty: String) -> ast::Param {
+    ast_from_text(&format!("fn f({}: {}) {{ }}", name, ty))
+}
+
+pub fn param_list(pats: impl IntoIterator<Item = ast::Param>) -> ast::ParamList {
+    let args = pats.into_iter().join(", ");
+    ast_from_text(&format!("fn f({}) {{ }}", args))
+}
+
+pub fn visibility_pub_crate() -> ast::Visibility {
+    ast_from_text("pub(crate) struct S")
+}
+
+pub fn fn_(
+    visibility: Option<ast::Visibility>,
+    fn_name: ast::Name,
+    type_params: Option<ast::GenericParamList>,
+    params: ast::ParamList,
+    body: ast::BlockExpr,
+) -> ast::Fn {
+    let type_params =
+        if let Some(type_params) = type_params { format!("<{}>", type_params) } else { "".into() };
+    let visibility = match visibility {
+        None => String::new(),
+        Some(it) => format!("{} ", it),
+    };
+    ast_from_text(&format!("{}fn {}{}{} {}", visibility, fn_name, type_params, params, body))
+}
+
+fn ast_from_text<N: AstNode>(text: &str) -> N {
+    let parse = SourceFile::parse(text);
+    let node = match parse.tree().syntax().descendants().find_map(N::cast) {
+        Some(it) => it,
+        None => {
+            panic!("Failed to make ast node `{}` from text {}", std::any::type_name::<N>(), text)
+        }
+    };
+    let node = node.syntax().clone();
+    let node = unroot(node);
+    let node = N::cast(node).unwrap();
+    assert_eq!(node.syntax().text_range().start(), 0.into());
+    node
+}
+
+fn try_ast_from_text<N: AstNode>(text: &str) -> Option<N> {
+    let parse = SourceFile::parse(text);
+    let node = parse.tree().syntax().descendants().find_map(N::cast)?;
+    let node = node.syntax().clone();
+    let node = unroot(node);
+    let node = N::cast(node).unwrap();
+    assert_eq!(node.syntax().text_range().start(), 0.into());
+    Some(node)
+}
+
+fn unroot(n: SyntaxNode) -> SyntaxNode {
+    SyntaxNode::new_root(n.green().clone())
+}
+
+pub mod tokens {
+    use once_cell::sync::Lazy;
+
+    use crate::{ast, AstNode, Parse, SourceFile, SyntaxKind::*, SyntaxToken};
+
+    pub(super) static SOURCE_FILE: Lazy<Parse<SourceFile>> =
+        Lazy::new(|| SourceFile::parse("const C: <()>::Item = (1 != 1, 2 == 2, !true)\n;"));
+
+    pub fn single_space() -> SyntaxToken {
+        SOURCE_FILE
+            .tree()
+            .syntax()
+            .descendants_with_tokens()
+            .filter_map(|it| it.into_token())
+            .find(|it| it.kind() == WHITESPACE && it.text().as_str() == " ")
+            .unwrap()
+    }
+
+    pub fn whitespace(text: &str) -> SyntaxToken {
+        assert!(text.trim().is_empty());
+        let sf = SourceFile::parse(text).ok().unwrap();
+        sf.syntax().first_child_or_token().unwrap().into_token().unwrap()
+    }
+
+    pub fn doc_comment(text: &str) -> SyntaxToken {
+        assert!(!text.trim().is_empty());
+        let sf = SourceFile::parse(text).ok().unwrap();
+        sf.syntax().first_child_or_token().unwrap().into_token().unwrap()
+    }
+
+    pub fn literal(text: &str) -> SyntaxToken {
+        assert_eq!(text.trim(), text);
+        let lit: ast::Literal = super::ast_from_text(&format!("fn f() {{ let _ = {}; }}", text));
+        lit.syntax().first_child_or_token().unwrap().into_token().unwrap()
+    }
+
+    pub fn single_newline() -> SyntaxToken {
+        SOURCE_FILE
+            .tree()
+            .syntax()
+            .descendants_with_tokens()
+            .filter_map(|it| it.into_token())
+            .find(|it| it.kind() == WHITESPACE && it.text().as_str() == "\n")
+            .unwrap()
+    }
+
+    pub struct WsBuilder(SourceFile);
+
+    impl WsBuilder {
+        pub fn new(text: &str) -> WsBuilder {
+            WsBuilder(SourceFile::parse(text).ok().unwrap())
+        }
+        pub fn ws(&self) -> SyntaxToken {
+            self.0.syntax().first_child_or_token().unwrap().into_token().unwrap()
+        }
+    }
+}

crates/syntax/src/ast/node_ext.rs

@@ -0,0 +1,485 @@
+//! Various extension methods to ast Nodes, which are hard to code-generate.
+//! Extensions for various expressions live in a sibling `expr_extensions` module.
+
+use std::fmt;
+
+use itertools::Itertools;
+use parser::SyntaxKind;
+
+use crate::{
+    ast::{self, support, AstNode, NameOwner, SyntaxNode},
+    SmolStr, SyntaxElement, SyntaxToken, T,
+};
+
+impl ast::Name {
+    pub fn text(&self) -> &SmolStr {
+        text_of_first_token(self.syntax())
+    }
+}
+
+impl ast::NameRef {
+    pub fn text(&self) -> &SmolStr {
+        text_of_first_token(self.syntax())
+    }
+
+    pub fn as_tuple_field(&self) -> Option<usize> {
+        self.text().parse().ok()
+    }
+}
+
+fn text_of_first_token(node: &SyntaxNode) -> &SmolStr {
+    node.green().children().next().and_then(|it| it.into_token()).unwrap().text()
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum AttrKind {
+    Inner,
+    Outer,
+}
+
+impl ast::Attr {
+    pub fn as_simple_atom(&self) -> Option<SmolStr> {
+        if self.eq_token().is_some() || self.token_tree().is_some() {
+            return None;
+        }
+        self.simple_name()
+    }
+
+    pub fn as_simple_call(&self) -> Option<(SmolStr, ast::TokenTree)> {
+        let tt = self.token_tree()?;
+        Some((self.simple_name()?, tt))
+    }
+
+    pub fn as_simple_key_value(&self) -> Option<(SmolStr, SmolStr)> {
+        let lit = self.literal()?;
+        let key = self.simple_name()?;
+        // FIXME: escape? raw string?
+        let value = lit.syntax().first_token()?.text().trim_matches('"').into();
+        Some((key, value))
+    }
+
+    pub fn simple_name(&self) -> Option<SmolStr> {
+        let path = self.path()?;
+        match (path.segment(), path.qualifier()) {
+            (Some(segment), None) => Some(segment.syntax().first_token()?.text().clone()),
+            _ => None,
+        }
+    }
+
+    pub fn kind(&self) -> AttrKind {
+        let first_token = self.syntax().first_token();
+        let first_token_kind = first_token.as_ref().map(SyntaxToken::kind);
+        let second_token_kind =
+            first_token.and_then(|token| token.next_token()).as_ref().map(SyntaxToken::kind);
+
+        match (first_token_kind, second_token_kind) {
+            (Some(SyntaxKind::POUND), Some(T![!])) => AttrKind::Inner,
+            _ => AttrKind::Outer,
+        }
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum PathSegmentKind {
+    Name(ast::NameRef),
+    Type { type_ref: Option<ast::Type>, trait_ref: Option<ast::PathType> },
+    SelfKw,
+    SuperKw,
+    CrateKw,
+}
+
+impl ast::PathSegment {
+    pub fn parent_path(&self) -> ast::Path {
+        self.syntax()
+            .parent()
+            .and_then(ast::Path::cast)
+            .expect("segments are always nested in paths")
+    }
+
+    pub fn kind(&self) -> Option<PathSegmentKind> {
+        let res = if let Some(name_ref) = self.name_ref() {
+            PathSegmentKind::Name(name_ref)
+        } else {
+            match self.syntax().first_child_or_token()?.kind() {
+                T![self] => PathSegmentKind::SelfKw,
+                T![super] => PathSegmentKind::SuperKw,
+                T![crate] => PathSegmentKind::CrateKw,
+                T![<] => {
+                    // <T> or <T as Trait>
+                    // T is any TypeRef, Trait has to be a PathType
+                    let mut type_refs =
+                        self.syntax().children().filter(|node| ast::Type::can_cast(node.kind()));
+                    let type_ref = type_refs.next().and_then(ast::Type::cast);
+                    let trait_ref = type_refs.next().and_then(ast::PathType::cast);
+                    PathSegmentKind::Type { type_ref, trait_ref }
+                }
+                _ => return None,
+            }
+        };
+        Some(res)
+    }
+}
+
+impl ast::Path {
+    pub fn parent_path(&self) -> Option<ast::Path> {
+        self.syntax().parent().and_then(ast::Path::cast)
+    }
+}
+
+impl ast::UseTreeList {
+    pub fn parent_use_tree(&self) -> ast::UseTree {
+        self.syntax()
+            .parent()
+            .and_then(ast::UseTree::cast)
+            .expect("UseTreeLists are always nested in UseTrees")
+    }
+}
+
+impl ast::Impl {
+    pub fn self_ty(&self) -> Option<ast::Type> {
+        match self.target() {
+            (Some(t), None) | (_, Some(t)) => Some(t),
+            _ => None,
+        }
+    }
+
+    pub fn trait_(&self) -> Option<ast::Type> {
+        match self.target() {
+            (Some(t), Some(_)) => Some(t),
+            _ => None,
+        }
+    }
+
+    fn target(&self) -> (Option<ast::Type>, Option<ast::Type>) {
+        let mut types = support::children(self.syntax());
+        let first = types.next();
+        let second = types.next();
+        (first, second)
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum StructKind {
+    Record(ast::RecordFieldList),
+    Tuple(ast::TupleFieldList),
+    Unit,
+}
+
+impl StructKind {
+    fn from_node<N: AstNode>(node: &N) -> StructKind {
+        if let Some(nfdl) = support::child::<ast::RecordFieldList>(node.syntax()) {
+            StructKind::Record(nfdl)
+        } else if let Some(pfl) = support::child::<ast::TupleFieldList>(node.syntax()) {
+            StructKind::Tuple(pfl)
+        } else {
+            StructKind::Unit
+        }
+    }
+}
+
+impl ast::Struct {
+    pub fn kind(&self) -> StructKind {
+        StructKind::from_node(self)
+    }
+}
+
+impl ast::RecordExprField {
+    pub fn for_field_name(field_name: &ast::NameRef) -> Option<ast::RecordExprField> {
+        let candidate =
+            field_name.syntax().parent().and_then(ast::RecordExprField::cast).or_else(|| {
+                field_name.syntax().ancestors().nth(4).and_then(ast::RecordExprField::cast)
+            })?;
+        if candidate.field_name().as_ref() == Some(field_name) {
+            Some(candidate)
+        } else {
+            None
+        }
+    }
+
+    /// Deals with field init shorthand
+    pub fn field_name(&self) -> Option<ast::NameRef> {
+        if let Some(name_ref) = self.name_ref() {
+            return Some(name_ref);
+        }
+        if let Some(ast::Expr::PathExpr(expr)) = self.expr() {
+            let path = expr.path()?;
+            let segment = path.segment()?;
+            let name_ref = segment.name_ref()?;
+            if path.qualifier().is_none() {
+                return Some(name_ref);
+            }
+        }
+        None
+    }
+}
+
+pub enum NameOrNameRef {
+    Name(ast::Name),
+    NameRef(ast::NameRef),
+}
+
+impl fmt::Display for NameOrNameRef {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            NameOrNameRef::Name(it) => fmt::Display::fmt(it, f),
+            NameOrNameRef::NameRef(it) => fmt::Display::fmt(it, f),
+        }
+    }
+}
+
+impl ast::RecordPatField {
+    /// Deals with field init shorthand
+    pub fn field_name(&self) -> Option<NameOrNameRef> {
+        if let Some(name_ref) = self.name_ref() {
+            return Some(NameOrNameRef::NameRef(name_ref));
+        }
+        if let Some(ast::Pat::IdentPat(pat)) = self.pat() {
+            let name = pat.name()?;
+            return Some(NameOrNameRef::Name(name));
+        }
+        None
+    }
+}
+
+impl ast::Variant {
+    pub fn parent_enum(&self) -> ast::Enum {
+        self.syntax()
+            .parent()
+            .and_then(|it| it.parent())
+            .and_then(ast::Enum::cast)
+            .expect("EnumVariants are always nested in Enums")
+    }
+    pub fn kind(&self) -> StructKind {
+        StructKind::from_node(self)
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum FieldKind {
+    Name(ast::NameRef),
+    Index(SyntaxToken),
+}
+
+impl ast::FieldExpr {
+    pub fn index_token(&self) -> Option<SyntaxToken> {
+        self.syntax
+            .children_with_tokens()
+            // FIXME: Accepting floats here to reject them in validation later
+            .find(|c| c.kind() == SyntaxKind::INT_NUMBER || c.kind() == SyntaxKind::FLOAT_NUMBER)
+            .as_ref()
+            .and_then(SyntaxElement::as_token)
+            .cloned()
+    }
+
+    pub fn field_access(&self) -> Option<FieldKind> {
+        if let Some(nr) = self.name_ref() {
+            Some(FieldKind::Name(nr))
+        } else if let Some(tok) = self.index_token() {
+            Some(FieldKind::Index(tok))
+        } else {
+            None
+        }
+    }
+}
+
+pub struct SlicePatComponents {
+    pub prefix: Vec<ast::Pat>,
+    pub slice: Option<ast::Pat>,
+    pub suffix: Vec<ast::Pat>,
+}
+
+impl ast::SlicePat {
+    pub fn components(&self) -> SlicePatComponents {
+        let mut args = self.pats().peekable();
+        let prefix = args
+            .peeking_take_while(|p| match p {
+                ast::Pat::RestPat(_) => false,
+                ast::Pat::IdentPat(bp) => match bp.pat() {
+                    Some(ast::Pat::RestPat(_)) => false,
+                    _ => true,
+                },
+                ast::Pat::RefPat(rp) => match rp.pat() {
+                    Some(ast::Pat::RestPat(_)) => false,
+                    Some(ast::Pat::IdentPat(bp)) => match bp.pat() {
+                        Some(ast::Pat::RestPat(_)) => false,
+                        _ => true,
+                    },
+                    _ => true,
+                },
+                _ => true,
+            })
+            .collect();
+        let slice = args.next();
+        let suffix = args.collect();
+
+        SlicePatComponents { prefix, slice, suffix }
+    }
+}
+
+#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
+pub enum SelfParamKind {
+    /// self
+    Owned,
+    /// &self
+    Ref,
+    /// &mut self
+    MutRef,
+}
+
+impl ast::SelfParam {
+    pub fn kind(&self) -> SelfParamKind {
+        if self.amp_token().is_some() {
+            if self.mut_token().is_some() {
+                SelfParamKind::MutRef
+            } else {
+                SelfParamKind::Ref
+            }
+        } else {
+            SelfParamKind::Owned
+        }
+    }
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Hash)]
+pub enum TypeBoundKind {
+    /// Trait
+    PathType(ast::PathType),
+    /// for<'a> ...
+    ForType(ast::ForType),
+    /// 'a
+    Lifetime(SyntaxToken),
+}
+
+impl ast::TypeBound {
+    pub fn kind(&self) -> TypeBoundKind {
+        if let Some(path_type) = support::children(self.syntax()).next() {
+            TypeBoundKind::PathType(path_type)
+        } else if let Some(for_type) = support::children(self.syntax()).next() {
+            TypeBoundKind::ForType(for_type)
+        } else if let Some(lifetime) = self.lifetime_token() {
+            TypeBoundKind::Lifetime(lifetime)
+        } else {
+            unreachable!()
+        }
+    }
+}
+
+pub enum VisibilityKind {
+    In(ast::Path),
+    PubCrate,
+    PubSuper,
+    PubSelf,
+    Pub,
+}
+
+impl ast::Visibility {
+    pub fn kind(&self) -> VisibilityKind {
+        if let Some(path) = support::children(self.syntax()).next() {
+            VisibilityKind::In(path)
+        } else if self.crate_token().is_some() {
+            VisibilityKind::PubCrate
+        } else if self.super_token().is_some() {
+            VisibilityKind::PubSuper
+        } else if self.self_token().is_some() {
+            VisibilityKind::PubSelf
+        } else {
+            VisibilityKind::Pub
+        }
+    }
+}
+
+impl ast::MacroCall {
+    pub fn is_macro_rules(&self) -> Option<ast::Name> {
+        let name_ref = self.path()?.segment()?.name_ref()?;
+        if name_ref.text() == "macro_rules" {
+            self.name()
+        } else {
+            None
+        }
+    }
+
+    pub fn is_bang(&self) -> bool {
+        self.is_macro_rules().is_none()
+    }
+}
+
+impl ast::LifetimeParam {
+    pub fn lifetime_bounds(&self) -> impl Iterator<Item = SyntaxToken> {
+        self.syntax()
+            .children_with_tokens()
+            .filter_map(|it| it.into_token())
+            .skip_while(|x| x.kind() != T![:])
+            .filter(|it| it.kind() == T![lifetime])
+    }
+}
+
+impl ast::RangePat {
+    pub fn start(&self) -> Option<ast::Pat> {
+        self.syntax()
+            .children_with_tokens()
+            .take_while(|it| !(it.kind() == T![..] || it.kind() == T![..=]))
+            .filter_map(|it| it.into_node())
+            .find_map(ast::Pat::cast)
+    }
+
+    pub fn end(&self) -> Option<ast::Pat> {
+        self.syntax()
+            .children_with_tokens()
+            .skip_while(|it| !(it.kind() == T![..] || it.kind() == T![..=]))
+            .filter_map(|it| it.into_node())
+            .find_map(ast::Pat::cast)
+    }
+}
+
+impl ast::TokenTree {
+    pub fn left_delimiter_token(&self) -> Option<SyntaxToken> {
+        self.syntax()
+            .first_child_or_token()?
+            .into_token()
+            .filter(|it| matches!(it.kind(), T!['{'] | T!['('] | T!['[']))
+    }
+
+    pub fn right_delimiter_token(&self) -> Option<SyntaxToken> {
+        self.syntax()
+            .last_child_or_token()?
+            .into_token()
+            .filter(|it| matches!(it.kind(), T!['}'] | T![')'] | T![']']))
+    }
+}
+
+impl ast::GenericParamList {
+    pub fn lifetime_params(&self) -> impl Iterator<Item = ast::LifetimeParam> {
+        self.generic_params().filter_map(|param| match param {
+            ast::GenericParam::LifetimeParam(it) => Some(it),
+            ast::GenericParam::TypeParam(_) | ast::GenericParam::ConstParam(_) => None,
+        })
+    }
+    pub fn type_params(&self) -> impl Iterator<Item = ast::TypeParam> {
+        self.generic_params().filter_map(|param| match param {
+            ast::GenericParam::TypeParam(it) => Some(it),
+            ast::GenericParam::LifetimeParam(_) | ast::GenericParam::ConstParam(_) => None,
+        })
+    }
+    pub fn const_params(&self) -> impl Iterator<Item = ast::ConstParam> {
+        self.generic_params().filter_map(|param| match param {
+            ast::GenericParam::ConstParam(it) => Some(it),
+            ast::GenericParam::TypeParam(_) | ast::GenericParam::LifetimeParam(_) => None,
+        })
+    }
+}
+
+impl ast::DocCommentsOwner for ast::SourceFile {}
+impl ast::DocCommentsOwner for ast::Fn {}
+impl ast::DocCommentsOwner for ast::Struct {}
+impl ast::DocCommentsOwner for ast::Union {}
+impl ast::DocCommentsOwner for ast::RecordField {}
+impl ast::DocCommentsOwner for ast::TupleField {}
+impl ast::DocCommentsOwner for ast::Enum {}
+impl ast::DocCommentsOwner for ast::Variant {}
+impl ast::DocCommentsOwner for ast::Trait {}
+impl ast::DocCommentsOwner for ast::Module {}
+impl ast::DocCommentsOwner for ast::Static {}
+impl ast::DocCommentsOwner for ast::Const {}
+impl ast::DocCommentsOwner for ast::TypeAlias {}
+impl ast::DocCommentsOwner for ast::Impl {}
+impl ast::DocCommentsOwner for ast::MacroCall {}

crates/syntax/src/ast/token_ext.rs

@@ -0,0 +1,538 @@
+//! There are many AstNodes, but only a few tokens, so we hand-write them here.
+
+use std::{
+    borrow::Cow,
+    convert::{TryFrom, TryInto},
+};
+
+use rustc_lexer::unescape::{unescape_literal, Mode};
+
+use crate::{
+    ast::{AstToken, Comment, RawString, String, Whitespace},
+    TextRange, TextSize,
+};
+
+impl Comment {
+    pub fn kind(&self) -> CommentKind {
+        kind_by_prefix(self.text())
+    }
+
+    pub fn prefix(&self) -> &'static str {
+        for (prefix, k) in COMMENT_PREFIX_TO_KIND.iter() {
+            if *k == self.kind() && self.text().starts_with(prefix) {
+                return prefix;
+            }
+        }
+        unreachable!()
+    }
+}
+
+#[derive(Debug, PartialEq, Eq, Clone, Copy)]
+pub struct CommentKind {
+    pub shape: CommentShape,
+    pub doc: Option<CommentPlacement>,
+}
+
+#[derive(Debug, PartialEq, Eq, Clone, Copy)]
+pub enum CommentShape {
+    Line,
+    Block,
+}
+
+impl CommentShape {
+    pub fn is_line(self) -> bool {
+        self == CommentShape::Line
+    }
+
+    pub fn is_block(self) -> bool {
+        self == CommentShape::Block
+    }
+}
+
+#[derive(Debug, PartialEq, Eq, Clone, Copy)]
+pub enum CommentPlacement {
+    Inner,
+    Outer,
+}
+
+const COMMENT_PREFIX_TO_KIND: &[(&str, CommentKind)] = {
+    use {CommentPlacement::*, CommentShape::*};
+    &[
+        ("////", CommentKind { shape: Line, doc: None }),
+        ("///", CommentKind { shape: Line, doc: Some(Outer) }),
+        ("//!", CommentKind { shape: Line, doc: Some(Inner) }),
+        ("/**", CommentKind { shape: Block, doc: Some(Outer) }),
+        ("/*!", CommentKind { shape: Block, doc: Some(Inner) }),
+        ("//", CommentKind { shape: Line, doc: None }),
+        ("/*", CommentKind { shape: Block, doc: None }),
+    ]
+};
+
+fn kind_by_prefix(text: &str) -> CommentKind {
+    if text == "/**/" {
+        return CommentKind { shape: CommentShape::Block, doc: None };
+    }
+    for (prefix, kind) in COMMENT_PREFIX_TO_KIND.iter() {
+        if text.starts_with(prefix) {
+            return *kind;
+        }
+    }
+    panic!("bad comment text: {:?}", text)
+}
+
+impl Whitespace {
+    pub fn spans_multiple_lines(&self) -> bool {
+        let text = self.text();
+        text.find('\n').map_or(false, |idx| text[idx + 1..].contains('\n'))
+    }
+}
+
+pub struct QuoteOffsets {
+    pub quotes: (TextRange, TextRange),
+    pub contents: TextRange,
+}
+
+impl QuoteOffsets {
+    fn new(literal: &str) -> Option<QuoteOffsets> {
+        let left_quote = literal.find('"')?;
+        let right_quote = literal.rfind('"')?;
+        if left_quote == right_quote {
+            // `literal` only contains one quote
+            return None;
+        }
+
+        let start = TextSize::from(0);
+        let left_quote = TextSize::try_from(left_quote).unwrap() + TextSize::of('"');
+        let right_quote = TextSize::try_from(right_quote).unwrap();
+        let end = TextSize::of(literal);
+
+        let res = QuoteOffsets {
+            quotes: (TextRange::new(start, left_quote), TextRange::new(right_quote, end)),
+            contents: TextRange::new(left_quote, right_quote),
+        };
+        Some(res)
+    }
+}
+
+pub trait HasQuotes: AstToken {
+    fn quote_offsets(&self) -> Option<QuoteOffsets> {
+        let text = self.text().as_str();
+        let offsets = QuoteOffsets::new(text)?;
+        let o = self.syntax().text_range().start();
+        let offsets = QuoteOffsets {
+            quotes: (offsets.quotes.0 + o, offsets.quotes.1 + o),
+            contents: offsets.contents + o,
+        };
+        Some(offsets)
+    }
+    fn open_quote_text_range(&self) -> Option<TextRange> {
+        self.quote_offsets().map(|it| it.quotes.0)
+    }
+
+    fn close_quote_text_range(&self) -> Option<TextRange> {
+        self.quote_offsets().map(|it| it.quotes.1)
+    }
+
+    fn text_range_between_quotes(&self) -> Option<TextRange> {
+        self.quote_offsets().map(|it| it.contents)
+    }
+}
+
+impl HasQuotes for String {}
+impl HasQuotes for RawString {}
+
+pub trait HasStringValue: HasQuotes {
+    fn value(&self) -> Option<Cow<'_, str>>;
+}
+
+impl HasStringValue for String {
+    fn value(&self) -> Option<Cow<'_, str>> {
+        let text = self.text().as_str();
+        let text = &text[self.text_range_between_quotes()? - self.syntax().text_range().start()];
+
+        let mut buf = std::string::String::with_capacity(text.len());
+        let mut has_error = false;
+        unescape_literal(text, Mode::Str, &mut |_, unescaped_char| match unescaped_char {
+            Ok(c) => buf.push(c),
+            Err(_) => has_error = true,
+        });
+
+        if has_error {
+            return None;
+        }
+        // FIXME: don't actually allocate for borrowed case
+        let res = if buf == text { Cow::Borrowed(text) } else { Cow::Owned(buf) };
+        Some(res)
+    }
+}
+
+impl HasStringValue for RawString {
+    fn value(&self) -> Option<Cow<'_, str>> {
+        let text = self.text().as_str();
+        let text = &text[self.text_range_between_quotes()? - self.syntax().text_range().start()];
+        Some(Cow::Borrowed(text))
+    }
+}
+
+impl RawString {
+    pub fn map_range_up(&self, range: TextRange) -> Option<TextRange> {
+        let contents_range = self.text_range_between_quotes()?;
+        assert!(TextRange::up_to(contents_range.len()).contains_range(range));
+        Some(range + contents_range.start())
+    }
+}
+
+#[derive(Debug)]
+pub enum FormatSpecifier {
+    Open,
+    Close,
+    Integer,
+    Identifier,
+    Colon,
+    Fill,
+    Align,
+    Sign,
+    NumberSign,
+    Zero,
+    DollarSign,
+    Dot,
+    Asterisk,
+    QuestionMark,
+}
+
+pub trait HasFormatSpecifier: AstToken {
+    fn char_ranges(
+        &self,
+    ) -> Option<Vec<(TextRange, Result<char, rustc_lexer::unescape::EscapeError>)>>;
+
+    fn lex_format_specifier<F>(&self, mut callback: F)
+    where
+        F: FnMut(TextRange, FormatSpecifier),
+    {
+        let char_ranges = if let Some(char_ranges) = self.char_ranges() {
+            char_ranges
+        } else {
+            return;
+        };
+        let mut chars = char_ranges.iter().peekable();
+
+        while let Some((range, first_char)) = chars.next() {
+            match first_char {
+                Ok('{') => {
+                    // Format specifier, see syntax at https://doc.rust-lang.org/std/fmt/index.html#syntax
+                    if let Some((_, Ok('{'))) = chars.peek() {
+                        // Escaped format specifier, `{{`
+                        chars.next();
+                        continue;
+                    }
+
+                    callback(*range, FormatSpecifier::Open);
+
+                    // check for integer/identifier
+                    match chars
+                        .peek()
+                        .and_then(|next| next.1.as_ref().ok())
+                        .copied()
+                        .unwrap_or_default()
+                    {
+                        '0'..='9' => {
+                            // integer
+                            read_integer(&mut chars, &mut callback);
+                        }
+                        c if c == '_' || c.is_alphabetic() => {
+                            // identifier
+                            read_identifier(&mut chars, &mut callback);
+                        }
+                        _ => {}
+                    }
+
+                    if let Some((_, Ok(':'))) = chars.peek() {
+                        skip_char_and_emit(&mut chars, FormatSpecifier::Colon, &mut callback);
+
+                        // check for fill/align
+                        let mut cloned = chars.clone().take(2);
+                        let first = cloned
+                            .next()
+                            .and_then(|next| next.1.as_ref().ok())
+                            .copied()
+                            .unwrap_or_default();
+                        let second = cloned
+                            .next()
+                            .and_then(|next| next.1.as_ref().ok())
+                            .copied()
+                            .unwrap_or_default();
+                        match second {
+                            '<' | '^' | '>' => {
+                                // alignment specifier, first char specifies fillment
+                                skip_char_and_emit(
+                                    &mut chars,
+                                    FormatSpecifier::Fill,
+                                    &mut callback,
+                                );
+                                skip_char_and_emit(
+                                    &mut chars,
+                                    FormatSpecifier::Align,
+                                    &mut callback,
+                                );
+                            }
+                            _ => match first {
+                                '<' | '^' | '>' => {
+                                    skip_char_and_emit(
+                                        &mut chars,
+                                        FormatSpecifier::Align,
+                                        &mut callback,
+                                    );
+                                }
+                                _ => {}
+                            },
+                        }
+
+                        // check for sign
+                        match chars
+                            .peek()
+                            .and_then(|next| next.1.as_ref().ok())
+                            .copied()
+                            .unwrap_or_default()
+                        {
+                            '+' | '-' => {
+                                skip_char_and_emit(
+                                    &mut chars,
+                                    FormatSpecifier::Sign,
+                                    &mut callback,
+                                );
+                            }
+                            _ => {}
+                        }
+
+                        // check for `#`
+                        if let Some((_, Ok('#'))) = chars.peek() {
+                            skip_char_and_emit(
+                                &mut chars,
+                                FormatSpecifier::NumberSign,
+                                &mut callback,
+                            );
+                        }
+
+                        // check for `0`
+                        let mut cloned = chars.clone().take(2);
+                        let first = cloned.next().and_then(|next| next.1.as_ref().ok()).copied();
+                        let second = cloned.next().and_then(|next| next.1.as_ref().ok()).copied();
+
+                        if first == Some('0') && second != Some('$') {
+                            skip_char_and_emit(&mut chars, FormatSpecifier::Zero, &mut callback);
+                        }
+
+                        // width
+                        match chars
+                            .peek()
+                            .and_then(|next| next.1.as_ref().ok())
+                            .copied()
+                            .unwrap_or_default()
+                        {
+                            '0'..='9' => {
+                                read_integer(&mut chars, &mut callback);
+                                if let Some((_, Ok('$'))) = chars.peek() {
+                                    skip_char_and_emit(
+                                        &mut chars,
+                                        FormatSpecifier::DollarSign,
+                                        &mut callback,
+                                    );
+                                }
+                            }
+                            c if c == '_' || c.is_alphabetic() => {
+                                read_identifier(&mut chars, &mut callback);
+                                // can be either width (indicated by dollar sign, or type in which case
+                                // the next sign has to be `}`)
+                                let next =
+                                    chars.peek().and_then(|next| next.1.as_ref().ok()).copied();
+                                match next {
+                                    Some('$') => skip_char_and_emit(
+                                        &mut chars,
+                                        FormatSpecifier::DollarSign,
+                                        &mut callback,
+                                    ),
+                                    Some('}') => {
+                                        skip_char_and_emit(
+                                            &mut chars,
+                                            FormatSpecifier::Close,
+                                            &mut callback,
+                                        );
+                                        continue;
+                                    }
+                                    _ => continue,
+                                };
+                            }
+                            _ => {}
+                        }
+
+                        // precision
+                        if let Some((_, Ok('.'))) = chars.peek() {
+                            skip_char_and_emit(&mut chars, FormatSpecifier::Dot, &mut callback);
+
+                            match chars
+                                .peek()
+                                .and_then(|next| next.1.as_ref().ok())
+                                .copied()
+                                .unwrap_or_default()
+                            {
+                                '*' => {
+                                    skip_char_and_emit(
+                                        &mut chars,
+                                        FormatSpecifier::Asterisk,
+                                        &mut callback,
+                                    );
+                                }
+                                '0'..='9' => {
+                                    read_integer(&mut chars, &mut callback);
+                                    if let Some((_, Ok('$'))) = chars.peek() {
+                                        skip_char_and_emit(
+                                            &mut chars,
+                                            FormatSpecifier::DollarSign,
+                                            &mut callback,
+                                        );
+                                    }
+                                }
+                                c if c == '_' || c.is_alphabetic() => {
+                                    read_identifier(&mut chars, &mut callback);
+                                    if chars.peek().and_then(|next| next.1.as_ref().ok()).copied()
+                                        != Some('$')
+                                    {
+                                        continue;
+                                    }
+                                    skip_char_and_emit(
+                                        &mut chars,
+                                        FormatSpecifier::DollarSign,
+                                        &mut callback,
+                                    );
+                                }
+                                _ => {
+                                    continue;
+                                }
+                            }
+                        }
+
+                        // type
+                        match chars
+                            .peek()
+                            .and_then(|next| next.1.as_ref().ok())
+                            .copied()
+                            .unwrap_or_default()
+                        {
+                            '?' => {
+                                skip_char_and_emit(
+                                    &mut chars,
+                                    FormatSpecifier::QuestionMark,
+                                    &mut callback,
+                                );
+                            }
+                            c if c == '_' || c.is_alphabetic() => {
+                                read_identifier(&mut chars, &mut callback);
+                            }
+                            _ => {}
+                        }
+                    }
+
+                    if let Some((_, Ok('}'))) = chars.peek() {
+                        skip_char_and_emit(&mut chars, FormatSpecifier::Close, &mut callback);
+                    } else {
+                        continue;
+                    }
+                }
+                _ => {
+                    while let Some((_, Ok(next_char))) = chars.peek() {
+                        match next_char {
+                            '{' => break,
+                            _ => {}
+                        }
+                        chars.next();
+                    }
+                }
+            };
+        }
+
+        fn skip_char_and_emit<'a, I, F>(
+            chars: &mut std::iter::Peekable<I>,
+            emit: FormatSpecifier,
+            callback: &mut F,
+        ) where
+            I: Iterator<Item = &'a (TextRange, Result<char, rustc_lexer::unescape::EscapeError>)>,
+            F: FnMut(TextRange, FormatSpecifier),
+        {
+            let (range, _) = chars.next().unwrap();
+            callback(*range, emit);
+        }
+
+        fn read_integer<'a, I, F>(chars: &mut std::iter::Peekable<I>, callback: &mut F)
+        where
+            I: Iterator<Item = &'a (TextRange, Result<char, rustc_lexer::unescape::EscapeError>)>,
+            F: FnMut(TextRange, FormatSpecifier),
+        {
+            let (mut range, c) = chars.next().unwrap();
+            assert!(c.as_ref().unwrap().is_ascii_digit());
+            while let Some((r, Ok(next_char))) = chars.peek() {
+                if next_char.is_ascii_digit() {
+                    chars.next();
+                    range = range.cover(*r);
+                } else {
+                    break;
+                }
+            }
+            callback(range, FormatSpecifier::Integer);
+        }
+
+        fn read_identifier<'a, I, F>(chars: &mut std::iter::Peekable<I>, callback: &mut F)
+        where
+            I: Iterator<Item = &'a (TextRange, Result<char, rustc_lexer::unescape::EscapeError>)>,
+            F: FnMut(TextRange, FormatSpecifier),
+        {
+            let (mut range, c) = chars.next().unwrap();
+            assert!(c.as_ref().unwrap().is_alphabetic() || *c.as_ref().unwrap() == '_');
+            while let Some((r, Ok(next_char))) = chars.peek() {
+                if *next_char == '_' || next_char.is_ascii_digit() || next_char.is_alphabetic() {
+                    chars.next();
+                    range = range.cover(*r);
+                } else {
+                    break;
+                }
+            }
+            callback(range, FormatSpecifier::Identifier);
+        }
+    }
+}
+
+impl HasFormatSpecifier for String {
+    fn char_ranges(
+        &self,
+    ) -> Option<Vec<(TextRange, Result<char, rustc_lexer::unescape::EscapeError>)>> {
+        let text = self.text().as_str();
+        let text = &text[self.text_range_between_quotes()? - self.syntax().text_range().start()];
+        let offset = self.text_range_between_quotes()?.start() - self.syntax().text_range().start();
+
+        let mut res = Vec::with_capacity(text.len());
+        unescape_literal(text, Mode::Str, &mut |range, unescaped_char| {
+            res.push((
+                TextRange::new(range.start.try_into().unwrap(), range.end.try_into().unwrap())
+                    + offset,
+                unescaped_char,
+            ))
+        });
+
+        Some(res)
+    }
+}
+
+impl HasFormatSpecifier for RawString {
+    fn char_ranges(
+        &self,
+    ) -> Option<Vec<(TextRange, Result<char, rustc_lexer::unescape::EscapeError>)>> {
+        let text = self.text().as_str();
+        let text = &text[self.text_range_between_quotes()? - self.syntax().text_range().start()];
+        let offset = self.text_range_between_quotes()?.start() - self.syntax().text_range().start();
+
+        let mut res = Vec::with_capacity(text.len());
+        for (idx, c) in text.char_indices() {
+            res.push((TextRange::at(idx.try_into().unwrap(), TextSize::of(c)) + offset, Ok(c)));
+        }
+        Some(res)
+    }
+}

crates/syntax/src/ast/traits.rs

@@ -0,0 +1,141 @@
+//! Various traits that are implemented by ast nodes.
+//!
+//! The implementations are usually trivial, and live in generated.rs
+use itertools::Itertools;
+
+use crate::{
+    ast::{self, support, AstChildren, AstNode, AstToken},
+    syntax_node::SyntaxElementChildren,
+    SyntaxToken, T,
+};
+
+pub trait NameOwner: AstNode {
+    fn name(&self) -> Option<ast::Name> {
+        support::child(self.syntax())
+    }
+}
+
+pub trait VisibilityOwner: AstNode {
+    fn visibility(&self) -> Option<ast::Visibility> {
+        support::child(self.syntax())
+    }
+}
+
+pub trait LoopBodyOwner: AstNode {
+    fn loop_body(&self) -> Option<ast::BlockExpr> {
+        support::child(self.syntax())
+    }
+
+    fn label(&self) -> Option<ast::Label> {
+        support::child(self.syntax())
+    }
+}
+
+pub trait ArgListOwner: AstNode {
+    fn arg_list(&self) -> Option<ast::ArgList> {
+        support::child(self.syntax())
+    }
+}
+
+pub trait ModuleItemOwner: AstNode {
+    fn items(&self) -> AstChildren<ast::Item> {
+        support::children(self.syntax())
+    }
+}
+
+pub trait GenericParamsOwner: AstNode {
+    fn generic_param_list(&self) -> Option<ast::GenericParamList> {
+        support::child(self.syntax())
+    }
+
+    fn where_clause(&self) -> Option<ast::WhereClause> {
+        support::child(self.syntax())
+    }
+}
+
+pub trait TypeBoundsOwner: AstNode {
+    fn type_bound_list(&self) -> Option<ast::TypeBoundList> {
+        support::child(self.syntax())
+    }
+
+    fn colon_token(&self) -> Option<SyntaxToken> {
+        support::token(self.syntax(), T![:])
+    }
+}
+
+pub trait AttrsOwner: AstNode {
+    fn attrs(&self) -> AstChildren<ast::Attr> {
+        support::children(self.syntax())
+    }
+    fn has_atom_attr(&self, atom: &str) -> bool {
+        self.attrs().filter_map(|x| x.as_simple_atom()).any(|x| x == atom)
+    }
+}
+
+pub trait DocCommentsOwner: AstNode {
+    fn doc_comments(&self) -> CommentIter {
+        CommentIter { iter: self.syntax().children_with_tokens() }
+    }
+
+    fn doc_comment_text(&self) -> Option<String> {
+        self.doc_comments().doc_comment_text()
+    }
+}
+
+impl CommentIter {
+    pub fn from_syntax_node(syntax_node: &ast::SyntaxNode) -> CommentIter {
+        CommentIter { iter: syntax_node.children_with_tokens() }
+    }
+
+    /// Returns the textual content of a doc comment block as a single string.
+    /// That is, strips leading `///` (+ optional 1 character of whitespace),
+    /// trailing `*/`, trailing whitespace and then joins the lines.
+    pub fn doc_comment_text(self) -> Option<String> {
+        let mut has_comments = false;
+        let docs = self
+            .filter(|comment| comment.kind().doc.is_some())
+            .map(|comment| {
+                has_comments = true;
+                let prefix_len = comment.prefix().len();
+
+                let line: &str = comment.text().as_str();
+
+                // Determine if the prefix or prefix + 1 char is stripped
+                let pos =
+                    if let Some(ws) = line.chars().nth(prefix_len).filter(|c| c.is_whitespace()) {
+                        prefix_len + ws.len_utf8()
+                    } else {
+                        prefix_len
+                    };
+
+                let end = if comment.kind().shape.is_block() && line.ends_with("*/") {
+                    line.len() - 2
+                } else {
+                    line.len()
+                };
+
+                // Note that we do not trim the end of the line here
+                // since whitespace can have special meaning at the end
+                // of a line in markdown.
+                line[pos..end].to_owned()
+            })
+            .join("\n");
+
+        if has_comments {
+            Some(docs)
+        } else {
+            None
+        }
+    }
+}
+
+pub struct CommentIter {
+    iter: SyntaxElementChildren,
+}
+
+impl Iterator for CommentIter {
+    type Item = ast::Comment;
+    fn next(&mut self) -> Option<ast::Comment> {
+        self.iter.by_ref().find_map(|el| el.into_token().and_then(ast::Comment::cast))
+    }
+}