Rename ra_parser -> parser

crates/parser/src/event.rs

@@ -0,0 +1,130 @@
+//! This module provides a way to construct a `File`.
+//! It is intended to be completely decoupled from the
+//! parser, so as to allow to evolve the tree representation
+//! and the parser algorithm independently.
+//!
+//! The `TreeSink` trait is the bridge between the parser and the
+//! tree builder: the parser produces a stream of events like
+//! `start node`, `finish node`, and `FileBuilder` converts
+//! this stream to a real tree.
+use std::mem;
+
+use crate::{
+    ParseError,
+    SyntaxKind::{self, *},
+    TreeSink,
+};
+
+/// `Parser` produces a flat list of `Event`s.
+/// They are converted to a tree-structure in
+/// a separate pass, via `TreeBuilder`.
+#[derive(Debug)]
+pub(crate) enum Event {
+    /// This event signifies the start of the node.
+    /// It should be either abandoned (in which case the
+    /// `kind` is `TOMBSTONE`, and the event is ignored),
+    /// or completed via a `Finish` event.
+    ///
+    /// All tokens between a `Start` and a `Finish` would
+    /// become the children of the respective node.
+    ///
+    /// For left-recursive syntactic constructs, the parser produces
+    /// a child node before it sees a parent. `forward_parent`
+    /// saves the position of current event's parent.
+    ///
+    /// Consider this path
+    ///
+    /// foo::bar
+    ///
+    /// The events for it would look like this:
+    ///
+    ///
+    /// START(PATH) IDENT('foo') FINISH START(PATH) T![::] IDENT('bar') FINISH
+    ///       |                          /\
+    ///       |                          |
+    ///       +------forward-parent------+
+    ///
+    /// And the tree would look like this
+    ///
+    ///    +--PATH---------+
+    ///    |   |           |
+    ///    |   |           |
+    ///    |  '::'       'bar'
+    ///    |
+    ///   PATH
+    ///    |
+    ///   'foo'
+    ///
+    /// See also `CompletedMarker::precede`.
+    Start {
+        kind: SyntaxKind,
+        forward_parent: Option<u32>,
+    },
+
+    /// Complete the previous `Start` event
+    Finish,
+
+    /// Produce a single leaf-element.
+    /// `n_raw_tokens` is used to glue complex contextual tokens.
+    /// For example, lexer tokenizes `>>` as `>`, `>`, and
+    /// `n_raw_tokens = 2` is used to produced a single `>>`.
+    Token {
+        kind: SyntaxKind,
+        n_raw_tokens: u8,
+    },
+
+    Error {
+        msg: ParseError,
+    },
+}
+
+impl Event {
+    pub(crate) fn tombstone() -> Self {
+        Event::Start { kind: TOMBSTONE, forward_parent: None }
+    }
+}
+
+/// Generate the syntax tree with the control of events.
+pub(super) fn process(sink: &mut dyn TreeSink, mut events: Vec<Event>) {
+    let mut forward_parents = Vec::new();
+
+    for i in 0..events.len() {
+        match mem::replace(&mut events[i], Event::tombstone()) {
+            Event::Start { kind: TOMBSTONE, .. } => (),
+
+            Event::Start { kind, forward_parent } => {
+                // For events[A, B, C], B is A's forward_parent, C is B's forward_parent,
+                // in the normal control flow, the parent-child relation: `A -> B -> C`,
+                // while with the magic forward_parent, it writes: `C <- B <- A`.
+
+                // append `A` into parents.
+                forward_parents.push(kind);
+                let mut idx = i;
+                let mut fp = forward_parent;
+                while let Some(fwd) = fp {
+                    idx += fwd as usize;
+                    // append `A`'s forward_parent `B`
+                    fp = match mem::replace(&mut events[idx], Event::tombstone()) {
+                        Event::Start { kind, forward_parent } => {
+                            if kind != TOMBSTONE {
+                                forward_parents.push(kind);
+                            }
+                            forward_parent
+                        }
+                        _ => unreachable!(),
+                    };
+                    // append `B`'s forward_parent `C` in the next stage.
+                }
+
+                for kind in forward_parents.drain(..).rev() {
+                    sink.start_node(kind);
+                }
+            }
+            Event::Finish => sink.finish_node(),
+            Event::Token { kind, n_raw_tokens } => {
+                sink.token(kind, n_raw_tokens);
+            }
+            Event::Error { msg } => sink.error(msg),
+        }
+    }
+}

crates/parser/src/grammar.rs

@@ -0,0 +1,293 @@
+//! This is the actual "grammar" of the Rust language.
+//!
+//! Each function in this module and its children corresponds
+//! to a production of the formal grammar. Submodules roughly
+//! correspond to different *areas* of the grammar. By convention,
+//! each submodule starts with `use super::*` import and exports
+//! "public" productions via `pub(super)`.
+//!
+//! See docs for `Parser` to learn about API, available to the grammar,
+//! and see docs for `Event` to learn how this actually manages to
+//! produce parse trees.
+//!
+//! Code in this module also contains inline tests, which start with
+//! `// test name-of-the-test` comment and look like this:
+//!
+//! ```
+//! // test function_with_zero_parameters
+//! // fn foo() {}
+//! ```
+//!
+//! After adding a new inline-test, run `cargo xtask codegen` to
+//! extract it as a standalone text-fixture into
+//! `crates/ra_syntax/test_data/parser/`, and run `cargo test` once to
+//! create the "gold" value.
+//!
+//! Coding convention: rules like `where_clause` always produce either a
+//! node or an error, rules like `opt_where_clause` may produce nothing.
+//! Non-opt rules typically start with `assert!(p.at(FIRST_TOKEN))`, the
+//! caller is responsible for branching on the first token.
+mod attributes;
+mod expressions;
+mod items;
+mod params;
+mod paths;
+mod patterns;
+mod type_args;
+mod type_params;
+mod types;
+
+use crate::{
+    parser::{CompletedMarker, Marker, Parser},
+    SyntaxKind::{self, *},
+    TokenSet,
+};
+
+pub(crate) fn root(p: &mut Parser) {
+    let m = p.start();
+    p.eat(SHEBANG);
+    items::mod_contents(p, false);
+    m.complete(p, SOURCE_FILE);
+}
+
+/// Various pieces of syntax that can be parsed by macros by example
+pub(crate) mod fragments {
+    use super::*;
+
+    pub(crate) use super::{
+        expressions::block_expr, paths::type_path as path, patterns::pattern, types::type_,
+    };
+
+    pub(crate) fn expr(p: &mut Parser) {
+        let _ = expressions::expr(p);
+    }
+
+    pub(crate) fn stmt(p: &mut Parser) {
+        expressions::stmt(p, expressions::StmtWithSemi::No)
+    }
+
+    pub(crate) fn opt_visibility(p: &mut Parser) {
+        let _ = super::opt_visibility(p);
+    }
+
+    // Parse a meta item , which excluded [], e.g : #[ MetaItem ]
+    pub(crate) fn meta_item(p: &mut Parser) {
+        fn is_delimiter(p: &mut Parser) -> bool {
+            matches!(p.current(), T!['{'] | T!['('] | T!['['])
+        }
+
+        if is_delimiter(p) {
+            items::token_tree(p);
+            return;
+        }
+
+        let m = p.start();
+        while !p.at(EOF) {
+            if is_delimiter(p) {
+                items::token_tree(p);
+                break;
+            } else {
+                // https://doc.rust-lang.org/reference/attributes.html
+                // https://doc.rust-lang.org/reference/paths.html#simple-paths
+                // The start of an meta must be a simple path
+                match p.current() {
+                    IDENT | T![::] | T![super] | T![self] | T![crate] => p.bump_any(),
+                    T![=] => {
+                        p.bump_any();
+                        match p.current() {
+                            c if c.is_literal() => p.bump_any(),
+                            T![true] | T![false] => p.bump_any(),
+                            _ => {}
+                        }
+                        break;
+                    }
+                    _ => break,
+                }
+            }
+        }
+
+        m.complete(p, TOKEN_TREE);
+    }
+
+    pub(crate) fn item(p: &mut Parser) {
+        items::item_or_macro(p, true)
+    }
+
+    pub(crate) fn macro_items(p: &mut Parser) {
+        let m = p.start();
+        items::mod_contents(p, false);
+        m.complete(p, MACRO_ITEMS);
+    }
+
+    pub(crate) fn macro_stmts(p: &mut Parser) {
+        let m = p.start();
+
+        while !p.at(EOF) {
+            if p.at(T![;]) {
+                p.bump(T![;]);
+                continue;
+            }
+
+            expressions::stmt(p, expressions::StmtWithSemi::Optional);
+        }
+
+        m.complete(p, MACRO_STMTS);
+    }
+}
+
+pub(crate) fn reparser(
+    node: SyntaxKind,
+    first_child: Option<SyntaxKind>,
+    parent: Option<SyntaxKind>,
+) -> Option<fn(&mut Parser)> {
+    let res = match node {
+        BLOCK_EXPR => expressions::block_expr,
+        RECORD_FIELD_LIST => items::record_field_def_list,
+        RECORD_EXPR_FIELD_LIST => items::record_field_list,
+        VARIANT_LIST => items::enum_variant_list,
+        MATCH_ARM_LIST => items::match_arm_list,
+        USE_TREE_LIST => items::use_tree_list,
+        EXTERN_ITEM_LIST => items::extern_item_list,
+        TOKEN_TREE if first_child? == T!['{'] => items::token_tree,
+        ASSOC_ITEM_LIST => match parent? {
+            IMPL => items::impl_item_list,
+            TRAIT => items::trait_item_list,
+            _ => return None,
+        },
+        ITEM_LIST => items::mod_item_list,
+        _ => return None,
+    };
+    Some(res)
+}
+
+#[derive(Clone, Copy, PartialEq, Eq)]
+enum BlockLike {
+    Block,
+    NotBlock,
+}
+
+impl BlockLike {
+    fn is_block(self) -> bool {
+        self == BlockLike::Block
+    }
+}
+
+fn opt_visibility(p: &mut Parser) -> bool {
+    match p.current() {
+        T![pub] => {
+            let m = p.start();
+            p.bump(T![pub]);
+            if p.at(T!['(']) {
+                match p.nth(1) {
+                    // test crate_visibility
+                    // pub(crate) struct S;
+                    // pub(self) struct S;
+                    // pub(self) struct S;
+                    // pub(self) struct S;
+                    T![crate] | T![self] | T![super] => {
+                        p.bump_any();
+                        p.bump_any();
+                        p.expect(T![')']);
+                    }
+                    T![in] => {
+                        p.bump_any();
+                        p.bump_any();
+                        paths::use_path(p);
+                        p.expect(T![')']);
+                    }
+                    _ => (),
+                }
+            }
+            m.complete(p, VISIBILITY);
+        }
+        // test crate_keyword_vis
+        // crate fn main() { }
+        // struct S { crate field: u32 }
+        // struct T(crate u32);
+        //
+        // test crate_keyword_path
+        // fn foo() { crate::foo(); }
+        T![crate] if !p.nth_at(1, T![::]) => {
+            let m = p.start();
+            p.bump(T![crate]);
+            m.complete(p, VISIBILITY);
+        }
+        _ => return false,
+    }
+    true
+}
+
+fn opt_alias(p: &mut Parser) {
+    if p.at(T![as]) {
+        let m = p.start();
+        p.bump(T![as]);
+        if !p.eat(T![_]) {
+            name(p);
+        }
+        m.complete(p, RENAME);
+    }
+}
+
+fn abi(p: &mut Parser) {
+    assert!(p.at(T![extern]));
+    let abi = p.start();
+    p.bump(T![extern]);
+    match p.current() {
+        STRING | RAW_STRING => p.bump_any(),
+        _ => (),
+    }
+    abi.complete(p, ABI);
+}
+
+fn opt_fn_ret_type(p: &mut Parser) -> bool {
+    if p.at(T![->]) {
+        let m = p.start();
+        p.bump(T![->]);
+        types::type_no_bounds(p);
+        m.complete(p, RET_TYPE);
+        true
+    } else {
+        false
+    }
+}
+
+fn name_r(p: &mut Parser, recovery: TokenSet) {
+    if p.at(IDENT) {
+        let m = p.start();
+        p.bump(IDENT);
+        m.complete(p, NAME);
+    } else {
+        p.err_recover("expected a name", recovery);
+    }
+}
+
+fn name(p: &mut Parser) {
+    name_r(p, TokenSet::EMPTY)
+}
+
+fn name_ref(p: &mut Parser) {
+    if p.at(IDENT) {
+        let m = p.start();
+        p.bump(IDENT);
+        m.complete(p, NAME_REF);
+    } else {
+        p.err_and_bump("expected identifier");
+    }
+}
+
+fn name_ref_or_index(p: &mut Parser) {
+    assert!(p.at(IDENT) || p.at(INT_NUMBER));
+    let m = p.start();
+    p.bump_any();
+    m.complete(p, NAME_REF);
+}
+
+fn error_block(p: &mut Parser, message: &str) {
+    assert!(p.at(T!['{']));
+    let m = p.start();
+    p.error(message);
+    p.bump(T!['{']);
+    expressions::expr_block_contents(p);
+    p.eat(T!['}']);
+    m.complete(p, ERROR);
+}

crates/parser/src/grammar/attributes.rs

@@ -0,0 +1,48 @@
+//! FIXME: write short doc here
+
+use super::*;
+
+pub(super) fn inner_attributes(p: &mut Parser) {
+    while p.at(T![#]) && p.nth(1) == T![!] {
+        attribute(p, true)
+    }
+}
+
+pub(super) fn outer_attributes(p: &mut Parser) {
+    while p.at(T![#]) {
+        attribute(p, false)
+    }
+}
+
+fn attribute(p: &mut Parser, inner: bool) {
+    let attr = p.start();
+    assert!(p.at(T![#]));
+    p.bump(T![#]);
+
+    if inner {
+        assert!(p.at(T![!]));
+        p.bump(T![!]);
+    }
+
+    if p.eat(T!['[']) {
+        paths::use_path(p);
+
+        match p.current() {
+            T![=] => {
+                p.bump(T![=]);
+                if expressions::literal(p).is_none() {
+                    p.error("expected literal");
+                }
+            }
+            T!['('] | T!['['] | T!['{'] => items::token_tree(p),
+            _ => {}
+        }
+
+        if !p.eat(T![']']) {
+            p.error("expected `]`");
+        }
+    } else {
+        p.error("expected `[`");
+    }
+    attr.complete(p, ATTR);
+}

crates/parser/src/grammar/expressions.rs

@@ -0,0 +1,651 @@
+//! FIXME: write short doc here
+
+mod atom;
+
+pub(crate) use self::atom::{block_expr, match_arm_list};
+pub(super) use self::atom::{literal, LITERAL_FIRST};
+use super::*;
+
+pub(super) enum StmtWithSemi {
+    Yes,
+    No,
+    Optional,
+}
+
+const EXPR_FIRST: TokenSet = LHS_FIRST;
+
+pub(super) fn expr(p: &mut Parser) -> (Option<CompletedMarker>, BlockLike) {
+    let r = Restrictions { forbid_structs: false, prefer_stmt: false };
+    expr_bp(p, r, 1)
+}
+
+pub(super) fn expr_with_attrs(p: &mut Parser) -> bool {
+    let m = p.start();
+    let has_attrs = p.at(T![#]);
+    attributes::outer_attributes(p);
+
+    let (cm, _block_like) = expr(p);
+    let success = cm.is_some();
+
+    match (has_attrs, cm) {
+        (true, Some(cm)) => {
+            let kind = cm.kind();
+            cm.undo_completion(p).abandon(p);
+            m.complete(p, kind);
+        }
+        _ => m.abandon(p),
+    }
+
+    success
+}
+
+pub(super) fn expr_stmt(p: &mut Parser) -> (Option<CompletedMarker>, BlockLike) {
+    let r = Restrictions { forbid_structs: false, prefer_stmt: true };
+    expr_bp(p, r, 1)
+}
+
+fn expr_no_struct(p: &mut Parser) {
+    let r = Restrictions { forbid_structs: true, prefer_stmt: false };
+    expr_bp(p, r, 1);
+}
+
+fn is_expr_stmt_attr_allowed(kind: SyntaxKind) -> bool {
+    let forbid = matches!(kind, BIN_EXPR | RANGE_EXPR);
+    !forbid
+}
+
+pub(super) fn stmt(p: &mut Parser, with_semi: StmtWithSemi) {
+    let m = p.start();
+    // test attr_on_expr_stmt
+    // fn foo() {
+    //     #[A] foo();
+    //     #[B] bar!{}
+    //     #[C] #[D] {}
+    //     #[D] return ();
+    // }
+    let has_attrs = p.at(T![#]);
+    attributes::outer_attributes(p);
+
+    if p.at(T![let]) {
+        let_stmt(p, m, with_semi);
+        return;
+    }
+
+    // test block_items
+    // fn a() { fn b() {} }
+    let m = match items::maybe_item(p, m) {
+        Ok(()) => return,
+        Err(m) => m,
+    };
+
+    let (cm, blocklike) = expr_stmt(p);
+    let kind = cm.as_ref().map(|cm| cm.kind()).unwrap_or(ERROR);
+
+    if has_attrs && !is_expr_stmt_attr_allowed(kind) {
+        // test_err attr_on_expr_not_allowed
+        // fn foo() {
+        //    #[A] 1 + 2;
+        //    #[B] if true {};
+        // }
+        p.error(format!("attributes are not allowed on {:?}", kind));
+    }
+
+    if p.at(T!['}']) {
+        // test attr_on_last_expr_in_block
+        // fn foo() {
+        //     { #[A] bar!()? }
+        //     #[B] &()
+        // }
+        if let Some(cm) = cm {
+            cm.undo_completion(p).abandon(p);
+            m.complete(p, kind);
+        } else {
+            m.abandon(p);
+        }
+    } else {
+        // test no_semi_after_block
+        // fn foo() {
+        //     if true {}
+        //     loop {}
+        //     match () {}
+        //     while true {}
+        //     for _ in () {}
+        //     {}
+        //     {}
+        //     macro_rules! test {
+        //          () => {}
+        //     }
+        //     test!{}
+        // }
+
+        match with_semi {
+            StmtWithSemi::Yes => {
+                if blocklike.is_block() {
+                    p.eat(T![;]);
+                } else {
+                    p.expect(T![;]);
+                }
+            }
+            StmtWithSemi::No => {}
+            StmtWithSemi::Optional => {
+                if p.at(T![;]) {
+                    p.eat(T![;]);
+                }
+            }
+        }
+
+        m.complete(p, EXPR_STMT);
+    }
+
+    // test let_stmt
+    // fn foo() {
+    //     let a;
+    //     let b: i32;
+    //     let c = 92;
+    //     let d: i32 = 92;
+    //     let e: !;
+    //     let _: ! = {};
+    //     let f = #[attr]||{};
+    // }
+    fn let_stmt(p: &mut Parser, m: Marker, with_semi: StmtWithSemi) {
+        assert!(p.at(T![let]));
+        p.bump(T![let]);
+        patterns::pattern(p);
+        if p.at(T![:]) {
+            types::ascription(p);
+        }
+        if p.eat(T![=]) {
+            expressions::expr_with_attrs(p);
+        }
+
+        match with_semi {
+            StmtWithSemi::Yes => {
+                p.expect(T![;]);
+            }
+            StmtWithSemi::No => {}
+            StmtWithSemi::Optional => {
+                if p.at(T![;]) {
+                    p.eat(T![;]);
+                }
+            }
+        }
+        m.complete(p, LET_STMT);
+    }
+}
+
+pub(super) fn expr_block_contents(p: &mut Parser) {
+    // This is checked by a validator
+    attributes::inner_attributes(p);
+
+    while !p.at(EOF) && !p.at(T!['}']) {
+        // test nocontentexpr
+        // fn foo(){
+        //     ;;;some_expr();;;;{;;;};;;;Ok(())
+        // }
+
+        // test nocontentexpr_after_item
+        // fn simple_function() {
+        //     enum LocalEnum {
+        //         One,
+        //         Two,
+        //     };
+        //     fn f() {};
+        //     struct S {};
+        // }
+
+        if p.at(T![;]) {
+            p.bump(T![;]);
+            continue;
+        }
+
+        stmt(p, StmtWithSemi::Yes)
+    }
+}
+
+#[derive(Clone, Copy)]
+struct Restrictions {
+    forbid_structs: bool,
+    prefer_stmt: bool,
+}
+
+/// Binding powers of operators for a Pratt parser.
+///
+/// See https://www.oilshell.org/blog/2016/11/03.html
+#[rustfmt::skip]
+fn current_op(p: &Parser) -> (u8, SyntaxKind) {
+    const NOT_AN_OP: (u8, SyntaxKind) = (0, T![@]);
+    match p.current() {
+        T![|] if p.at(T![||])  => (3,  T![||]),
+        T![|] if p.at(T![|=])  => (1,  T![|=]),
+        T![|]                  => (6,  T![|]),
+        T![>] if p.at(T![>>=]) => (1,  T![>>=]),
+        T![>] if p.at(T![>>])  => (9,  T![>>]),
+        T![>] if p.at(T![>=])  => (5,  T![>=]),
+        T![>]                  => (5,  T![>]),
+        T![=] if p.at(T![=>])  => NOT_AN_OP,
+        T![=] if p.at(T![==])  => (5,  T![==]),
+        T![=]                  => (1,  T![=]),
+        T![<] if p.at(T![<=])  => (5,  T![<=]),
+        T![<] if p.at(T![<<=]) => (1,  T![<<=]),
+        T![<] if p.at(T![<<])  => (9,  T![<<]),
+        T![<]                  => (5,  T![<]),
+        T![+] if p.at(T![+=])  => (1,  T![+=]),
+        T![+]                  => (10, T![+]),
+        T![^] if p.at(T![^=])  => (1,  T![^=]),
+        T![^]                  => (7,  T![^]),
+        T![%] if p.at(T![%=])  => (1,  T![%=]),
+        T![%]                  => (11, T![%]),
+        T![&] if p.at(T![&=])  => (1,  T![&=]),
+        T![&] if p.at(T![&&])  => (4,  T![&&]),
+        T![&]                  => (8,  T![&]),
+        T![/] if p.at(T![/=])  => (1,  T![/=]),
+        T![/]                  => (11, T![/]),
+        T![*] if p.at(T![*=])  => (1,  T![*=]),
+        T![*]                  => (11, T![*]),
+        T![.] if p.at(T![..=]) => (2,  T![..=]),
+        T![.] if p.at(T![..])  => (2,  T![..]),
+        T![!] if p.at(T![!=])  => (5,  T![!=]),
+        T![-] if p.at(T![-=])  => (1,  T![-=]),
+        T![-]                  => (10, T![-]),
+        T![as]                 => (12, T![as]),
+
+        _                      => NOT_AN_OP
+    }
+}
+
+// Parses expression with binding power of at least bp.
+fn expr_bp(p: &mut Parser, mut r: Restrictions, bp: u8) -> (Option<CompletedMarker>, BlockLike) {
+    let mut lhs = match lhs(p, r) {
+        Some((lhs, blocklike)) => {
+            // test stmt_bin_expr_ambiguity
+            // fn foo() {
+            //     let _ = {1} & 2;
+            //     {1} &2;
+            // }
+            if r.prefer_stmt && blocklike.is_block() {
+                return (Some(lhs), BlockLike::Block);
+            }
+            lhs
+        }
+        None => return (None, BlockLike::NotBlock),
+    };
+
+    loop {
+        let is_range = p.at(T![..]) || p.at(T![..=]);
+        let (op_bp, op) = current_op(p);
+        if op_bp < bp {
+            break;
+        }
+        // test as_precedence
+        // fn foo() {
+        //     let _ = &1 as *const i32;
+        // }
+        if p.at(T![as]) {
+            lhs = cast_expr(p, lhs);
+            continue;
+        }
+        let m = lhs.precede(p);
+        p.bump(op);
+
+        // test binop_resets_statementness
+        // fn foo() {
+        //     v = {1}&2;
+        // }
+        r = Restrictions { prefer_stmt: false, ..r };
+
+        if is_range {
+            // test postfix_range
+            // fn foo() {
+            //     let x = 1..;
+            //     match 1.. { _ => () };
+            //     match a.b()..S { _ => () };
+            // }
+            let has_trailing_expression =
+                p.at_ts(EXPR_FIRST) && !(r.forbid_structs && p.at(T!['{']));
+            if !has_trailing_expression {
+                // no RHS
+                lhs = m.complete(p, RANGE_EXPR);
+                break;
+            }
+        }
+
+        expr_bp(p, Restrictions { prefer_stmt: false, ..r }, op_bp + 1);
+        lhs = m.complete(p, if is_range { RANGE_EXPR } else { BIN_EXPR });
+    }
+    (Some(lhs), BlockLike::NotBlock)
+}
+
+const LHS_FIRST: TokenSet =
+    atom::ATOM_EXPR_FIRST.union(token_set![T![&], T![*], T![!], T![.], T![-]]);
+
+fn lhs(p: &mut Parser, r: Restrictions) -> Option<(CompletedMarker, BlockLike)> {
+    let m;
+    let kind = match p.current() {
+        // test ref_expr
+        // fn foo() {
+        //     // reference operator
+        //     let _ = &1;
+        //     let _ = &mut &f();
+        //     let _ = &raw;
+        //     let _ = &raw.0;
+        //     // raw reference operator
+        //     let _ = &raw mut foo;
+        //     let _ = &raw const foo;
+        // }
+        T![&] => {
+            m = p.start();
+            p.bump(T![&]);
+            if p.at(IDENT)
+                && p.at_contextual_kw("raw")
+                && (p.nth_at(1, T![mut]) || p.nth_at(1, T![const]))
+            {
+                p.bump_remap(T![raw]);
+                p.bump_any();
+            } else {
+                p.eat(T![mut]);
+            }
+            REF_EXPR
+        }
+        // test unary_expr
+        // fn foo() {
+        //     **&1;
+        //     !!true;
+        //     --1;
+        // }
+        T![*] | T![!] | T![-] => {
+            m = p.start();
+            p.bump_any();
+            PREFIX_EXPR
+        }
+        _ => {
+            // test full_range_expr
+            // fn foo() { xs[..]; }
+            for &op in [T![..=], T![..]].iter() {
+                if p.at(op) {
+                    m = p.start();
+                    p.bump(op);
+                    if p.at_ts(EXPR_FIRST) && !(r.forbid_structs && p.at(T!['{'])) {
+                        expr_bp(p, r, 2);
+                    }
+                    return Some((m.complete(p, RANGE_EXPR), BlockLike::NotBlock));
+                }
+            }
+
+            // test expression_after_block
+            // fn foo() {
+            //    let mut p = F{x: 5};
+            //    {p}.x = 10;
+            // }
+            //
+            let (lhs, blocklike) = atom::atom_expr(p, r)?;
+            return Some(postfix_expr(p, lhs, blocklike, !(r.prefer_stmt && blocklike.is_block())));
+        }
+    };
+    // parse the interior of the unary expression
+    expr_bp(p, r, 255);
+    Some((m.complete(p, kind), BlockLike::NotBlock))
+}
+
+fn postfix_expr(
+    p: &mut Parser,
+    mut lhs: CompletedMarker,
+    // Calls are disallowed if the type is a block and we prefer statements because the call cannot be disambiguated from a tuple
+    // E.g. `while true {break}();` is parsed as
+    // `while true {break}; ();`
+    mut block_like: BlockLike,
+    mut allow_calls: bool,
+) -> (CompletedMarker, BlockLike) {
+    loop {
+        lhs = match p.current() {
+            // test stmt_postfix_expr_ambiguity
+            // fn foo() {
+            //     match () {
+            //         _ => {}
+            //         () => {}
+            //         [] => {}
+            //     }
+            // }
+            T!['('] if allow_calls => call_expr(p, lhs),
+            T!['['] if allow_calls => index_expr(p, lhs),
+            T![.] => match postfix_dot_expr(p, lhs) {
+                Ok(it) => it,
+                Err(it) => {
+                    lhs = it;
+                    break;
+                }
+            },
+            T![?] => try_expr(p, lhs),
+            _ => break,
+        };
+        allow_calls = true;
+        block_like = BlockLike::NotBlock;
+    }
+    return (lhs, block_like);
+
+    fn postfix_dot_expr(
+        p: &mut Parser,
+        lhs: CompletedMarker,
+    ) -> Result<CompletedMarker, CompletedMarker> {
+        assert!(p.at(T![.]));
+        if p.nth(1) == IDENT && (p.nth(2) == T!['('] || p.nth_at(2, T![::])) {
+            return Ok(method_call_expr(p, lhs));
+        }
+
+        // test await_expr
+        // fn foo() {
+        //     x.await;
+        //     x.0.await;
+        //     x.0().await?.hello();
+        // }
+        if p.nth(1) == T![await] {
+            let m = lhs.precede(p);
+            p.bump(T![.]);
+            p.bump(T![await]);
+            return Ok(m.complete(p, AWAIT_EXPR));
+        }
+
+        if p.at(T![..=]) || p.at(T![..]) {
+            return Err(lhs);
+        }
+
+        Ok(field_expr(p, lhs))
+    }
+}
+
+// test call_expr
+// fn foo() {
+//     let _ = f();
+//     let _ = f()(1)(1, 2,);
+//     let _ = f(<Foo>::func());
+//     f(<Foo as Trait>::func());
+// }
+fn call_expr(p: &mut Parser, lhs: CompletedMarker) -> CompletedMarker {
+    assert!(p.at(T!['(']));
+    let m = lhs.precede(p);
+    arg_list(p);
+    m.complete(p, CALL_EXPR)
+}
+
+// test index_expr
+// fn foo() {
+//     x[1][2];
+// }
+fn index_expr(p: &mut Parser, lhs: CompletedMarker) -> CompletedMarker {
+    assert!(p.at(T!['[']));
+    let m = lhs.precede(p);
+    p.bump(T!['[']);
+    expr(p);
+    p.expect(T![']']);
+    m.complete(p, INDEX_EXPR)
+}
+
+// test method_call_expr
+// fn foo() {
+//     x.foo();
+//     y.bar::<T>(1, 2,);
+// }
+fn method_call_expr(p: &mut Parser, lhs: CompletedMarker) -> CompletedMarker {
+    assert!(p.at(T![.]) && p.nth(1) == IDENT && (p.nth(2) == T!['('] || p.nth_at(2, T![::])));
+    let m = lhs.precede(p);
+    p.bump_any();
+    name_ref(p);
+    type_args::opt_type_arg_list(p, true);
+    if p.at(T!['(']) {
+        arg_list(p);
+    }
+    m.complete(p, METHOD_CALL_EXPR)
+}
+
+// test field_expr
+// fn foo() {
+//     x.foo;
+//     x.0.bar;
+//     x.0();
+// }
+
+// test_err bad_tuple_index_expr
+// fn foo() {
+//     x.0.;
+//     x.1i32;
+//     x.0x01;
+// }
+fn field_expr(p: &mut Parser, lhs: CompletedMarker) -> CompletedMarker {
+    assert!(p.at(T![.]));
+    let m = lhs.precede(p);
+    p.bump(T![.]);
+    if p.at(IDENT) || p.at(INT_NUMBER) {
+        name_ref_or_index(p)
+    } else if p.at(FLOAT_NUMBER) {
+        // FIXME: How to recover and instead parse INT + T![.]?
+        p.bump_any();
+    } else {
+        p.error("expected field name or number")
+    }
+    m.complete(p, FIELD_EXPR)
+}
+
+// test try_expr
+// fn foo() {
+//     x?;
+// }
+fn try_expr(p: &mut Parser, lhs: CompletedMarker) -> CompletedMarker {
+    assert!(p.at(T![?]));
+    let m = lhs.precede(p);
+    p.bump(T![?]);
+    m.complete(p, TRY_EXPR)
+}
+
+// test cast_expr
+// fn foo() {
+//     82 as i32;
+//     81 as i8 + 1;
+//     79 as i16 - 1;
+//     0x36 as u8 <= 0x37;
+// }
+fn cast_expr(p: &mut Parser, lhs: CompletedMarker) -> CompletedMarker {
+    assert!(p.at(T![as]));
+    let m = lhs.precede(p);
+    p.bump(T![as]);
+    // Use type_no_bounds(), because cast expressions are not
+    // allowed to have bounds.
+    types::type_no_bounds(p);
+    m.complete(p, CAST_EXPR)
+}
+
+fn arg_list(p: &mut Parser) {
+    assert!(p.at(T!['(']));
+    let m = p.start();
+    p.bump(T!['(']);
+    while !p.at(T![')']) && !p.at(EOF) {
+        // test arg_with_attr
+        // fn main() {
+        //     foo(#[attr] 92)
+        // }
+        if !expr_with_attrs(p) {
+            break;
+        }
+        if !p.at(T![')']) && !p.expect(T![,]) {
+            break;
+        }
+    }
+    p.eat(T![')']);
+    m.complete(p, ARG_LIST);
+}
+
+// test path_expr
+// fn foo() {
+//     let _ = a;
+//     let _ = a::b;
+//     let _ = ::a::<b>;
+//     let _ = format!();
+// }
+fn path_expr(p: &mut Parser, r: Restrictions) -> (CompletedMarker, BlockLike) {
+    assert!(paths::is_path_start(p));
+    let m = p.start();
+    paths::expr_path(p);
+    match p.current() {
+        T!['{'] if !r.forbid_structs => {
+            record_field_list(p);
+            (m.complete(p, RECORD_EXPR), BlockLike::NotBlock)
+        }
+        T![!] if !p.at(T![!=]) => {
+            let block_like = items::macro_call_after_excl(p);
+            (m.complete(p, MACRO_CALL), block_like)
+        }
+        _ => (m.complete(p, PATH_EXPR), BlockLike::NotBlock),
+    }
+}
+
+// test record_lit
+// fn foo() {
+//     S {};
+//     S { x, y: 32, };
+//     S { x, y: 32, ..Default::default() };
+//     TupleStruct { 0: 1 };
+// }
+pub(crate) fn record_field_list(p: &mut Parser) {
+    assert!(p.at(T!['{']));
+    let m = p.start();
+    p.bump(T!['{']);
+    while !p.at(EOF) && !p.at(T!['}']) {
+        let m = p.start();
+        // test record_literal_field_with_attr
+        // fn main() {
+        //     S { #[cfg(test)] field: 1 }
+        // }
+        attributes::outer_attributes(p);
+
+        match p.current() {
+            IDENT | INT_NUMBER => {
+                // test_err record_literal_before_ellipsis_recovery
+                // fn main() {
+                //     S { field ..S::default() }
+                // }
+                if p.nth_at(1, T![:]) || p.nth_at(1, T![..]) {
+                    name_ref_or_index(p);
+                    p.expect(T![:]);
+                }
+                expr(p);
+                m.complete(p, RECORD_EXPR_FIELD);
+            }
+            T![.] if p.at(T![..]) => {
+                m.abandon(p);
+                p.bump(T![..]);
+                expr(p);
+            }
+            T!['{'] => {
+                error_block(p, "expected a field");
+                m.abandon(p);
+            }
+            _ => {
+                p.err_and_bump("expected identifier");
+                m.abandon(p);
+            }
+        }
+        if !p.at(T!['}']) {
+            p.expect(T![,]);
+        }
+    }
+    p.expect(T!['}']);
+    m.complete(p, RECORD_EXPR_FIELD_LIST);
+}

crates/parser/src/grammar/expressions/atom.rs

@@ -0,0 +1,611 @@
+//! FIXME: write short doc here
+
+use super::*;
+
+// test expr_literals
+// fn foo() {
+//     let _ = true;
+//     let _ = false;
+//     let _ = 1;
+//     let _ = 2.0;
+//     let _ = b'a';
+//     let _ = 'b';
+//     let _ = "c";
+//     let _ = r"d";
+//     let _ = b"e";
+//     let _ = br"f";
+// }
+pub(crate) const LITERAL_FIRST: TokenSet = token_set![
+    TRUE_KW,
+    FALSE_KW,
+    INT_NUMBER,
+    FLOAT_NUMBER,
+    BYTE,
+    CHAR,
+    STRING,
+    RAW_STRING,
+    BYTE_STRING,
+    RAW_BYTE_STRING
+];
+
+pub(crate) fn literal(p: &mut Parser) -> Option<CompletedMarker> {
+    if !p.at_ts(LITERAL_FIRST) {
+        return None;
+    }
+    let m = p.start();
+    p.bump_any();
+    Some(m.complete(p, LITERAL))
+}
+
+// E.g. for after the break in `if break {}`, this should not match
+pub(super) const ATOM_EXPR_FIRST: TokenSet =
+    LITERAL_FIRST.union(paths::PATH_FIRST).union(token_set![
+        T!['('],
+        T!['{'],
+        T!['['],
+        L_DOLLAR,
+        T![|],
+        T![move],
+        T![box],
+        T![if],
+        T![while],
+        T![match],
+        T![unsafe],
+        T![return],
+        T![break],
+        T![continue],
+        T![async],
+        T![try],
+        T![loop],
+        T![for],
+        LIFETIME,
+    ]);
+
+const EXPR_RECOVERY_SET: TokenSet = token_set![LET_KW, R_DOLLAR];
+
+pub(super) fn atom_expr(p: &mut Parser, r: Restrictions) -> Option<(CompletedMarker, BlockLike)> {
+    if let Some(m) = literal(p) {
+        return Some((m, BlockLike::NotBlock));
+    }
+    if paths::is_path_start(p) {
+        return Some(path_expr(p, r));
+    }
+    let la = p.nth(1);
+    let done = match p.current() {
+        T!['('] => tuple_expr(p),
+        T!['['] => array_expr(p),
+        L_DOLLAR => meta_var_expr(p),
+        T![|] => lambda_expr(p),
+        T![move] if la == T![|] => lambda_expr(p),
+        T![async] if la == T![|] || (la == T![move] && p.nth(2) == T![|]) => lambda_expr(p),
+        T![if] => if_expr(p),
+
+        T![loop] => loop_expr(p, None),
+        T![box] => box_expr(p, None),
+        T![for] => for_expr(p, None),
+        T![while] => while_expr(p, None),
+        T![try] => try_block_expr(p, None),
+        LIFETIME if la == T![:] => {
+            let m = p.start();
+            label(p);
+            match p.current() {
+                T![loop] => loop_expr(p, Some(m)),
+                T![for] => for_expr(p, Some(m)),
+                T![while] => while_expr(p, Some(m)),
+                // test labeled_block
+                // fn f() { 'label: {}; }
+                T!['{'] => {
+                    block_expr(p);
+                    m.complete(p, EFFECT_EXPR)
+                }
+                _ => {
+                    // test_err misplaced_label_err
+                    // fn main() {
+                    //     'loop: impl
+                    // }
+                    p.error("expected a loop");
+                    m.complete(p, ERROR);
+                    return None;
+                }
+            }
+        }
+        T![async] if la == T!['{'] || (la == T![move] && p.nth(2) == T!['{']) => {
+            let m = p.start();
+            p.bump(T![async]);
+            p.eat(T![move]);
+            block_expr(p);
+            m.complete(p, EFFECT_EXPR)
+        }
+        T![match] => match_expr(p),
+        // test unsafe_block
+        // fn f() { unsafe { } }
+        T![unsafe] if la == T!['{'] => {
+            let m = p.start();
+            p.bump(T![unsafe]);
+            block_expr(p);
+            m.complete(p, EFFECT_EXPR)
+        }
+        T!['{'] => {
+            // test for_range_from
+            // fn foo() {
+            //    for x in 0 .. {
+            //        break;
+            //    }
+            // }
+            block_expr_unchecked(p)
+        }
+        T![return] => return_expr(p),
+        T![continue] => continue_expr(p),
+        T![break] => break_expr(p, r),
+        _ => {
+            p.err_recover("expected expression", EXPR_RECOVERY_SET);
+            return None;
+        }
+    };
+    let blocklike = match done.kind() {
+        IF_EXPR | WHILE_EXPR | FOR_EXPR | LOOP_EXPR | MATCH_EXPR | BLOCK_EXPR | EFFECT_EXPR => {
+            BlockLike::Block
+        }
+        _ => BlockLike::NotBlock,
+    };
+    Some((done, blocklike))
+}
+
+// test tuple_expr
+// fn foo() {
+//     ();
+//     (1);
+//     (1,);
+// }
+fn tuple_expr(p: &mut Parser) -> CompletedMarker {
+    assert!(p.at(T!['(']));
+    let m = p.start();
+    p.expect(T!['(']);
+
+    let mut saw_comma = false;
+    let mut saw_expr = false;
+    while !p.at(EOF) && !p.at(T![')']) {
+        saw_expr = true;
+        if !p.at_ts(EXPR_FIRST) {
+            p.error("expected expression");
+            break;
+        }
+        expr(p);
+        if !p.at(T![')']) {
+            saw_comma = true;
+            p.expect(T![,]);
+        }
+    }
+    p.expect(T![')']);
+    m.complete(p, if saw_expr && !saw_comma { PAREN_EXPR } else { TUPLE_EXPR })
+}
+
+// test array_expr
+// fn foo() {
+//     [];
+//     [1];
+//     [1, 2,];
+//     [1; 2];
+// }
+fn array_expr(p: &mut Parser) -> CompletedMarker {
+    assert!(p.at(T!['[']));
+    let m = p.start();
+
+    let mut n_exprs = 0u32;
+    let mut has_semi = false;
+
+    p.bump(T!['[']);
+    while !p.at(EOF) && !p.at(T![']']) {
+        n_exprs += 1;
+
+        // test array_attrs
+        // const A: &[i64] = &[1, #[cfg(test)] 2];
+        if !expr_with_attrs(p) {
+            break;
+        }
+
+        if n_exprs == 1 && p.eat(T![;]) {
+            has_semi = true;
+            continue;
+        }
+
+        if has_semi || !p.at(T![']']) && !p.expect(T![,]) {
+            break;
+        }
+    }
+    p.expect(T![']']);
+
+    m.complete(p, ARRAY_EXPR)
+}
+
+// test lambda_expr
+// fn foo() {
+//     || ();
+//     || -> i32 { 92 };
+//     |x| x;
+//     move |x: i32,| x;
+//     async || {};
+//     move || {};
+//     async move || {};
+// }
+fn lambda_expr(p: &mut Parser) -> CompletedMarker {
+    assert!(
+        p.at(T![|])
+            || (p.at(T![move]) && p.nth(1) == T![|])
+            || (p.at(T![async]) && p.nth(1) == T![|])
+            || (p.at(T![async]) && p.nth(1) == T![move] && p.nth(2) == T![|])
+    );
+    let m = p.start();
+    p.eat(T![async]);
+    p.eat(T![move]);
+    params::param_list_closure(p);
+    if opt_fn_ret_type(p) {
+        // test lambda_ret_block
+        // fn main() { || -> i32 { 92 }(); }
+        block_expr(p);
+    } else {
+        if p.at_ts(EXPR_FIRST) {
+            expr(p);
+        } else {
+            p.error("expected expression");
+        }
+    }
+    m.complete(p, CLOSURE_EXPR)
+}
+
+// test if_expr
+// fn foo() {
+//     if true {};
+//     if true {} else {};
+//     if true {} else if false {} else {};
+//     if S {};
+//     if { true } { } else { };
+// }
+fn if_expr(p: &mut Parser) -> CompletedMarker {
+    assert!(p.at(T![if]));
+    let m = p.start();
+    p.bump(T![if]);
+    cond(p);
+    block_expr(p);
+    if p.at(T![else]) {
+        p.bump(T![else]);
+        if p.at(T![if]) {
+            if_expr(p);
+        } else {
+            block_expr(p);
+        }
+    }
+    m.complete(p, IF_EXPR)
+}
+
+// test label
+// fn foo() {
+//     'a: loop {}
+//     'b: while true {}
+//     'c: for x in () {}
+// }
+fn label(p: &mut Parser) {
+    assert!(p.at(LIFETIME) && p.nth(1) == T![:]);
+    let m = p.start();
+    p.bump(LIFETIME);
+    p.bump_any();
+    m.complete(p, LABEL);
+}
+
+// test loop_expr
+// fn foo() {
+//     loop {};
+// }
+fn loop_expr(p: &mut Parser, m: Option<Marker>) -> CompletedMarker {
+    assert!(p.at(T![loop]));
+    let m = m.unwrap_or_else(|| p.start());
+    p.bump(T![loop]);
+    block_expr(p);
+    m.complete(p, LOOP_EXPR)
+}
+
+// test while_expr
+// fn foo() {
+//     while true {};
+//     while let Some(x) = it.next() {};
+//     while { true } {};
+// }
+fn while_expr(p: &mut Parser, m: Option<Marker>) -> CompletedMarker {
+    assert!(p.at(T![while]));
+    let m = m.unwrap_or_else(|| p.start());
+    p.bump(T![while]);
+    cond(p);
+    block_expr(p);
+    m.complete(p, WHILE_EXPR)
+}
+
+// test for_expr
+// fn foo() {
+//     for x in [] {};
+// }
+fn for_expr(p: &mut Parser, m: Option<Marker>) -> CompletedMarker {
+    assert!(p.at(T![for]));
+    let m = m.unwrap_or_else(|| p.start());
+    p.bump(T![for]);
+    patterns::pattern(p);
+    p.expect(T![in]);
+    expr_no_struct(p);
+    block_expr(p);
+    m.complete(p, FOR_EXPR)
+}
+
+// test cond
+// fn foo() { if let Some(_) = None {} }
+// fn bar() {
+//     if let Some(_) | Some(_) = None {}
+//     if let | Some(_) = None {}
+//     while let Some(_) | Some(_) = None {}
+//     while let | Some(_) = None {}
+// }
+fn cond(p: &mut Parser) {
+    let m = p.start();
+    if p.eat(T![let]) {
+        patterns::pattern_top(p);
+        p.expect(T![=]);
+    }
+    expr_no_struct(p);
+    m.complete(p, CONDITION);
+}
+
+// test match_expr
+// fn foo() {
+//     match () { };
+//     match S {};
+//     match { } { _ => () };
+//     match { S {} } {};
+// }
+fn match_expr(p: &mut Parser) -> CompletedMarker {
+    assert!(p.at(T![match]));
+    let m = p.start();
+    p.bump(T![match]);
+    expr_no_struct(p);
+    if p.at(T!['{']) {
+        match_arm_list(p);
+    } else {
+        p.error("expected `{`")
+    }
+    m.complete(p, MATCH_EXPR)
+}
+
+pub(crate) fn match_arm_list(p: &mut Parser) {
+    assert!(p.at(T!['{']));
+    let m = p.start();
+    p.eat(T!['{']);
+
+    // test match_arms_inner_attribute
+    // fn foo() {
+    //     match () {
+    //         #![doc("Inner attribute")]
+    //         #![doc("Can be")]
+    //         #![doc("Stacked")]
+    //         _ => (),
+    //     }
+    // }
+    attributes::inner_attributes(p);
+
+    while !p.at(EOF) && !p.at(T!['}']) {
+        if p.at(T!['{']) {
+            error_block(p, "expected match arm");
+            continue;
+        }
+
+        // test match_arms_commas
+        // fn foo() {
+        //     match () {
+        //         _ => (),
+        //         _ => {}
+        //         _ => ()
+        //     }
+        // }
+        if match_arm(p).is_block() {
+            p.eat(T![,]);
+        } else if !p.at(T!['}']) {
+            p.expect(T![,]);
+        }
+    }
+    p.expect(T!['}']);
+    m.complete(p, MATCH_ARM_LIST);
+}
+
+// test match_arm
+// fn foo() {
+//     match () {
+//         _ => (),
+//         _ if Test > Test{field: 0} => (),
+//         X | Y if Z => (),
+//         | X | Y if Z => (),
+//         | X => (),
+//     };
+// }
+fn match_arm(p: &mut Parser) -> BlockLike {
+    let m = p.start();
+    // test match_arms_outer_attributes
+    // fn foo() {
+    //     match () {
+    //         #[cfg(feature = "some")]
+    //         _ => (),
+    //         #[cfg(feature = "other")]
+    //         _ => (),
+    //         #[cfg(feature = "many")]
+    //         #[cfg(feature = "attributes")]
+    //         #[cfg(feature = "before")]
+    //         _ => (),
+    //     }
+    // }
+    attributes::outer_attributes(p);
+
+    patterns::pattern_top_r(p, TokenSet::EMPTY);
+    if p.at(T![if]) {
+        match_guard(p);
+    }
+    p.expect(T![=>]);
+    let blocklike = expr_stmt(p).1;
+    m.complete(p, MATCH_ARM);
+    blocklike
+}
+
+// test match_guard
+// fn foo() {
+//     match () {
+//         _ if foo => (),
+//     }
+// }
+fn match_guard(p: &mut Parser) -> CompletedMarker {
+    assert!(p.at(T![if]));
+    let m = p.start();
+    p.bump(T![if]);
+    expr(p);
+    m.complete(p, MATCH_GUARD)
+}
+
+// test block
+// fn a() {}
+// fn b() { let _ = 1; }
+// fn c() { 1; 2; }
+// fn d() { 1; 2 }
+pub(crate) fn block_expr(p: &mut Parser) {
+    if !p.at(T!['{']) {
+        p.error("expected a block");
+        return;
+    }
+    block_expr_unchecked(p);
+}
+
+fn block_expr_unchecked(p: &mut Parser) -> CompletedMarker {
+    assert!(p.at(T!['{']));
+    let m = p.start();
+    p.bump(T!['{']);
+    expr_block_contents(p);
+    p.expect(T!['}']);
+    m.complete(p, BLOCK_EXPR)
+}
+
+// test return_expr
+// fn foo() {
+//     return;
+//     return 92;
+// }
+fn return_expr(p: &mut Parser) -> CompletedMarker {
+    assert!(p.at(T![return]));
+    let m = p.start();
+    p.bump(T![return]);
+    if p.at_ts(EXPR_FIRST) {
+        expr(p);
+    }
+    m.complete(p, RETURN_EXPR)
+}
+
+// test continue_expr
+// fn foo() {
+//     loop {
+//         continue;
+//         continue 'l;
+//     }
+// }
+fn continue_expr(p: &mut Parser) -> CompletedMarker {
+    assert!(p.at(T![continue]));
+    let m = p.start();
+    p.bump(T![continue]);
+    p.eat(LIFETIME);
+    m.complete(p, CONTINUE_EXPR)
+}
+
+// test break_expr
+// fn foo() {
+//     loop {
+//         break;
+//         break 'l;
+//         break 92;
+//         break 'l 92;
+//     }
+// }
+fn break_expr(p: &mut Parser, r: Restrictions) -> CompletedMarker {
+    assert!(p.at(T![break]));
+    let m = p.start();
+    p.bump(T![break]);
+    p.eat(LIFETIME);
+    // test break_ambiguity
+    // fn foo(){
+    //     if break {}
+    //     while break {}
+    //     for i in break {}
+    //     match break {}
+    // }
+    if p.at_ts(EXPR_FIRST) && !(r.forbid_structs && p.at(T!['{'])) {
+        expr(p);
+    }
+    m.complete(p, BREAK_EXPR)
+}
+
+// test try_block_expr
+// fn foo() {
+//     let _ = try {};
+// }
+fn try_block_expr(p: &mut Parser, m: Option<Marker>) -> CompletedMarker {
+    assert!(p.at(T![try]));
+    let m = m.unwrap_or_else(|| p.start());
+    // Special-case `try!` as macro.
+    // This is a hack until we do proper edition support
+    if p.nth_at(1, T![!]) {
+        // test try_macro_fallback
+        // fn foo() { try!(Ok(())); }
+        let path = p.start();
+        let path_segment = p.start();
+        let name_ref = p.start();
+        p.bump_remap(IDENT);
+        name_ref.complete(p, NAME_REF);
+        path_segment.complete(p, PATH_SEGMENT);
+        path.complete(p, PATH);
+        let _block_like = items::macro_call_after_excl(p);
+        return m.complete(p, MACRO_CALL);
+    }
+
+    p.bump(T![try]);
+    block_expr(p);
+    m.complete(p, EFFECT_EXPR)
+}
+
+// test box_expr
+// fn foo() {
+//     let x = box 1i32;
+//     let y = (box 1i32, box 2i32);
+//     let z = Foo(box 1i32, box 2i32);
+// }
+fn box_expr(p: &mut Parser, m: Option<Marker>) -> CompletedMarker {
+    assert!(p.at(T![box]));
+    let m = m.unwrap_or_else(|| p.start());
+    p.bump(T![box]);
+    if p.at_ts(EXPR_FIRST) {
+        expr(p);
+    }
+    m.complete(p, BOX_EXPR)
+}
+
+/// Expression from `$var` macro expansion, wrapped in dollars
+fn meta_var_expr(p: &mut Parser) -> CompletedMarker {
+    assert!(p.at(L_DOLLAR));
+    let m = p.start();
+    p.bump(L_DOLLAR);
+    let (completed, _is_block) =
+        expr_bp(p, Restrictions { forbid_structs: false, prefer_stmt: false }, 1);
+
+    match (completed, p.current()) {
+        (Some(it), R_DOLLAR) => {
+            p.bump(R_DOLLAR);
+            m.abandon(p);
+            it
+        }
+        _ => {
+            while !p.at(R_DOLLAR) {
+                p.bump_any()
+            }
+            p.bump(R_DOLLAR);
+            m.complete(p, ERROR)
+        }
+    }
+}

crates/parser/src/grammar/items.rs

@@ -0,0 +1,432 @@
+//! FIXME: write short doc here
+
+mod consts;
+mod adt;
+mod traits;
+mod use_item;
+
+pub(crate) use self::{
+    adt::{enum_variant_list, record_field_def_list},
+    expressions::{match_arm_list, record_field_list},
+    traits::{impl_item_list, trait_item_list},
+    use_item::use_tree_list,
+};
+use super::*;
+
+// test mod_contents
+// fn foo() {}
+// macro_rules! foo {}
+// foo::bar!();
+// super::baz! {}
+// struct S;
+pub(super) fn mod_contents(p: &mut Parser, stop_on_r_curly: bool) {
+    attributes::inner_attributes(p);
+    while !(stop_on_r_curly && p.at(T!['}']) || p.at(EOF)) {
+        item_or_macro(p, stop_on_r_curly)
+    }
+}
+
+pub(super) const ITEM_RECOVERY_SET: TokenSet = token_set![
+    FN_KW, STRUCT_KW, ENUM_KW, IMPL_KW, TRAIT_KW, CONST_KW, STATIC_KW, LET_KW, MOD_KW, PUB_KW,
+    CRATE_KW, USE_KW, MACRO_KW
+];
+
+pub(super) fn item_or_macro(p: &mut Parser, stop_on_r_curly: bool) {
+    let m = p.start();
+    attributes::outer_attributes(p);
+    let m = match maybe_item(p, m) {
+        Ok(()) => {
+            if p.at(T![;]) {
+                p.err_and_bump(
+                    "expected item, found `;`\n\
+                     consider removing this semicolon",
+                );
+            }
+            return;
+        }
+        Err(m) => m,
+    };
+    if paths::is_use_path_start(p) {
+        match macro_call(p) {
+            BlockLike::Block => (),
+            BlockLike::NotBlock => {
+                p.expect(T![;]);
+            }
+        }
+        m.complete(p, MACRO_CALL);
+    } else {
+        m.abandon(p);
+        if p.at(T!['{']) {
+            error_block(p, "expected an item");
+        } else if p.at(T!['}']) && !stop_on_r_curly {
+            let e = p.start();
+            p.error("unmatched `}`");
+            p.bump(T!['}']);
+            e.complete(p, ERROR);
+        } else if !p.at(EOF) && !p.at(T!['}']) {
+            p.err_and_bump("expected an item");
+        } else {
+            p.error("expected an item");
+        }
+    }
+}
+
+pub(super) fn maybe_item(p: &mut Parser, m: Marker) -> Result<(), Marker> {
+    // test_err pub_expr
+    // fn foo() { pub 92; }
+    let has_visibility = opt_visibility(p);
+
+    let m = match items_without_modifiers(p, m) {
+        Ok(()) => return Ok(()),
+        Err(m) => m,
+    };
+
+    let mut has_mods = false;
+
+    // modifiers
+    has_mods |= p.eat(T![const]);
+
+    // test_err async_without_semicolon
+    // fn foo() { let _ = async {} }
+    if p.at(T![async]) && p.nth(1) != T!['{'] && p.nth(1) != T![move] && p.nth(1) != T![|] {
+        p.eat(T![async]);
+        has_mods = true;
+    }
+
+    // test_err unsafe_block_in_mod
+    // fn foo(){} unsafe { } fn bar(){}
+    if p.at(T![unsafe]) && p.nth(1) != T!['{'] {
+        p.eat(T![unsafe]);
+        has_mods = true;
+    }
+
+    if p.at(T![extern]) {
+        has_mods = true;
+        abi(p);
+    }
+    if p.at(IDENT) && p.at_contextual_kw("auto") && p.nth(1) == T![trait] {
+        p.bump_remap(T![auto]);
+        has_mods = true;
+    }
+
+    // test default_item
+    // default impl T for Foo {}
+    if p.at(IDENT) && p.at_contextual_kw("default") {
+        match p.nth(1) {
+            T![fn] | T![type] | T![const] | T![impl] => {
+                p.bump_remap(T![default]);
+                has_mods = true;
+            }
+            T![unsafe] => {
+                // test default_unsafe_item
+                // default unsafe impl T for Foo {
+                //     default unsafe fn foo() {}
+                // }
+                if matches!(p.nth(2), T![impl] | T![fn]) {
+                    p.bump_remap(T![default]);
+                    p.bump(T![unsafe]);
+                    has_mods = true;
+                }
+            }
+            _ => (),
+        }
+    }
+
+    // test existential_type
+    // existential type Foo: Fn() -> usize;
+    if p.at(IDENT) && p.at_contextual_kw("existential") && p.nth(1) == T![type] {
+        p.bump_remap(T![existential]);
+        has_mods = true;
+    }
+
+    // items
+    match p.current() {
+        // test fn
+        // fn foo() {}
+        T![fn] => {
+            fn_def(p);
+            m.complete(p, FN);
+        }
+
+        // test trait
+        // trait T {}
+        T![trait] => {
+            traits::trait_def(p);
+            m.complete(p, TRAIT);
+        }
+
+        T![const] => {
+            consts::const_def(p, m);
+        }
+
+        // test impl
+        // impl T for S {}
+        T![impl] => {
+            traits::impl_def(p);
+            m.complete(p, IMPL);
+        }
+
+        T![type] => {
+            type_def(p, m);
+        }
+        _ => {
+            if !has_visibility && !has_mods {
+                return Err(m);
+            } else {
+                if has_mods {
+                    p.error("expected existential, fn, trait or impl");
+                } else {
+                    p.error("expected an item");
+                }
+                m.complete(p, ERROR);
+            }
+        }
+    }
+    Ok(())
+}
+
+fn items_without_modifiers(p: &mut Parser, m: Marker) -> Result<(), Marker> {
+    let la = p.nth(1);
+    match p.current() {
+        // test extern_crate
+        // extern crate foo;
+        T![extern] if la == T![crate] => extern_crate_item(p, m),
+        T![type] => {
+            type_def(p, m);
+        }
+        T![mod] => mod_item(p, m),
+        T![struct] => {
+            // test struct_items
+            // struct Foo;
+            // struct Foo {}
+            // struct Foo();
+            // struct Foo(String, usize);
+            // struct Foo {
+            //     a: i32,
+            //     b: f32,
+            // }
+            adt::struct_def(p, m);
+        }
+        // test pub_macro_def
+        // pub macro m($:ident) {}
+        T![macro] => {
+            macro_def(p, m);
+        }
+        IDENT if p.at_contextual_kw("union") && p.nth(1) == IDENT => {
+            // test union_items
+            // union Foo {}
+            // union Foo {
+            //     a: i32,
+            //     b: f32,
+            // }
+            adt::union_def(p, m);
+        }
+        T![enum] => adt::enum_def(p, m),
+        T![use] => use_item::use_item(p, m),
+        T![const] if (la == IDENT || la == T![_] || la == T![mut]) => consts::const_def(p, m),
+        T![static] => consts::static_def(p, m),
+        // test extern_block
+        // extern {}
+        T![extern]
+            if la == T!['{'] || ((la == STRING || la == RAW_STRING) && p.nth(2) == T!['{']) =>
+        {
+            abi(p);
+            extern_item_list(p);
+            m.complete(p, EXTERN_BLOCK);
+        }
+        _ => return Err(m),
+    };
+    Ok(())
+}
+
+fn extern_crate_item(p: &mut Parser, m: Marker) {
+    assert!(p.at(T![extern]));
+    p.bump(T![extern]);
+    assert!(p.at(T![crate]));
+    p.bump(T![crate]);
+
+    if p.at(T![self]) {
+        p.bump(T![self]);
+    } else {
+        name_ref(p);
+    }
+
+    opt_alias(p);
+    p.expect(T![;]);
+    m.complete(p, EXTERN_CRATE);
+}
+
+pub(crate) fn extern_item_list(p: &mut Parser) {
+    assert!(p.at(T!['{']));
+    let m = p.start();
+    p.bump(T!['{']);
+    mod_contents(p, true);
+    p.expect(T!['}']);
+    m.complete(p, EXTERN_ITEM_LIST);
+}
+
+fn fn_def(p: &mut Parser) {
+    assert!(p.at(T![fn]));
+    p.bump(T![fn]);
+
+    name_r(p, ITEM_RECOVERY_SET);
+    // test function_type_params
+    // fn foo<T: Clone + Copy>(){}
+    type_params::opt_type_param_list(p);
+
+    if p.at(T!['(']) {
+        params::param_list_fn_def(p);
+    } else {
+        p.error("expected function arguments");
+    }
+    // test function_ret_type
+    // fn foo() {}
+    // fn bar() -> () {}
+    opt_fn_ret_type(p);
+
+    // test function_where_clause
+    // fn foo<T>() where T: Copy {}
+    type_params::opt_where_clause(p);
+
+    // test fn_decl
+    // trait T { fn foo(); }
+    if p.at(T![;]) {
+        p.bump(T![;]);
+    } else {
+        expressions::block_expr(p)
+    }
+}
+
+// test type_item
+// type Foo = Bar;
+fn type_def(p: &mut Parser, m: Marker) {
+    assert!(p.at(T![type]));
+    p.bump(T![type]);
+
+    name(p);
+
+    // test type_item_type_params
+    // type Result<T> = ();
+    type_params::opt_type_param_list(p);
+
+    if p.at(T![:]) {
+        type_params::bounds(p);
+    }
+
+    // test type_item_where_clause
+    // type Foo where Foo: Copy = ();
+    type_params::opt_where_clause(p);
+    if p.eat(T![=]) {
+        types::type_(p);
+    }
+    p.expect(T![;]);
+    m.complete(p, TYPE_ALIAS);
+}
+
+pub(crate) fn mod_item(p: &mut Parser, m: Marker) {
+    assert!(p.at(T![mod]));
+    p.bump(T![mod]);
+
+    name(p);
+    if p.at(T!['{']) {
+        mod_item_list(p);
+    } else if !p.eat(T![;]) {
+        p.error("expected `;` or `{`");
+    }
+    m.complete(p, MODULE);
+}
+
+pub(crate) fn mod_item_list(p: &mut Parser) {
+    assert!(p.at(T!['{']));
+    let m = p.start();
+    p.bump(T!['{']);
+    mod_contents(p, true);
+    p.expect(T!['}']);
+    m.complete(p, ITEM_LIST);
+}
+
+// test macro_def
+// macro m { ($i:ident) => {} }
+// macro m($i:ident) {}
+fn macro_def(p: &mut Parser, m: Marker) {
+    p.expect(T![macro]);
+    name_r(p, ITEM_RECOVERY_SET);
+    if p.at(T!['{']) {
+        token_tree(p);
+    } else if !p.at(T!['(']) {
+        p.error("unmatched `(`");
+    } else {
+        let m = p.start();
+        token_tree(p);
+        match p.current() {
+            T!['{'] | T!['['] | T!['('] => token_tree(p),
+            _ => p.error("expected `{`, `[`, `(`"),
+        }
+        m.complete(p, TOKEN_TREE);
+    }
+
+    m.complete(p, MACRO_DEF);
+}
+
+fn macro_call(p: &mut Parser) -> BlockLike {
+    assert!(paths::is_use_path_start(p));
+    paths::use_path(p);
+    macro_call_after_excl(p)
+}
+
+pub(super) fn macro_call_after_excl(p: &mut Parser) -> BlockLike {
+    p.expect(T![!]);
+    if p.at(IDENT) {
+        name(p);
+    }
+    // Special-case `macro_rules! try`.
+    // This is a hack until we do proper edition support
+
+    // test try_macro_rules
+    // macro_rules! try { () => {} }
+    if p.at(T![try]) {
+        let m = p.start();
+        p.bump_remap(IDENT);
+        m.complete(p, NAME);
+    }
+
+    match p.current() {
+        T!['{'] => {
+            token_tree(p);
+            BlockLike::Block
+        }
+        T!['('] | T!['['] => {
+            token_tree(p);
+            BlockLike::NotBlock
+        }
+        _ => {
+            p.error("expected `{`, `[`, `(`");
+            BlockLike::NotBlock
+        }
+    }
+}
+
+pub(crate) fn token_tree(p: &mut Parser) {
+    let closing_paren_kind = match p.current() {
+        T!['{'] => T!['}'],
+        T!['('] => T![')'],
+        T!['['] => T![']'],
+        _ => unreachable!(),
+    };
+    let m = p.start();
+    p.bump_any();
+    while !p.at(EOF) && !p.at(closing_paren_kind) {
+        match p.current() {
+            T!['{'] | T!['('] | T!['['] => token_tree(p),
+            T!['}'] => {
+                p.error("unmatched `}`");
+                m.complete(p, TOKEN_TREE);
+                return;
+            }
+            T![')'] | T![']'] => p.err_and_bump("unmatched brace"),
+            _ => p.bump_any(),
+        }
+    }
+    p.expect(closing_paren_kind);
+    m.complete(p, TOKEN_TREE);
+}

crates/parser/src/grammar/items/adt.rs

@@ -0,0 +1,178 @@
+//! FIXME: write short doc here
+
+use super::*;
+
+pub(super) fn struct_def(p: &mut Parser, m: Marker) {
+    assert!(p.at(T![struct]));
+    p.bump(T![struct]);
+    struct_or_union(p, m, T![struct], STRUCT);
+}
+
+pub(super) fn union_def(p: &mut Parser, m: Marker) {
+    assert!(p.at_contextual_kw("union"));
+    p.bump_remap(T![union]);
+    struct_or_union(p, m, T![union], UNION);
+}
+
+fn struct_or_union(p: &mut Parser, m: Marker, kw: SyntaxKind, def: SyntaxKind) {
+    name_r(p, ITEM_RECOVERY_SET);
+    type_params::opt_type_param_list(p);
+    match p.current() {
+        T![where] => {
+            type_params::opt_where_clause(p);
+            match p.current() {
+                T![;] => {
+                    p.bump(T![;]);
+                }
+                T!['{'] => record_field_def_list(p),
+                _ => {
+                    //FIXME: special case `(` error message
+                    p.error("expected `;` or `{`");
+                }
+            }
+        }
+        T![;] if kw == T![struct] => {
+            p.bump(T![;]);
+        }
+        T!['{'] => record_field_def_list(p),
+        T!['('] if kw == T![struct] => {
+            tuple_field_def_list(p);
+            // test tuple_struct_where
+            // struct Test<T>(T) where T: Clone;
+            // struct Test<T>(T);
+            type_params::opt_where_clause(p);
+            p.expect(T![;]);
+        }
+        _ if kw == T![struct] => {
+            p.error("expected `;`, `{`, or `(`");
+        }
+        _ => {
+            p.error("expected `{`");
+        }
+    }
+    m.complete(p, def);
+}
+
+pub(super) fn enum_def(p: &mut Parser, m: Marker) {
+    assert!(p.at(T![enum]));
+    p.bump(T![enum]);
+    name_r(p, ITEM_RECOVERY_SET);
+    type_params::opt_type_param_list(p);
+    type_params::opt_where_clause(p);
+    if p.at(T!['{']) {
+        enum_variant_list(p);
+    } else {
+        p.error("expected `{`")
+    }
+    m.complete(p, ENUM);
+}
+
+pub(crate) fn enum_variant_list(p: &mut Parser) {
+    assert!(p.at(T!['{']));
+    let m = p.start();
+    p.bump(T!['{']);
+    while !p.at(EOF) && !p.at(T!['}']) {
+        if p.at(T!['{']) {
+            error_block(p, "expected enum variant");
+            continue;
+        }
+        let var = p.start();
+        attributes::outer_attributes(p);
+        if p.at(IDENT) {
+            name(p);
+            match p.current() {
+                T!['{'] => record_field_def_list(p),
+                T!['('] => tuple_field_def_list(p),
+                _ => (),
+            }
+
+            // test variant_discriminant
+            // enum E { X(i32) = 10 }
+            if p.eat(T![=]) {
+                expressions::expr(p);
+            }
+            var.complete(p, VARIANT);
+        } else {
+            var.abandon(p);
+            p.err_and_bump("expected enum variant");
+        }
+        if !p.at(T!['}']) {
+            p.expect(T![,]);
+        }
+    }
+    p.expect(T!['}']);
+    m.complete(p, VARIANT_LIST);
+}
+
+pub(crate) fn record_field_def_list(p: &mut Parser) {
+    assert!(p.at(T!['{']));
+    let m = p.start();
+    p.bump(T!['{']);
+    while !p.at(T!['}']) && !p.at(EOF) {
+        if p.at(T!['{']) {
+            error_block(p, "expected field");
+            continue;
+        }
+        record_field_def(p);
+        if !p.at(T!['}']) {
+            p.expect(T![,]);
+        }
+    }
+    p.expect(T!['}']);
+    m.complete(p, RECORD_FIELD_LIST);
+
+    fn record_field_def(p: &mut Parser) {
+        let m = p.start();
+        // test record_field_attrs
+        // struct S {
+        //     #[serde(with = "url_serde")]
+        //     pub uri: Uri,
+        // }
+        attributes::outer_attributes(p);
+        opt_visibility(p);
+        if p.at(IDENT) {
+            name(p);
+            p.expect(T![:]);
+            types::type_(p);
+            m.complete(p, RECORD_FIELD);
+        } else {
+            m.abandon(p);
+            p.err_and_bump("expected field declaration");
+        }
+    }
+}
+
+fn tuple_field_def_list(p: &mut Parser) {
+    assert!(p.at(T!['(']));
+    let m = p.start();
+    if !p.expect(T!['(']) {
+        return;
+    }
+    while !p.at(T![')']) && !p.at(EOF) {
+        let m = p.start();
+        // test tuple_field_attrs
+        // struct S (
+        //     #[serde(with = "url_serde")]
+        //     pub Uri,
+        // );
+        //
+        // enum S {
+        //     Uri(#[serde(with = "url_serde")] Uri),
+        // }
+        attributes::outer_attributes(p);
+        opt_visibility(p);
+        if !p.at_ts(types::TYPE_FIRST) {
+            p.error("expected a type");
+            m.complete(p, ERROR);
+            break;
+        }
+        types::type_(p);
+        m.complete(p, TUPLE_FIELD);
+
+        if !p.at(T![')']) {
+            p.expect(T![,]);
+        }
+    }
+    p.expect(T![')']);
+    m.complete(p, TUPLE_FIELD_LIST);
+}

crates/parser/src/grammar/items/consts.rs

@@ -0,0 +1,33 @@
+//! FIXME: write short doc here
+
+use super::*;
+
+pub(super) fn static_def(p: &mut Parser, m: Marker) {
+    const_or_static(p, m, T![static], STATIC)
+}
+
+pub(super) fn const_def(p: &mut Parser, m: Marker) {
+    const_or_static(p, m, T![const], CONST)
+}
+
+fn const_or_static(p: &mut Parser, m: Marker, kw: SyntaxKind, def: SyntaxKind) {
+    assert!(p.at(kw));
+    p.bump(kw);
+    p.eat(T![mut]); // FIXME: validator to forbid const mut
+
+    // Allow `_` in place of an identifier in a `const`.
+    let is_const_underscore = kw == T![const] && p.eat(T![_]);
+    if !is_const_underscore {
+        name(p);
+    }
+
+    // test_err static_underscore
+    // static _: i32 = 5;
+
+    types::ascription(p);
+    if p.eat(T![=]) {
+        expressions::expr(p);
+    }
+    p.expect(T![;]);
+    m.complete(p, def);
+}

crates/parser/src/grammar/items/traits.rs

@@ -0,0 +1,153 @@
+//! FIXME: write short doc here
+
+use super::*;
+
+// test trait_item
+// trait T<U>: Hash + Clone where U: Copy {}
+// trait X<U: Debug + Display>: Hash + Clone where U: Copy {}
+pub(super) fn trait_def(p: &mut Parser) {
+    assert!(p.at(T![trait]));
+    p.bump(T![trait]);
+    name_r(p, ITEM_RECOVERY_SET);
+    type_params::opt_type_param_list(p);
+    // test trait_alias
+    // trait Z<U> = T<U>;
+    // trait Z<U> = T<U> where U: Copy;
+    // trait Z<U> = where Self: T<U>;
+    if p.eat(T![=]) {
+        type_params::bounds_without_colon(p);
+        type_params::opt_where_clause(p);
+        p.expect(T![;]);
+        return;
+    }
+    if p.at(T![:]) {
+        type_params::bounds(p);
+    }
+    type_params::opt_where_clause(p);
+    if p.at(T!['{']) {
+        trait_item_list(p);
+    } else {
+        p.error("expected `{`");
+    }
+}
+
+// test trait_item_list
+// impl F {
+//     type A: Clone;
+//     const B: i32;
+//     fn foo() {}
+//     fn bar(&self);
+// }
+pub(crate) fn trait_item_list(p: &mut Parser) {
+    assert!(p.at(T!['{']));
+    let m = p.start();
+    p.bump(T!['{']);
+    while !p.at(EOF) && !p.at(T!['}']) {
+        if p.at(T!['{']) {
+            error_block(p, "expected an item");
+            continue;
+        }
+        item_or_macro(p, true);
+    }
+    p.expect(T!['}']);
+    m.complete(p, ASSOC_ITEM_LIST);
+}
+
+// test impl_def
+// impl Foo {}
+pub(super) fn impl_def(p: &mut Parser) {
+    assert!(p.at(T![impl]));
+    p.bump(T![impl]);
+    if choose_type_params_over_qpath(p) {
+        type_params::opt_type_param_list(p);
+    }
+
+    // FIXME: never type
+    // impl ! {}
+
+    // test impl_def_neg
+    // impl !Send for X {}
+    p.eat(T![!]);
+    impl_type(p);
+    if p.eat(T![for]) {
+        impl_type(p);
+    }
+    type_params::opt_where_clause(p);
+    if p.at(T!['{']) {
+        impl_item_list(p);
+    } else {
+        p.error("expected `{`");
+    }
+}
+
+// test impl_item_list
+// impl F {
+//     type A = i32;
+//     const B: i32 = 92;
+//     fn foo() {}
+//     fn bar(&self) {}
+// }
+pub(crate) fn impl_item_list(p: &mut Parser) {
+    assert!(p.at(T!['{']));
+    let m = p.start();
+    p.bump(T!['{']);
+    // test impl_inner_attributes
+    // enum F{}
+    // impl F {
+    //      //! This is a doc comment
+    //      #![doc("This is also a doc comment")]
+    // }
+    attributes::inner_attributes(p);
+
+    while !p.at(EOF) && !p.at(T!['}']) {
+        if p.at(T!['{']) {
+            error_block(p, "expected an item");
+            continue;
+        }
+        item_or_macro(p, true);
+    }
+    p.expect(T!['}']);
+    m.complete(p, ASSOC_ITEM_LIST);
+}
+
+// test impl_type_params
+// impl<const N: u32> Bar<N> {}
+fn choose_type_params_over_qpath(p: &Parser) -> bool {
+    // There's an ambiguity between generic parameters and qualified paths in impls.
+    // If we see `<` it may start both, so we have to inspect some following tokens.
+    // The following combinations can only start generics,
+    // but not qualified paths (with one exception):
+    //     `<` `>` - empty generic parameters
+    //     `<` `#` - generic parameters with attributes
+    //     `<` `const` - const generic parameters
+    //     `<` (LIFETIME|IDENT) `>` - single generic parameter
+    //     `<` (LIFETIME|IDENT) `,` - first generic parameter in a list
+    //     `<` (LIFETIME|IDENT) `:` - generic parameter with bounds
+    //     `<` (LIFETIME|IDENT) `=` - generic parameter with a default
+    // The only truly ambiguous case is
+    //     `<` IDENT `>` `::` IDENT ...
+    // we disambiguate it in favor of generics (`impl<T> ::absolute::Path<T> { ... }`)
+    // because this is what almost always expected in practice, qualified paths in impls
+    // (`impl <Type>::AssocTy { ... }`) aren't even allowed by type checker at the moment.
+    if !p.at(T![<]) {
+        return false;
+    }
+    if p.nth(1) == T![#] || p.nth(1) == T![>] || p.nth(1) == CONST_KW {
+        return true;
+    }
+    (p.nth(1) == LIFETIME || p.nth(1) == IDENT)
+        && (p.nth(2) == T![>] || p.nth(2) == T![,] || p.nth(2) == T![:] || p.nth(2) == T![=])
+}
+
+// test_err impl_type
+// impl Type {}
+// impl Trait1 for T {}
+// impl impl NotType {}
+// impl Trait2 for impl NotType {}
+pub(crate) fn impl_type(p: &mut Parser) {
+    if p.at(T![impl]) {
+        p.error("expected trait or type");
+        return;
+    }
+    types::type_(p);
+}

crates/parser/src/grammar/items/use_item.rs

@@ -0,0 +1,132 @@
+//! FIXME: write short doc here
+
+use super::*;
+
+pub(super) fn use_item(p: &mut Parser, m: Marker) {
+    assert!(p.at(T![use]));
+    p.bump(T![use]);
+    use_tree(p, true);
+    p.expect(T![;]);
+    m.complete(p, USE);
+}
+
+/// Parse a use 'tree', such as `some::path` in `use some::path;`
+/// Note that this is called both by `use_item` and `use_tree_list`,
+/// so handles both `some::path::{inner::path}` and `inner::path` in
+/// `use some::path::{inner::path};`
+fn use_tree(p: &mut Parser, top_level: bool) {
+    let m = p.start();
+    match p.current() {
+        // Finish the use_tree for cases of e.g.
+        // `use some::path::{self, *};` or `use *;`
+        // This does not handle cases such as `use some::path::*`
+        // N.B. in Rust 2015 `use *;` imports all from crate root
+        // however in Rust 2018 `use *;` errors: ('cannot glob-import all possible crates')
+        // FIXME: Add this error (if not out of scope)
+
+        // test use_star
+        // use *;
+        // use ::*;
+        // use some::path::{*};
+        // use some::path::{::*};
+        T![*] => p.bump(T![*]),
+        T![:] if p.at(T![::]) && p.nth(2) == T![*] => {
+            // Parse `use ::*;`, which imports all from the crate root in Rust 2015
+            // This is invalid inside a use_tree_list, (e.g. `use some::path::{::*}`)
+            // but still parses and errors later: ('crate root in paths can only be used in start position')
+            // FIXME: Add this error (if not out of scope)
+            // In Rust 2018, it is always invalid (see above)
+            p.bump(T![::]);
+            p.bump(T![*]);
+        }
+        // Open a use tree list
+        // Handles cases such as `use {some::path};` or `{inner::path}` in
+        // `use some::path::{{inner::path}, other::path}`
+
+        // test use_tree_list
+        // use {crate::path::from::root, or::path::from::crate_name}; // Rust 2018 (with a crate named `or`)
+        // use {path::from::root}; // Rust 2015
+        // use ::{some::arbritrary::path}; // Rust 2015
+        // use ::{{{root::export}}}; // Nonsensical but perfectly legal nesting
+        T!['{'] => {
+            use_tree_list(p);
+        }
+        T![:] if p.at(T![::]) && p.nth(2) == T!['{'] => {
+            p.bump(T![::]);
+            use_tree_list(p);
+        }
+        // Parse a 'standard' path.
+        // Also handles aliases (e.g. `use something as something_else`)
+
+        // test use_path
+        // use ::crate_name; // Rust 2018 - All flavours
+        // use crate_name; // Rust 2018 - Anchored paths
+        // use item_in_scope_or_crate_name; // Rust 2018 - Uniform Paths
+        //
+        // use self::module::Item;
+        // use crate::Item;
+        // use self::some::Struct;
+        // use crate_name::some_item;
+        _ if paths::is_use_path_start(p) => {
+            paths::use_path(p);
+            match p.current() {
+                T![as] => {
+                    // test use_alias
+                    // use some::path as some_name;
+                    // use some::{
+                    //  other::path as some_other_name,
+                    //  different::path as different_name,
+                    //  yet::another::path,
+                    //  running::out::of::synonyms::for_::different::*
+                    // };
+                    // use Trait as _;
+                    opt_alias(p);
+                }
+                T![:] if p.at(T![::]) => {
+                    p.bump(T![::]);
+                    match p.current() {
+                        T![*] => {
+                            p.bump(T![*]);
+                        }
+                        // test use_tree_list_after_path
+                        // use crate::{Item};
+                        // use self::{Item};
+                        T!['{'] => use_tree_list(p),
+                        _ => {
+                            // is this unreachable?
+                            p.error("expected `{` or `*`");
+                        }
+                    }
+                }
+                _ => (),
+            }
+        }
+        _ => {
+            m.abandon(p);
+            let msg = "expected one of `*`, `::`, `{`, `self`, `super` or an identifier";
+            if top_level {
+                p.err_recover(msg, ITEM_RECOVERY_SET);
+            } else {
+                // if we are parsing a nested tree, we have to eat a token to
+                // main balanced `{}`
+                p.err_and_bump(msg);
+            }
+            return;
+        }
+    }
+    m.complete(p, USE_TREE);
+}
+
+pub(crate) fn use_tree_list(p: &mut Parser) {
+    assert!(p.at(T!['{']));
+    let m = p.start();
+    p.bump(T!['{']);
+    while !p.at(EOF) && !p.at(T!['}']) {
+        use_tree(p, false);
+        if !p.at(T!['}']) {
+            p.expect(T![,]);
+        }
+    }
+    p.expect(T!['}']);
+    m.complete(p, USE_TREE_LIST);
+}

crates/parser/src/grammar/params.rs

@@ -0,0 +1,188 @@
+//! FIXME: write short doc here
+
+use super::*;
+
+// test param_list
+// fn a() {}
+// fn b(x: i32) {}
+// fn c(x: i32, ) {}
+// fn d(x: i32, y: ()) {}
+pub(super) fn param_list_fn_def(p: &mut Parser) {
+    list_(p, Flavor::FnDef)
+}
+
+// test param_list_opt_patterns
+// fn foo<F: FnMut(&mut Foo<'a>)>(){}
+pub(super) fn param_list_fn_trait(p: &mut Parser) {
+    list_(p, Flavor::FnTrait)
+}
+
+pub(super) fn param_list_fn_ptr(p: &mut Parser) {
+    list_(p, Flavor::FnPointer)
+}
+
+pub(super) fn param_list_closure(p: &mut Parser) {
+    list_(p, Flavor::Closure)
+}
+
+#[derive(Debug, Clone, Copy)]
+enum Flavor {
+    FnDef,   // Includes trait fn params; omitted param idents are not supported
+    FnTrait, // Params for `Fn(...)`/`FnMut(...)`/`FnOnce(...)` annotations
+    FnPointer,
+    Closure,
+}
+
+fn list_(p: &mut Parser, flavor: Flavor) {
+    use Flavor::*;
+
+    let (bra, ket) = match flavor {
+        Closure => (T![|], T![|]),
+        FnDef | FnTrait | FnPointer => (T!['('], T![')']),
+    };
+
+    let m = p.start();
+    p.bump(bra);
+
+    if let FnDef = flavor {
+        // test self_param_outer_attr
+        // fn f(#[must_use] self) {}
+        attributes::outer_attributes(p);
+        opt_self_param(p);
+    }
+
+    while !p.at(EOF) && !p.at(ket) {
+        // test param_outer_arg
+        // fn f(#[attr1] pat: Type) {}
+        attributes::outer_attributes(p);
+
+        if !p.at_ts(VALUE_PARAMETER_FIRST) {
+            p.error("expected value parameter");
+            break;
+        }
+        let param = value_parameter(p, flavor);
+        if !p.at(ket) {
+            p.expect(T![,]);
+        }
+        if let Variadic(true) = param {
+            break;
+        }
+    }
+
+    p.expect(ket);
+    m.complete(p, PARAM_LIST);
+}
+
+const VALUE_PARAMETER_FIRST: TokenSet = patterns::PATTERN_FIRST.union(types::TYPE_FIRST);
+
+struct Variadic(bool);
+
+fn value_parameter(p: &mut Parser, flavor: Flavor) -> Variadic {
+    let mut res = Variadic(false);
+    let m = p.start();
+    match flavor {
+        // test param_list_vararg
+        // extern "C" { fn printf(format: *const i8, ...) -> i32; }
+        Flavor::FnDef | Flavor::FnPointer if p.eat(T![...]) => res = Variadic(true),
+
+        // test fn_def_param
+        // fn foo((x, y): (i32, i32)) {}
+        Flavor::FnDef => {
+            patterns::pattern(p);
+            if variadic_param(p) {
+                res = Variadic(true)
+            } else {
+                types::ascription(p);
+            }
+        }
+        // test value_parameters_no_patterns
+        // type F = Box<Fn(i32, &i32, &i32, ())>;
+        Flavor::FnTrait => {
+            types::type_(p);
+        }
+        // test fn_pointer_param_ident_path
+        // type Foo = fn(Bar::Baz);
+        // type Qux = fn(baz: Bar::Baz);
+
+        // test fn_pointer_unnamed_arg
+        // type Foo = fn(_: bar);
+        Flavor::FnPointer => {
+            if (p.at(IDENT) || p.at(UNDERSCORE)) && p.nth(1) == T![:] && !p.nth_at(1, T![::]) {
+                patterns::pattern_single(p);
+                if variadic_param(p) {
+                    res = Variadic(true)
+                } else {
+                    types::ascription(p);
+                }
+            } else {
+                types::type_(p);
+            }
+        }
+        // test closure_params
+        // fn main() {
+        //    let foo = |bar, baz: Baz, qux: Qux::Quux| ();
+        // }
+        Flavor::Closure => {
+            patterns::pattern_single(p);
+            if p.at(T![:]) && !p.at(T![::]) {
+                types::ascription(p);
+            }
+        }
+    }
+    m.complete(p, PARAM);
+    res
+}
+
+fn variadic_param(p: &mut Parser) -> bool {
+    if p.at(T![:]) && p.nth_at(1, T![...]) {
+        p.bump(T![:]);
+        p.bump(T![...]);
+        true
+    } else {
+        false
+    }
+}
+
+// test self_param
+// impl S {
+//     fn a(self) {}
+//     fn b(&self,) {}
+//     fn c(&'a self,) {}
+//     fn d(&'a mut self, x: i32) {}
+//     fn e(mut self) {}
+// }
+fn opt_self_param(p: &mut Parser) {
+    let m;
+    if p.at(T![self]) || p.at(T![mut]) && p.nth(1) == T![self] {
+        m = p.start();
+        p.eat(T![mut]);
+        p.eat(T![self]);
+        // test arb_self_types
+        // impl S {
+        //     fn a(self: &Self) {}
+        //     fn b(mut self: Box<Self>) {}
+        // }
+        if p.at(T![:]) {
+            types::ascription(p);
+        }
+    } else {
+        let la1 = p.nth(1);
+        let la2 = p.nth(2);
+        let la3 = p.nth(3);
+        let n_toks = match (p.current(), la1, la2, la3) {
+            (T![&], T![self], _, _) => 2,
+            (T![&], T![mut], T![self], _) => 3,
+            (T![&], LIFETIME, T![self], _) => 3,
+            (T![&], LIFETIME, T![mut], T![self]) => 4,
+            _ => return,
+        };
+        m = p.start();
+        for _ in 0..n_toks {
+            p.bump_any();
+        }
+    }
+    m.complete(p, SELF_PARAM);
+    if !p.at(T![')']) {
+        p.expect(T![,]);
+    }
+}

crates/parser/src/grammar/paths.rs

@@ -0,0 +1,115 @@
+//! FIXME: write short doc here
+
+use super::*;
+
+pub(super) const PATH_FIRST: TokenSet =
+    token_set![IDENT, T![self], T![super], T![crate], T![:], T![<]];
+
+pub(super) fn is_path_start(p: &Parser) -> bool {
+    is_use_path_start(p) || p.at(T![<])
+}
+
+pub(super) fn is_use_path_start(p: &Parser) -> bool {
+    match p.current() {
+        IDENT | T![self] | T![super] | T![crate] => true,
+        T![:] if p.at(T![::]) => true,
+        _ => false,
+    }
+}
+
+pub(super) fn use_path(p: &mut Parser) {
+    path(p, Mode::Use)
+}
+
+pub(crate) fn type_path(p: &mut Parser) {
+    path(p, Mode::Type)
+}
+
+pub(super) fn expr_path(p: &mut Parser) {
+    path(p, Mode::Expr)
+}
+
+#[derive(Clone, Copy, Eq, PartialEq)]
+enum Mode {
+    Use,
+    Type,
+    Expr,
+}
+
+fn path(p: &mut Parser, mode: Mode) {
+    let path = p.start();
+    path_segment(p, mode, true);
+    let mut qual = path.complete(p, PATH);
+    loop {
+        let use_tree = matches!(p.nth(2), T![*] | T!['{']);
+        if p.at(T![::]) && !use_tree {
+            let path = qual.precede(p);
+            p.bump(T![::]);
+            path_segment(p, mode, false);
+            let path = path.complete(p, PATH);
+            qual = path;
+        } else {
+            break;
+        }
+    }
+}
+
+fn path_segment(p: &mut Parser, mode: Mode, first: bool) {
+    let m = p.start();
+    // test qual_paths
+    // type X = <A as B>::Output;
+    // fn foo() { <usize as Default>::default(); }
+    if first && p.eat(T![<]) {
+        types::type_(p);
+        if p.eat(T![as]) {
+            if is_use_path_start(p) {
+                types::path_type(p);
+            } else {
+                p.error("expected a trait");
+            }
+        }
+        p.expect(T![>]);
+    } else {
+        let mut empty = true;
+        if first {
+            p.eat(T![::]);
+            empty = false;
+        }
+        match p.current() {
+            IDENT => {
+                name_ref(p);
+                opt_path_type_args(p, mode);
+            }
+            // test crate_path
+            // use crate::foo;
+            T![self] | T![super] | T![crate] => p.bump_any(),
+            _ => {
+                p.err_recover("expected identifier", items::ITEM_RECOVERY_SET);
+                if empty {
+                    // test_err empty_segment
+                    // use crate::;
+                    m.abandon(p);
+                    return;
+                }
+            }
+        };
+    }
+    m.complete(p, PATH_SEGMENT);
+}
+
+fn opt_path_type_args(p: &mut Parser, mode: Mode) {
+    match mode {
+        Mode::Use => {}
+        Mode::Type => {
+            // test path_fn_trait_args
+            // type F = Box<Fn(i32) -> ()>;
+            if p.at(T!['(']) {
+                params::param_list_fn_trait(p);
+                opt_fn_ret_type(p);
+            } else {
+                type_args::opt_type_arg_list(p, false)
+            }
+        }
+        Mode::Expr => type_args::opt_type_arg_list(p, true),
+    }
+}

crates/parser/src/grammar/patterns.rs

@@ -0,0 +1,379 @@
+//! FIXME: write short doc here
+
+use super::*;
+
+pub(super) const PATTERN_FIRST: TokenSet = expressions::LITERAL_FIRST
+    .union(paths::PATH_FIRST)
+    .union(token_set![T![box], T![ref], T![mut], T!['('], T!['['], T![&], T![_], T![-], T![.]]);
+
+pub(crate) fn pattern(p: &mut Parser) {
+    pattern_r(p, PAT_RECOVERY_SET);
+}
+
+/// Parses a pattern list separated by pipes `|`
+pub(super) fn pattern_top(p: &mut Parser) {
+    pattern_top_r(p, PAT_RECOVERY_SET)
+}
+
+pub(crate) fn pattern_single(p: &mut Parser) {
+    pattern_single_r(p, PAT_RECOVERY_SET);
+}
+
+/// Parses a pattern list separated by pipes `|`
+/// using the given `recovery_set`
+pub(super) fn pattern_top_r(p: &mut Parser, recovery_set: TokenSet) {
+    p.eat(T![|]);
+    pattern_r(p, recovery_set);
+}
+
+/// Parses a pattern list separated by pipes `|`, with no leading `|`,using the
+/// given `recovery_set`
+// test or_pattern
+// fn main() {
+//     match () {
+//         (_ | _) => (),
+//         &(_ | _) => (),
+//         (_ | _,) => (),
+//         [_ | _,] => (),
+//     }
+// }
+fn pattern_r(p: &mut Parser, recovery_set: TokenSet) {
+    let m = p.start();
+    pattern_single_r(p, recovery_set);
+
+    if !p.at(T![|]) {
+        m.abandon(p);
+        return;
+    }
+    while p.eat(T![|]) {
+        pattern_single_r(p, recovery_set);
+    }
+    m.complete(p, OR_PAT);
+}
+
+fn pattern_single_r(p: &mut Parser, recovery_set: TokenSet) {
+    if let Some(lhs) = atom_pat(p, recovery_set) {
+        // test range_pat
+        // fn main() {
+        //     match 92 {
+        //         0 ... 100 => (),
+        //         101 ..= 200 => (),
+        //         200 .. 301=> (),
+        //     }
+        // }
+        for &range_op in [T![...], T![..=], T![..]].iter() {
+            if p.at(range_op) {
+                let m = lhs.precede(p);
+                p.bump(range_op);
+                atom_pat(p, recovery_set);
+                m.complete(p, RANGE_PAT);
+                return;
+            }
+        }
+    }
+}
+
+const PAT_RECOVERY_SET: TokenSet =
+    token_set![LET_KW, IF_KW, WHILE_KW, LOOP_KW, MATCH_KW, R_PAREN, COMMA];
+
+fn atom_pat(p: &mut Parser, recovery_set: TokenSet) -> Option<CompletedMarker> {
+    let m = match p.nth(0) {
+        T![box] => box_pat(p),
+        T![ref] | T![mut] => bind_pat(p, true),
+        IDENT => match p.nth(1) {
+            // Checks the token after an IDENT to see if a pattern is a path (Struct { .. }) or macro
+            // (T![x]).
+            T!['('] | T!['{'] | T![!] => path_or_macro_pat(p),
+            T![:] if p.nth_at(1, T![::]) => path_or_macro_pat(p),
+            _ => bind_pat(p, true),
+        },
+
+        // test type_path_in_pattern
+        // fn main() { let <_>::Foo = (); }
+        _ if paths::is_path_start(p) => path_or_macro_pat(p),
+        _ if is_literal_pat_start(p) => literal_pat(p),
+
+        T![.] if p.at(T![..]) => dot_dot_pat(p),
+        T![_] => placeholder_pat(p),
+        T![&] => ref_pat(p),
+        T!['('] => tuple_pat(p),
+        T!['['] => slice_pat(p),
+
+        _ => {
+            p.err_recover("expected pattern", recovery_set);
+            return None;
+        }
+    };
+
+    Some(m)
+}
+
+fn is_literal_pat_start(p: &Parser) -> bool {
+    p.at(T![-]) && (p.nth(1) == INT_NUMBER || p.nth(1) == FLOAT_NUMBER)
+        || p.at_ts(expressions::LITERAL_FIRST)
+}
+
+// test literal_pattern
+// fn main() {
+//     match () {
+//         -1 => (),
+//         92 => (),
+//         'c' => (),
+//         "hello" => (),
+//     }
+// }
+fn literal_pat(p: &mut Parser) -> CompletedMarker {
+    assert!(is_literal_pat_start(p));
+    let m = p.start();
+    if p.at(T![-]) {
+        p.bump(T![-]);
+    }
+    expressions::literal(p);
+    m.complete(p, LITERAL_PAT)
+}
+
+// test path_part
+// fn foo() {
+//     let foo::Bar = ();
+//     let ::Bar = ();
+//     let Bar { .. } = ();
+//     let Bar(..) = ();
+// }
+fn path_or_macro_pat(p: &mut Parser) -> CompletedMarker {
+    assert!(paths::is_path_start(p));
+    let m = p.start();
+    paths::expr_path(p);
+    let kind = match p.current() {
+        T!['('] => {
+            tuple_pat_fields(p);
+            TUPLE_STRUCT_PAT
+        }
+        T!['{'] => {
+            record_field_pat_list(p);
+            RECORD_PAT
+        }
+        // test marco_pat
+        // fn main() {
+        //     let m!(x) = 0;
+        // }
+        T![!] => {
+            items::macro_call_after_excl(p);
+            return m.complete(p, MACRO_CALL).precede(p).complete(p, MACRO_PAT);
+        }
+        _ => PATH_PAT,
+    };
+    m.complete(p, kind)
+}
+
+// test tuple_pat_fields
+// fn foo() {
+//     let S() = ();
+//     let S(_) = ();
+//     let S(_,) = ();
+//     let S(_, .. , x) = ();
+// }
+fn tuple_pat_fields(p: &mut Parser) {
+    assert!(p.at(T!['(']));
+    p.bump(T!['(']);
+    pat_list(p, T![')']);
+    p.expect(T![')']);
+}
+
+// test record_field_pat_list
+// fn foo() {
+//     let S {} = ();
+//     let S { f, ref mut g } = ();
+//     let S { h: _, ..} = ();
+//     let S { h: _, } = ();
+// }
+fn record_field_pat_list(p: &mut Parser) {
+    assert!(p.at(T!['{']));
+    let m = p.start();
+    p.bump(T!['{']);
+    while !p.at(EOF) && !p.at(T!['}']) {
+        match p.current() {
+            // A trailing `..` is *not* treated as a REST_PAT.
+            T![.] if p.at(T![..]) => p.bump(T![..]),
+            T!['{'] => error_block(p, "expected ident"),
+
+            c => {
+                let m = p.start();
+                match c {
+                    // test record_field_pat
+                    // fn foo() {
+                    //     let S { 0: 1 } = ();
+                    //     let S { x: 1 } = ();
+                    // }
+                    IDENT | INT_NUMBER if p.nth(1) == T![:] => {
+                        name_ref_or_index(p);
+                        p.bump(T![:]);
+                        pattern(p);
+                    }
+                    T![box] => {
+                        // FIXME: not all box patterns should be allowed
+                        box_pat(p);
+                    }
+                    _ => {
+                        bind_pat(p, false);
+                    }
+                }
+                m.complete(p, RECORD_PAT_FIELD);
+            }
+        }
+        if !p.at(T!['}']) {
+            p.expect(T![,]);
+        }
+    }
+    p.expect(T!['}']);
+    m.complete(p, RECORD_PAT_FIELD_LIST);
+}
+
+// test placeholder_pat
+// fn main() { let _ = (); }
+fn placeholder_pat(p: &mut Parser) -> CompletedMarker {
+    assert!(p.at(T![_]));
+    let m = p.start();
+    p.bump(T![_]);
+    m.complete(p, WILDCARD_PAT)
+}
+
+// test dot_dot_pat
+// fn main() {
+//     let .. = ();
+//     //
+//     // Tuples
+//     //
+//     let (a, ..) = ();
+//     let (a, ..,) = ();
+//     let Tuple(a, ..) = ();
+//     let Tuple(a, ..,) = ();
+//     let (.., ..) = ();
+//     let Tuple(.., ..) = ();
+//     let (.., a, ..) = ();
+//     let Tuple(.., a, ..) = ();
+//     //
+//     // Slices
+//     //
+//     let [..] = ();
+//     let [head, ..] = ();
+//     let [head, tail @ ..] = ();
+//     let [head, .., cons] = ();
+//     let [head, mid @ .., cons] = ();
+//     let [head, .., .., cons] = ();
+//     let [head, .., mid, tail @ ..] = ();
+//     let [head, .., mid, .., cons] = ();
+// }
+fn dot_dot_pat(p: &mut Parser) -> CompletedMarker {
+    assert!(p.at(T![..]));
+    let m = p.start();
+    p.bump(T![..]);
+    m.complete(p, REST_PAT)
+}
+
+// test ref_pat
+// fn main() {
+//     let &a = ();
+//     let &mut b = ();
+// }
+fn ref_pat(p: &mut Parser) -> CompletedMarker {
+    assert!(p.at(T![&]));
+    let m = p.start();
+    p.bump(T![&]);
+    p.eat(T![mut]);
+    pattern_single(p);
+    m.complete(p, REF_PAT)
+}
+
+// test tuple_pat
+// fn main() {
+//     let (a, b, ..) = ();
+//     let (a,) = ();
+//     let (..) = ();
+//     let () = ();
+// }
+fn tuple_pat(p: &mut Parser) -> CompletedMarker {
+    assert!(p.at(T!['(']));
+    let m = p.start();
+    p.bump(T!['(']);
+    let mut has_comma = false;
+    let mut has_pat = false;
+    let mut has_rest = false;
+    while !p.at(EOF) && !p.at(T![')']) {
+        has_pat = true;
+        if !p.at_ts(PATTERN_FIRST) {
+            p.error("expected a pattern");
+            break;
+        }
+        has_rest |= p.at(T![..]);
+
+        pattern(p);
+        if !p.at(T![')']) {
+            has_comma = true;
+            p.expect(T![,]);
+        }
+    }
+    p.expect(T![')']);
+
+    m.complete(p, if !has_comma && !has_rest && has_pat { PAREN_PAT } else { TUPLE_PAT })
+}
+
+// test slice_pat
+// fn main() {
+//     let [a, b, ..] = [];
+// }
+fn slice_pat(p: &mut Parser) -> CompletedMarker {
+    assert!(p.at(T!['[']));
+    let m = p.start();
+    p.bump(T!['[']);
+    pat_list(p, T![']']);
+    p.expect(T![']']);
+    m.complete(p, SLICE_PAT)
+}
+
+fn pat_list(p: &mut Parser, ket: SyntaxKind) {
+    while !p.at(EOF) && !p.at(ket) {
+        if !p.at_ts(PATTERN_FIRST) {
+            p.error("expected a pattern");
+            break;
+        }
+
+        pattern(p);
+        if !p.at(ket) {
+            p.expect(T![,]);
+        }
+    }
+}
+
+// test bind_pat
+// fn main() {
+//     let a = ();
+//     let mut b = ();
+//     let ref c = ();
+//     let ref mut d = ();
+//     let e @ _ = ();
+//     let ref mut f @ g @ _ = ();
+// }
+fn bind_pat(p: &mut Parser, with_at: bool) -> CompletedMarker {
+    let m = p.start();
+    p.eat(T![ref]);
+    p.eat(T![mut]);
+    name(p);
+    if with_at && p.eat(T![@]) {
+        pattern_single(p);
+    }
+    m.complete(p, IDENT_PAT)
+}
+
+// test box_pat
+// fn main() {
+//     let box i = ();
+//     let box Outer { box i, j: box Inner(box &x) } = ();
+//     let box ref mut i = ();
+// }
+fn box_pat(p: &mut Parser) -> CompletedMarker {
+    assert!(p.at(T![box]));
+    let m = p.start();
+    p.bump(T![box]);
+    pattern_single(p);
+    m.complete(p, BOX_PAT)
+}

crates/parser/src/grammar/type_args.rs

@@ -0,0 +1,63 @@
+//! FIXME: write short doc here
+
+use super::*;
+
+pub(super) fn opt_type_arg_list(p: &mut Parser, colon_colon_required: bool) {
+    let m;
+    if p.at(T![::]) && p.nth(2) == T![<] {
+        m = p.start();
+        p.bump(T![::]);
+        p.bump(T![<]);
+    } else if !colon_colon_required && p.at(T![<]) && p.nth(1) != T![=] {
+        m = p.start();
+        p.bump(T![<]);
+    } else {
+        return;
+    }
+
+    while !p.at(EOF) && !p.at(T![>]) {
+        type_arg(p);
+        if !p.at(T![>]) && !p.expect(T![,]) {
+            break;
+        }
+    }
+    p.expect(T![>]);
+    m.complete(p, GENERIC_ARG_LIST);
+}
+
+// test type_arg
+// type A = B<'static, i32, 1, { 2 }, Item=u64>;
+fn type_arg(p: &mut Parser) {
+    let m = p.start();
+    match p.current() {
+        LIFETIME => {
+            p.bump(LIFETIME);
+            m.complete(p, LIFETIME_ARG);
+        }
+        // test associated_type_bounds
+        // fn print_all<T: Iterator<Item: Display>>(printables: T) {}
+        IDENT if p.nth(1) == T![:] && p.nth(2) != T![:] => {
+            name_ref(p);
+            type_params::bounds(p);
+            m.complete(p, ASSOC_TYPE_ARG);
+        }
+        IDENT if p.nth(1) == T![=] => {
+            name_ref(p);
+            p.bump_any();
+            types::type_(p);
+            m.complete(p, ASSOC_TYPE_ARG);
+        }
+        T!['{'] => {
+            expressions::block_expr(p);
+            m.complete(p, CONST_ARG);
+        }
+        k if k.is_literal() => {
+            expressions::literal(p);
+            m.complete(p, CONST_ARG);
+        }
+        _ => {
+            types::type_(p);
+            m.complete(p, TYPE_ARG);
+        }
+    }
+}

crates/parser/src/grammar/type_params.rs

@@ -0,0 +1,209 @@
+//! FIXME: write short doc here
+
+use super::*;
+
+pub(super) fn opt_type_param_list(p: &mut Parser) {
+    if !p.at(T![<]) {
+        return;
+    }
+    type_param_list(p);
+}
+
+fn type_param_list(p: &mut Parser) {
+    assert!(p.at(T![<]));
+    let m = p.start();
+    p.bump(T![<]);
+
+    while !p.at(EOF) && !p.at(T![>]) {
+        let m = p.start();
+
+        // test generic_lifetime_type_attribute
+        // fn foo<#[derive(Lifetime)] 'a, #[derive(Type)] T>(_: &'a T) {
+        // }
+        attributes::outer_attributes(p);
+
+        match p.current() {
+            LIFETIME => lifetime_param(p, m),
+            IDENT => type_param(p, m),
+            CONST_KW => type_const_param(p, m),
+            _ => {
+                m.abandon(p);
+                p.err_and_bump("expected type parameter")
+            }
+        }
+        if !p.at(T![>]) && !p.expect(T![,]) {
+            break;
+        }
+    }
+    p.expect(T![>]);
+    m.complete(p, GENERIC_PARAM_LIST);
+}
+
+fn lifetime_param(p: &mut Parser, m: Marker) {
+    assert!(p.at(LIFETIME));
+    p.bump(LIFETIME);
+    if p.at(T![:]) {
+        lifetime_bounds(p);
+    }
+    m.complete(p, LIFETIME_PARAM);
+}
+
+fn type_param(p: &mut Parser, m: Marker) {
+    assert!(p.at(IDENT));
+    name(p);
+    if p.at(T![:]) {
+        bounds(p);
+    }
+    // test type_param_default
+    // struct S<T = i32>;
+    if p.at(T![=]) {
+        p.bump(T![=]);
+        types::type_(p)
+    }
+    m.complete(p, TYPE_PARAM);
+}
+
+// test const_param
+// struct S<const N: u32>;
+fn type_const_param(p: &mut Parser, m: Marker) {
+    assert!(p.at(CONST_KW));
+    p.bump(T![const]);
+    name(p);
+    types::ascription(p);
+    m.complete(p, CONST_PARAM);
+}
+
+// test type_param_bounds
+// struct S<T: 'a + ?Sized + (Copy)>;
+pub(super) fn bounds(p: &mut Parser) {
+    assert!(p.at(T![:]));
+    p.bump(T![:]);
+    bounds_without_colon(p);
+}
+
+fn lifetime_bounds(p: &mut Parser) {
+    assert!(p.at(T![:]));
+    p.bump(T![:]);
+    while p.at(LIFETIME) {
+        p.bump(LIFETIME);
+        if !p.eat(T![+]) {
+            break;
+        }
+    }
+}
+
+pub(super) fn bounds_without_colon_m(p: &mut Parser, marker: Marker) -> CompletedMarker {
+    while type_bound(p) {
+        if !p.eat(T![+]) {
+            break;
+        }
+    }
+
+    marker.complete(p, TYPE_BOUND_LIST)
+}
+
+pub(super) fn bounds_without_colon(p: &mut Parser) {
+    let m = p.start();
+    bounds_without_colon_m(p, m);
+}
+
+fn type_bound(p: &mut Parser) -> bool {
+    let m = p.start();
+    let has_paren = p.eat(T!['(']);
+    p.eat(T![?]);
+    match p.current() {
+        LIFETIME => p.bump(LIFETIME),
+        T![for] => types::for_type(p),
+        _ if paths::is_use_path_start(p) => types::path_type_(p, false),
+        _ => {
+            m.abandon(p);
+            return false;
+        }
+    }
+    if has_paren {
+        p.expect(T![')']);
+    }
+    m.complete(p, TYPE_BOUND);
+
+    true
+}
+
+// test where_clause
+// fn foo()
+// where
+//    'a: 'b + 'c,
+//    T: Clone + Copy + 'static,
+//    Iterator::Item: 'a,
+//    <T as Iterator>::Item: 'a
+// {}
+pub(super) fn opt_where_clause(p: &mut Parser) {
+    if !p.at(T![where]) {
+        return;
+    }
+    let m = p.start();
+    p.bump(T![where]);
+
+    while is_where_predicate(p) {
+        where_predicate(p);
+
+        let comma = p.eat(T![,]);
+
+        if is_where_clause_end(p) {
+            break;
+        }
+
+        if !comma {
+            p.error("expected comma");
+        }
+    }
+
+    m.complete(p, WHERE_CLAUSE);
+}
+
+fn is_where_predicate(p: &mut Parser) -> bool {
+    match p.current() {
+        LIFETIME => true,
+        T![impl] => false,
+        token => types::TYPE_FIRST.contains(token),
+    }
+}
+
+fn is_where_clause_end(p: &mut Parser) -> bool {
+    matches!(p.current(), T!['{'] | T![;] | T![=])
+}
+
+fn where_predicate(p: &mut Parser) {
+    let m = p.start();
+    match p.current() {
+        LIFETIME => {
+            p.bump(LIFETIME);
+            if p.at(T![:]) {
+                bounds(p);
+            } else {
+                p.error("expected colon");
+            }
+        }
+        T![impl] => {
+            p.error("expected lifetime or type");
+        }
+        _ => {
+            // test where_pred_for
+            // fn for_trait<F>()
+            // where
+            //    for<'a> F: Fn(&'a str)
+            // { }
+            if p.at(T![for]) {
+                types::for_binder(p);
+            }
+
+            types::type_(p);
+
+            if p.at(T![:]) {
+                bounds(p);
+            } else {
+                p.error("expected colon");
+            }
+        }
+    }
+    m.complete(p, WHERE_PRED);
+}

crates/parser/src/grammar/types.rs

@@ -0,0 +1,324 @@
+//! FIXME: write short doc here
+
+use super::*;
+
+pub(super) const TYPE_FIRST: TokenSet = paths::PATH_FIRST.union(token_set![
+    T!['('],
+    T!['['],
+    T![<],
+    T![!],
+    T![*],
+    T![&],
+    T![_],
+    T![fn],
+    T![unsafe],
+    T![extern],
+    T![for],
+    T![impl],
+    T![dyn],
+]);
+
+const TYPE_RECOVERY_SET: TokenSet = token_set![R_PAREN, COMMA, L_DOLLAR];
+
+pub(crate) fn type_(p: &mut Parser) {
+    type_with_bounds_cond(p, true);
+}
+
+pub(super) fn type_no_bounds(p: &mut Parser) {
+    type_with_bounds_cond(p, false);
+}
+
+fn type_with_bounds_cond(p: &mut Parser, allow_bounds: bool) {
+    match p.current() {
+        T!['('] => paren_or_tuple_type(p),
+        T![!] => never_type(p),
+        T![*] => pointer_type(p),
+        T!['['] => array_or_slice_type(p),
+        T![&] => reference_type(p),
+        T![_] => placeholder_type(p),
+        T![fn] | T![unsafe] | T![extern] => fn_pointer_type(p),
+        T![for] => for_type(p),
+        T![impl] => impl_trait_type(p),
+        T![dyn] => dyn_trait_type(p),
+        // Some path types are not allowed to have bounds (no plus)
+        T![<] => path_type_(p, allow_bounds),
+        _ if paths::is_use_path_start(p) => path_or_macro_type_(p, allow_bounds),
+        _ => {
+            p.err_recover("expected type", TYPE_RECOVERY_SET);
+        }
+    }
+}
+
+pub(super) fn ascription(p: &mut Parser) {
+    p.expect(T![:]);
+    type_(p)
+}
+
+fn paren_or_tuple_type(p: &mut Parser) {
+    assert!(p.at(T!['(']));
+    let m = p.start();
+    p.bump(T!['(']);
+    let mut n_types: u32 = 0;
+    let mut trailing_comma: bool = false;
+    while !p.at(EOF) && !p.at(T![')']) {
+        n_types += 1;
+        type_(p);
+        if p.eat(T![,]) {
+            trailing_comma = true;
+        } else {
+            trailing_comma = false;
+            break;
+        }
+    }
+    p.expect(T![')']);
+
+    let kind = if n_types == 1 && !trailing_comma {
+        // test paren_type
+        // type T = (i32);
+        PAREN_TYPE
+    } else {
+        // test unit_type
+        // type T = ();
+
+        // test singleton_tuple_type
+        // type T = (i32,);
+        TUPLE_TYPE
+    };
+    m.complete(p, kind);
+}
+
+// test never_type
+// type Never = !;
+fn never_type(p: &mut Parser) {
+    assert!(p.at(T![!]));
+    let m = p.start();
+    p.bump(T![!]);
+    m.complete(p, NEVER_TYPE);
+}
+
+fn pointer_type(p: &mut Parser) {
+    assert!(p.at(T![*]));
+    let m = p.start();
+    p.bump(T![*]);
+
+    match p.current() {
+        // test pointer_type_mut
+        // type M = *mut ();
+        // type C = *mut ();
+        T![mut] | T![const] => p.bump_any(),
+        _ => {
+            // test_err pointer_type_no_mutability
+            // type T = *();
+            p.error(
+                "expected mut or const in raw pointer type \
+                 (use `*mut T` or `*const T` as appropriate)",
+            );
+        }
+    };
+
+    type_no_bounds(p);
+    m.complete(p, PTR_TYPE);
+}
+
+fn array_or_slice_type(p: &mut Parser) {
+    assert!(p.at(T!['[']));
+    let m = p.start();
+    p.bump(T!['[']);
+
+    type_(p);
+    let kind = match p.current() {
+        // test slice_type
+        // type T = [()];
+        T![']'] => {
+            p.bump(T![']']);
+            SLICE_TYPE
+        }
+
+        // test array_type
+        // type T = [(); 92];
+        T![;] => {
+            p.bump(T![;]);
+            expressions::expr(p);
+            p.expect(T![']']);
+            ARRAY_TYPE
+        }
+        // test_err array_type_missing_semi
+        // type T = [() 92];
+        _ => {
+            p.error("expected `;` or `]`");
+            SLICE_TYPE
+        }
+    };
+    m.complete(p, kind);
+}
+
+// test reference_type;
+// type A = &();
+// type B = &'static ();
+// type C = &mut ();
+fn reference_type(p: &mut Parser) {
+    assert!(p.at(T![&]));
+    let m = p.start();
+    p.bump(T![&]);
+    p.eat(LIFETIME);
+    p.eat(T![mut]);
+    type_no_bounds(p);
+    m.complete(p, REF_TYPE);
+}
+
+// test placeholder_type
+// type Placeholder = _;
+fn placeholder_type(p: &mut Parser) {
+    assert!(p.at(T![_]));
+    let m = p.start();
+    p.bump(T![_]);
+    m.complete(p, INFER_TYPE);
+}
+
+// test fn_pointer_type
+// type A = fn();
+// type B = unsafe fn();
+// type C = unsafe extern "C" fn();
+// type D = extern "C" fn ( u8 , ... ) -> u8;
+fn fn_pointer_type(p: &mut Parser) {
+    let m = p.start();
+    p.eat(T![unsafe]);
+    if p.at(T![extern]) {
+        abi(p);
+    }
+    // test_err fn_pointer_type_missing_fn
+    // type F = unsafe ();
+    if !p.eat(T![fn]) {
+        m.abandon(p);
+        p.error("expected `fn`");
+        return;
+    }
+    if p.at(T!['(']) {
+        params::param_list_fn_ptr(p);
+    } else {
+        p.error("expected parameters")
+    }
+    // test fn_pointer_type_with_ret
+    // type F = fn() -> ();
+    opt_fn_ret_type(p);
+    m.complete(p, FN_PTR_TYPE);
+}
+
+pub(super) fn for_binder(p: &mut Parser) {
+    assert!(p.at(T![for]));
+    p.bump(T![for]);
+    if p.at(T![<]) {
+        type_params::opt_type_param_list(p);
+    } else {
+        p.error("expected `<`");
+    }
+}
+
+// test for_type
+// type A = for<'a> fn() -> ();
+// type B = for<'a> unsafe extern "C" fn(&'a ()) -> ();
+// type Obj = for<'a> PartialEq<&'a i32>;
+pub(super) fn for_type(p: &mut Parser) {
+    assert!(p.at(T![for]));
+    let m = p.start();
+    for_binder(p);
+    match p.current() {
+        T![fn] | T![unsafe] | T![extern] => {}
+        // OK: legacy trait object format
+        _ if paths::is_use_path_start(p) => {}
+        _ => {
+            p.error("expected a function pointer or path");
+        }
+    }
+    type_no_bounds(p);
+    m.complete(p, FOR_TYPE);
+}
+
+// test impl_trait_type
+// type A = impl Iterator<Item=Foo<'a>> + 'a;
+fn impl_trait_type(p: &mut Parser) {
+    assert!(p.at(T![impl]));
+    let m = p.start();
+    p.bump(T![impl]);
+    type_params::bounds_without_colon(p);
+    m.complete(p, IMPL_TRAIT_TYPE);
+}
+
+// test dyn_trait_type
+// type A = dyn Iterator<Item=Foo<'a>> + 'a;
+fn dyn_trait_type(p: &mut Parser) {
+    assert!(p.at(T![dyn]));
+    let m = p.start();
+    p.bump(T![dyn]);
+    type_params::bounds_without_colon(p);
+    m.complete(p, DYN_TRAIT_TYPE);
+}
+
+// test path_type
+// type A = Foo;
+// type B = ::Foo;
+// type C = self::Foo;
+// type D = super::Foo;
+pub(super) fn path_type(p: &mut Parser) {
+    path_type_(p, true)
+}
+
+// test macro_call_type
+// type A = foo!();
+// type B = crate::foo!();
+fn path_or_macro_type_(p: &mut Parser, allow_bounds: bool) {
+    assert!(paths::is_path_start(p));
+    let m = p.start();
+    paths::type_path(p);
+
+    let kind = if p.at(T![!]) && !p.at(T![!=]) {
+        items::macro_call_after_excl(p);
+        MACRO_CALL
+    } else {
+        PATH_TYPE
+    };
+
+    let path = m.complete(p, kind);
+
+    if allow_bounds {
+        opt_path_type_bounds_as_dyn_trait_type(p, path);
+    }
+}
+
+pub(super) fn path_type_(p: &mut Parser, allow_bounds: bool) {
+    assert!(paths::is_path_start(p));
+    let m = p.start();
+    paths::type_path(p);
+
+    // test path_type_with_bounds
+    // fn foo() -> Box<T + 'f> {}
+    // fn foo() -> Box<dyn T + 'f> {}
+    let path = m.complete(p, PATH_TYPE);
+    if allow_bounds {
+        opt_path_type_bounds_as_dyn_trait_type(p, path);
+    }
+}
+
+/// This turns a parsed PATH_TYPE optionally into a DYN_TRAIT_TYPE
+/// with a TYPE_BOUND_LIST
+fn opt_path_type_bounds_as_dyn_trait_type(p: &mut Parser, path_type_marker: CompletedMarker) {
+    if !p.at(T![+]) {
+        return;
+    }
+
+    // First create a TYPE_BOUND from the completed PATH_TYPE
+    let m = path_type_marker.precede(p).complete(p, TYPE_BOUND);
+
+    // Next setup a marker for the TYPE_BOUND_LIST
+    let m = m.precede(p);
+
+    // This gets consumed here so it gets properly set
+    // in the TYPE_BOUND_LIST
+    p.eat(T![+]);
+
+    // Parse rest of the bounds into the TYPE_BOUND_LIST
+    let m = type_params::bounds_without_colon_m(p, m);
+
+    // Finally precede everything with DYN_TRAIT_TYPE
+    m.precede(p).complete(p, DYN_TRAIT_TYPE);
+}

crates/parser/src/lib.rs

@@ -0,0 +1,149 @@
+//! The Rust parser.
+//!
+//! The parser doesn't know about concrete representation of tokens and syntax
+//! trees. Abstract `TokenSource` and `TreeSink` traits are used instead. As a
+//! consequence, this crates does not contain a lexer.
+//!
+//! The `Parser` struct from the `parser` module is a cursor into the sequence
+//! of tokens. Parsing routines use `Parser` to inspect current state and
+//! advance the parsing.
+//!
+//! The actual parsing happens in the `grammar` module.
+//!
+//! Tests for this crate live in `ra_syntax` crate.
+
+#[macro_use]
+mod token_set;
+#[macro_use]
+mod syntax_kind;
+mod event;
+mod parser;
+mod grammar;
+
+pub(crate) use token_set::TokenSet;
+
+pub use syntax_kind::SyntaxKind;
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct ParseError(pub Box<String>);
+
+/// `TokenSource` abstracts the source of the tokens parser operates on.
+///
+/// Hopefully this will allow us to treat text and token trees in the same way!
+pub trait TokenSource {
+    fn current(&self) -> Token;
+
+    /// Lookahead n token
+    fn lookahead_nth(&self, n: usize) -> Token;
+
+    /// bump cursor to next token
+    fn bump(&mut self);
+
+    /// Is the current token a specified keyword?
+    fn is_keyword(&self, kw: &str) -> bool;
+}
+
+/// `Token` abstracts the cursor of `TokenSource` operates on.
+#[derive(Debug, Copy, Clone, Eq, PartialEq)]
+pub struct Token {
+    /// What is the current token?
+    pub kind: SyntaxKind,
+
+    /// Is the current token joined to the next one (`> >` vs `>>`).
+    pub is_jointed_to_next: bool,
+}
+
+/// `TreeSink` abstracts details of a particular syntax tree implementation.
+pub trait TreeSink {
+    /// Adds new token to the current branch.
+    fn token(&mut self, kind: SyntaxKind, n_tokens: u8);
+
+    /// Start new branch and make it current.
+    fn start_node(&mut self, kind: SyntaxKind);
+
+    /// Finish current branch and restore previous
+    /// branch as current.
+    fn finish_node(&mut self);
+
+    fn error(&mut self, error: ParseError);
+}
+
+fn parse_from_tokens<F>(token_source: &mut dyn TokenSource, tree_sink: &mut dyn TreeSink, f: F)
+where
+    F: FnOnce(&mut parser::Parser),
+{
+    let mut p = parser::Parser::new(token_source);
+    f(&mut p);
+    let events = p.finish();
+    event::process(tree_sink, events);
+}
+
+/// Parse given tokens into the given sink as a rust file.
+pub fn parse(token_source: &mut dyn TokenSource, tree_sink: &mut dyn TreeSink) {
+    parse_from_tokens(token_source, tree_sink, grammar::root);
+}
+
+#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)]
+pub enum FragmentKind {
+    Path,
+    Expr,
+    Statement,
+    Type,
+    Pattern,
+    Item,
+    Block,
+    Visibility,
+    MetaItem,
+
+    // These kinds are used when parsing the result of expansion
+    // FIXME: use separate fragment kinds for macro inputs and outputs?
+    Items,
+    Statements,
+}
+
+pub fn parse_fragment(
+    token_source: &mut dyn TokenSource,
+    tree_sink: &mut dyn TreeSink,
+    fragment_kind: FragmentKind,
+) {
+    let parser: fn(&'_ mut parser::Parser) = match fragment_kind {
+        FragmentKind::Path => grammar::fragments::path,
+        FragmentKind::Expr => grammar::fragments::expr,
+        FragmentKind::Type => grammar::fragments::type_,
+        FragmentKind::Pattern => grammar::fragments::pattern,
+        FragmentKind::Item => grammar::fragments::item,
+        FragmentKind::Block => grammar::fragments::block_expr,
+        FragmentKind::Visibility => grammar::fragments::opt_visibility,
+        FragmentKind::MetaItem => grammar::fragments::meta_item,
+        FragmentKind::Statement => grammar::fragments::stmt,
+        FragmentKind::Items => grammar::fragments::macro_items,
+        FragmentKind::Statements => grammar::fragments::macro_stmts,
+    };
+    parse_from_tokens(token_source, tree_sink, parser)
+}
+
+/// A parsing function for a specific braced-block.
+pub struct Reparser(fn(&mut parser::Parser));
+
+impl Reparser {
+    /// If the node is a braced block, return the corresponding `Reparser`.
+    pub fn for_node(
+        node: SyntaxKind,
+        first_child: Option<SyntaxKind>,
+        parent: Option<SyntaxKind>,
+    ) -> Option<Reparser> {
+        grammar::reparser(node, first_child, parent).map(Reparser)
+    }
+
+    /// Re-parse given tokens using this `Reparser`.
+    ///
+    /// Tokens must start with `{`, end with `}` and form a valid brace
+    /// sequence.
+    pub fn parse(self, token_source: &mut dyn TokenSource, tree_sink: &mut dyn TreeSink) {
+        let Reparser(r) = self;
+        let mut p = parser::Parser::new(token_source);
+        r(&mut p);
+        let events = p.finish();
+        event::process(tree_sink, events);
+    }
+}

crates/parser/src/parser.rs

@@ -0,0 +1,350 @@
+//! FIXME: write short doc here
+
+use std::cell::Cell;
+
+use drop_bomb::DropBomb;
+
+use crate::{
+    event::Event,
+    ParseError,
+    SyntaxKind::{self, EOF, ERROR, TOMBSTONE},
+    TokenSet, TokenSource, T,
+};
+
+/// `Parser` struct provides the low-level API for
+/// navigating through the stream of tokens and
+/// constructing the parse tree. The actual parsing
+/// happens in the `grammar` module.
+///
+/// However, the result of this `Parser` is not a real
+/// tree, but rather a flat stream of events of the form
+/// "start expression, consume number literal,
+/// finish expression". See `Event` docs for more.
+pub(crate) struct Parser<'t> {
+    token_source: &'t mut dyn TokenSource,
+    events: Vec<Event>,
+    steps: Cell<u32>,
+}
+
+impl<'t> Parser<'t> {
+    pub(super) fn new(token_source: &'t mut dyn TokenSource) -> Parser<'t> {
+        Parser { token_source, events: Vec::new(), steps: Cell::new(0) }
+    }
+
+    pub(crate) fn finish(self) -> Vec<Event> {
+        self.events
+    }
+
+    /// Returns the kind of the current token.
+    /// If parser has already reached the end of input,
+    /// the special `EOF` kind is returned.
+    pub(crate) fn current(&self) -> SyntaxKind {
+        self.nth(0)
+    }
+
+    /// Lookahead operation: returns the kind of the next nth
+    /// token.
+    pub(crate) fn nth(&self, n: usize) -> SyntaxKind {
+        assert!(n <= 3);
+
+        let steps = self.steps.get();
+        assert!(steps <= 10_000_000, "the parser seems stuck");
+        self.steps.set(steps + 1);
+
+        self.token_source.lookahead_nth(n).kind
+    }
+
+    /// Checks if the current token is `kind`.
+    pub(crate) fn at(&self, kind: SyntaxKind) -> bool {
+        self.nth_at(0, kind)
+    }
+
+    pub(crate) fn nth_at(&self, n: usize, kind: SyntaxKind) -> bool {
+        match kind {
+            T![-=] => self.at_composite2(n, T![-], T![=]),
+            T![->] => self.at_composite2(n, T![-], T![>]),
+            T![::] => self.at_composite2(n, T![:], T![:]),
+            T![!=] => self.at_composite2(n, T![!], T![=]),
+            T![..] => self.at_composite2(n, T![.], T![.]),
+            T![*=] => self.at_composite2(n, T![*], T![=]),
+            T![/=] => self.at_composite2(n, T![/], T![=]),
+            T![&&] => self.at_composite2(n, T![&], T![&]),
+            T![&=] => self.at_composite2(n, T![&], T![=]),
+            T![%=] => self.at_composite2(n, T![%], T![=]),
+            T![^=] => self.at_composite2(n, T![^], T![=]),
+            T![+=] => self.at_composite2(n, T![+], T![=]),
+            T![<<] => self.at_composite2(n, T![<], T![<]),
+            T![<=] => self.at_composite2(n, T![<], T![=]),
+            T![==] => self.at_composite2(n, T![=], T![=]),
+            T![=>] => self.at_composite2(n, T![=], T![>]),
+            T![>=] => self.at_composite2(n, T![>], T![=]),
+            T![>>] => self.at_composite2(n, T![>], T![>]),
+            T![|=] => self.at_composite2(n, T![|], T![=]),
+            T![||] => self.at_composite2(n, T![|], T![|]),
+
+            T![...] => self.at_composite3(n, T![.], T![.], T![.]),
+            T![..=] => self.at_composite3(n, T![.], T![.], T![=]),
+            T![<<=] => self.at_composite3(n, T![<], T![<], T![=]),
+            T![>>=] => self.at_composite3(n, T![>], T![>], T![=]),
+
+            _ => self.token_source.lookahead_nth(n).kind == kind,
+        }
+    }
+
+    /// Consume the next token if `kind` matches.
+    pub(crate) fn eat(&mut self, kind: SyntaxKind) -> bool {
+        if !self.at(kind) {
+            return false;
+        }
+        let n_raw_tokens = match kind {
+            T![-=]
+            | T![->]
+            | T![::]
+            | T![!=]
+            | T![..]
+            | T![*=]
+            | T![/=]
+            | T![&&]
+            | T![&=]
+            | T![%=]
+            | T![^=]
+            | T![+=]
+            | T![<<]
+            | T![<=]
+            | T![==]
+            | T![=>]
+            | T![>=]
+            | T![>>]
+            | T![|=]
+            | T![||] => 2,
+
+            T![...] | T![..=] | T![<<=] | T![>>=] => 3,
+            _ => 1,
+        };
+        self.do_bump(kind, n_raw_tokens);
+        true
+    }
+
+    fn at_composite2(&self, n: usize, k1: SyntaxKind, k2: SyntaxKind) -> bool {
+        let t1 = self.token_source.lookahead_nth(n);
+        if t1.kind != k1 || !t1.is_jointed_to_next {
+            return false;
+        }
+        let t2 = self.token_source.lookahead_nth(n + 1);
+        t2.kind == k2
+    }
+
+    fn at_composite3(&self, n: usize, k1: SyntaxKind, k2: SyntaxKind, k3: SyntaxKind) -> bool {
+        let t1 = self.token_source.lookahead_nth(n);
+        if t1.kind != k1 || !t1.is_jointed_to_next {
+            return false;
+        }
+        let t2 = self.token_source.lookahead_nth(n + 1);
+        if t2.kind != k2 || !t2.is_jointed_to_next {
+            return false;
+        }
+        let t3 = self.token_source.lookahead_nth(n + 2);
+        t3.kind == k3
+    }
+
+    /// Checks if the current token is in `kinds`.
+    pub(crate) fn at_ts(&self, kinds: TokenSet) -> bool {
+        kinds.contains(self.current())
+    }
+
+    /// Checks if the current token is contextual keyword with text `t`.
+    pub(crate) fn at_contextual_kw(&self, kw: &str) -> bool {
+        self.token_source.is_keyword(kw)
+    }
+
+    /// Starts a new node in the syntax tree. All nodes and tokens
+    /// consumed between the `start` and the corresponding `Marker::complete`
+    /// belong to the same node.
+    pub(crate) fn start(&mut self) -> Marker {
+        let pos = self.events.len() as u32;
+        self.push_event(Event::tombstone());
+        Marker::new(pos)
+    }
+
+    /// Consume the next token if `kind` matches.
+    pub(crate) fn bump(&mut self, kind: SyntaxKind) {
+        assert!(self.eat(kind));
+    }
+
+    /// Advances the parser by one token
+    pub(crate) fn bump_any(&mut self) {
+        let kind = self.nth(0);
+        if kind == EOF {
+            return;
+        }
+        self.do_bump(kind, 1)
+    }
+
+    /// Advances the parser by one token, remapping its kind.
+    /// This is useful to create contextual keywords from
+    /// identifiers. For example, the lexer creates an `union`
+    /// *identifier* token, but the parser remaps it to the
+    /// `union` keyword, and keyword is what ends up in the
+    /// final tree.
+    pub(crate) fn bump_remap(&mut self, kind: SyntaxKind) {
+        if self.nth(0) == EOF {
+            // FIXME: panic!?
+            return;
+        }
+        self.do_bump(kind, 1);
+    }
+
+    /// Emit error with the `message`
+    /// FIXME: this should be much more fancy and support
+    /// structured errors with spans and notes, like rustc
+    /// does.
+    pub(crate) fn error<T: Into<String>>(&mut self, message: T) {
+        let msg = ParseError(Box::new(message.into()));
+        self.push_event(Event::Error { msg })
+    }
+
+    /// Consume the next token if it is `kind` or emit an error
+    /// otherwise.
+    pub(crate) fn expect(&mut self, kind: SyntaxKind) -> bool {
+        if self.eat(kind) {
+            return true;
+        }
+        self.error(format!("expected {:?}", kind));
+        false
+    }
+
+    /// Create an error node and consume the next token.
+    pub(crate) fn err_and_bump(&mut self, message: &str) {
+        self.err_recover(message, TokenSet::EMPTY);
+    }
+
+    /// Create an error node and consume the next token.
+    pub(crate) fn err_recover(&mut self, message: &str, recovery: TokenSet) {
+        match self.current() {
+            T!['{'] | T!['}'] => {
+                self.error(message);
+                return;
+            }
+            _ => (),
+        }
+
+        if self.at_ts(recovery) {
+            self.error(message);
+            return;
+        }
+
+        let m = self.start();
+        self.error(message);
+        self.bump_any();
+        m.complete(self, ERROR);
+    }
+
+    fn do_bump(&mut self, kind: SyntaxKind, n_raw_tokens: u8) {
+        for _ in 0..n_raw_tokens {
+            self.token_source.bump();
+        }
+
+        self.push_event(Event::Token { kind, n_raw_tokens });
+    }
+
+    fn push_event(&mut self, event: Event) {
+        self.events.push(event)
+    }
+}
+
+/// See `Parser::start`.
+pub(crate) struct Marker {
+    pos: u32,
+    bomb: DropBomb,
+}
+
+impl Marker {
+    fn new(pos: u32) -> Marker {
+        Marker { pos, bomb: DropBomb::new("Marker must be either completed or abandoned") }
+    }
+
+    /// Finishes the syntax tree node and assigns `kind` to it,
+    /// and mark the create a `CompletedMarker` for possible future
+    /// operation like `.precede()` to deal with forward_parent.
+    pub(crate) fn complete(mut self, p: &mut Parser, kind: SyntaxKind) -> CompletedMarker {
+        self.bomb.defuse();
+        let idx = self.pos as usize;
+        match &mut p.events[idx] {
+            Event::Start { kind: slot, .. } => {
+                *slot = kind;
+            }
+            _ => unreachable!(),
+        }
+        let finish_pos = p.events.len() as u32;
+        p.push_event(Event::Finish);
+        CompletedMarker::new(self.pos, finish_pos, kind)
+    }
+
+    /// Abandons the syntax tree node. All its children
+    /// are attached to its parent instead.
+    pub(crate) fn abandon(mut self, p: &mut Parser) {
+        self.bomb.defuse();
+        let idx = self.pos as usize;
+        if idx == p.events.len() - 1 {
+            match p.events.pop() {
+                Some(Event::Start { kind: TOMBSTONE, forward_parent: None }) => (),
+                _ => unreachable!(),
+            }
+        }
+    }
+}
+
+pub(crate) struct CompletedMarker {
+    start_pos: u32,
+    finish_pos: u32,
+    kind: SyntaxKind,
+}
+
+impl CompletedMarker {
+    fn new(start_pos: u32, finish_pos: u32, kind: SyntaxKind) -> Self {
+        CompletedMarker { start_pos, finish_pos, kind }
+    }
+
+    /// This method allows to create a new node which starts
+    /// *before* the current one. That is, parser could start
+    /// node `A`, then complete it, and then after parsing the
+    /// whole `A`, decide that it should have started some node
+    /// `B` before starting `A`. `precede` allows to do exactly
+    /// that. See also docs about `forward_parent` in `Event::Start`.
+    ///
+    /// Given completed events `[START, FINISH]` and its corresponding
+    /// `CompletedMarker(pos: 0, _)`.
+    /// Append a new `START` events as `[START, FINISH, NEWSTART]`,
+    /// then mark `NEWSTART` as `START`'s parent with saving its relative
+    /// distance to `NEWSTART` into forward_parent(=2 in this case);
+    pub(crate) fn precede(self, p: &mut Parser) -> Marker {
+        let new_pos = p.start();
+        let idx = self.start_pos as usize;
+        match &mut p.events[idx] {
+            Event::Start { forward_parent, .. } => {
+                *forward_parent = Some(new_pos.pos - self.start_pos);
+            }
+            _ => unreachable!(),
+        }
+        new_pos
+    }
+
+    /// Undo this completion and turns into a `Marker`
+    pub(crate) fn undo_completion(self, p: &mut Parser) -> Marker {
+        let start_idx = self.start_pos as usize;
+        let finish_idx = self.finish_pos as usize;
+        match &mut p.events[start_idx] {
+            Event::Start { kind, forward_parent: None } => *kind = TOMBSTONE,
+            _ => unreachable!(),
+        }
+        match &mut p.events[finish_idx] {
+            slot @ Event::Finish => *slot = Event::tombstone(),
+            _ => unreachable!(),
+        }
+        Marker::new(self.start_pos)
+    }
+
+    pub(crate) fn kind(&self) -> SyntaxKind {
+        self.kind
+    }
+}

crates/parser/src/syntax_kind.rs

@@ -0,0 +1,25 @@
+//! FIXME: write short doc here
+
+#[macro_use]
+mod generated;
+
+pub use self::generated::SyntaxKind;
+
+impl From<u16> for SyntaxKind {
+    fn from(d: u16) -> SyntaxKind {
+        assert!(d <= (SyntaxKind::__LAST as u16));
+        unsafe { std::mem::transmute::<u16, SyntaxKind>(d) }
+    }
+}
+
+impl From<SyntaxKind> for u16 {
+    fn from(k: SyntaxKind) -> u16 {
+        k as u16
+    }
+}
+
+impl SyntaxKind {
+    pub fn is_trivia(self) -> bool {
+        matches!(self, SyntaxKind::WHITESPACE | SyntaxKind::COMMENT)
+    }
+}

crates/parser/src/syntax_kind/generated.rs

@@ -0,0 +1,367 @@
+//! Generated file, do not edit by hand, see `xtask/src/codegen`
+
+#![allow(bad_style, missing_docs, unreachable_pub)]
+#[doc = r" The kind of syntax node, e.g. `IDENT`, `USE_KW`, or `STRUCT`."]
+#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)]
+#[repr(u16)]
+pub enum SyntaxKind {
+    #[doc(hidden)]
+    TOMBSTONE,
+    #[doc(hidden)]
+    EOF,
+    SEMICOLON,
+    COMMA,
+    L_PAREN,
+    R_PAREN,
+    L_CURLY,
+    R_CURLY,
+    L_BRACK,
+    R_BRACK,
+    L_ANGLE,
+    R_ANGLE,
+    AT,
+    POUND,
+    TILDE,
+    QUESTION,
+    DOLLAR,
+    AMP,
+    PIPE,
+    PLUS,
+    STAR,
+    SLASH,
+    CARET,
+    PERCENT,
+    UNDERSCORE,
+    DOT,
+    DOT2,
+    DOT3,
+    DOT2EQ,
+    COLON,
+    COLON2,
+    EQ,
+    EQ2,
+    FAT_ARROW,
+    BANG,
+    NEQ,
+    MINUS,
+    THIN_ARROW,
+    LTEQ,
+    GTEQ,
+    PLUSEQ,
+    MINUSEQ,
+    PIPEEQ,
+    AMPEQ,
+    CARETEQ,
+    SLASHEQ,
+    STAREQ,
+    PERCENTEQ,
+    AMP2,
+    PIPE2,
+    SHL,
+    SHR,
+    SHLEQ,
+    SHREQ,
+    AS_KW,
+    ASYNC_KW,
+    AWAIT_KW,
+    BOX_KW,
+    BREAK_KW,
+    CONST_KW,
+    CONTINUE_KW,
+    CRATE_KW,
+    DYN_KW,
+    ELSE_KW,
+    ENUM_KW,
+    EXTERN_KW,
+    FALSE_KW,
+    FN_KW,
+    FOR_KW,
+    IF_KW,
+    IMPL_KW,
+    IN_KW,
+    LET_KW,
+    LOOP_KW,
+    MACRO_KW,
+    MATCH_KW,
+    MOD_KW,
+    MOVE_KW,
+    MUT_KW,
+    PUB_KW,
+    REF_KW,
+    RETURN_KW,
+    SELF_KW,
+    STATIC_KW,
+    STRUCT_KW,
+    SUPER_KW,
+    TRAIT_KW,
+    TRUE_KW,
+    TRY_KW,
+    TYPE_KW,
+    UNSAFE_KW,
+    USE_KW,
+    WHERE_KW,
+    WHILE_KW,
+    AUTO_KW,
+    DEFAULT_KW,
+    EXISTENTIAL_KW,
+    UNION_KW,
+    RAW_KW,
+    INT_NUMBER,
+    FLOAT_NUMBER,
+    CHAR,
+    BYTE,
+    STRING,
+    RAW_STRING,
+    BYTE_STRING,
+    RAW_BYTE_STRING,
+    ERROR,
+    IDENT,
+    WHITESPACE,
+    LIFETIME,
+    COMMENT,
+    SHEBANG,
+    L_DOLLAR,
+    R_DOLLAR,
+    SOURCE_FILE,
+    STRUCT,
+    UNION,
+    ENUM,
+    FN,
+    RET_TYPE,
+    EXTERN_CRATE,
+    MODULE,
+    USE,
+    STATIC,
+    CONST,
+    TRAIT,
+    IMPL,
+    TYPE_ALIAS,
+    MACRO_CALL,
+    TOKEN_TREE,
+    MACRO_DEF,
+    PAREN_TYPE,
+    TUPLE_TYPE,
+    NEVER_TYPE,
+    PATH_TYPE,
+    PTR_TYPE,
+    ARRAY_TYPE,
+    SLICE_TYPE,
+    REF_TYPE,
+    INFER_TYPE,
+    FN_PTR_TYPE,
+    FOR_TYPE,
+    IMPL_TRAIT_TYPE,
+    DYN_TRAIT_TYPE,
+    OR_PAT,
+    PAREN_PAT,
+    REF_PAT,
+    BOX_PAT,
+    IDENT_PAT,
+    WILDCARD_PAT,
+    REST_PAT,
+    PATH_PAT,
+    RECORD_PAT,
+    RECORD_PAT_FIELD_LIST,
+    RECORD_PAT_FIELD,
+    TUPLE_STRUCT_PAT,
+    TUPLE_PAT,
+    SLICE_PAT,
+    RANGE_PAT,
+    LITERAL_PAT,
+    MACRO_PAT,
+    TUPLE_EXPR,
+    ARRAY_EXPR,
+    PAREN_EXPR,
+    PATH_EXPR,
+    CLOSURE_EXPR,
+    IF_EXPR,
+    WHILE_EXPR,
+    CONDITION,
+    LOOP_EXPR,
+    FOR_EXPR,
+    CONTINUE_EXPR,
+    BREAK_EXPR,
+    LABEL,
+    BLOCK_EXPR,
+    RETURN_EXPR,
+    MATCH_EXPR,
+    MATCH_ARM_LIST,
+    MATCH_ARM,
+    MATCH_GUARD,
+    RECORD_EXPR,
+    RECORD_EXPR_FIELD_LIST,
+    RECORD_EXPR_FIELD,
+    EFFECT_EXPR,
+    BOX_EXPR,
+    CALL_EXPR,
+    INDEX_EXPR,
+    METHOD_CALL_EXPR,
+    FIELD_EXPR,
+    AWAIT_EXPR,
+    TRY_EXPR,
+    CAST_EXPR,
+    REF_EXPR,
+    PREFIX_EXPR,
+    RANGE_EXPR,
+    BIN_EXPR,
+    EXTERN_BLOCK,
+    EXTERN_ITEM_LIST,
+    VARIANT,
+    RECORD_FIELD_LIST,
+    RECORD_FIELD,
+    TUPLE_FIELD_LIST,
+    TUPLE_FIELD,
+    VARIANT_LIST,
+    ITEM_LIST,
+    ASSOC_ITEM_LIST,
+    ATTR,
+    META_ITEM,
+    USE_TREE,
+    USE_TREE_LIST,
+    PATH,
+    PATH_SEGMENT,
+    LITERAL,
+    RENAME,
+    VISIBILITY,
+    WHERE_CLAUSE,
+    WHERE_PRED,
+    ABI,
+    NAME,
+    NAME_REF,
+    LET_STMT,
+    EXPR_STMT,
+    GENERIC_PARAM_LIST,
+    GENERIC_PARAM,
+    LIFETIME_PARAM,
+    TYPE_PARAM,
+    CONST_PARAM,
+    GENERIC_ARG_LIST,
+    LIFETIME_ARG,
+    TYPE_ARG,
+    ASSOC_TYPE_ARG,
+    CONST_ARG,
+    PARAM_LIST,
+    PARAM,
+    SELF_PARAM,
+    ARG_LIST,
+    TYPE_BOUND,
+    TYPE_BOUND_LIST,
+    MACRO_ITEMS,
+    MACRO_STMTS,
+    #[doc(hidden)]
+    __LAST,
+}
+use self::SyntaxKind::*;
+impl SyntaxKind {
+    pub fn is_keyword(self) -> bool {
+        match self {
+            AS_KW | ASYNC_KW | AWAIT_KW | BOX_KW | BREAK_KW | CONST_KW | CONTINUE_KW | CRATE_KW
+            | DYN_KW | ELSE_KW | ENUM_KW | EXTERN_KW | FALSE_KW | FN_KW | FOR_KW | IF_KW
+            | IMPL_KW | IN_KW | LET_KW | LOOP_KW | MACRO_KW | MATCH_KW | MOD_KW | MOVE_KW
+            | MUT_KW | PUB_KW | REF_KW | RETURN_KW | SELF_KW | STATIC_KW | STRUCT_KW | SUPER_KW
+            | TRAIT_KW | TRUE_KW | TRY_KW | TYPE_KW | UNSAFE_KW | USE_KW | WHERE_KW | WHILE_KW
+            | AUTO_KW | DEFAULT_KW | EXISTENTIAL_KW | UNION_KW | RAW_KW => true,
+            _ => false,
+        }
+    }
+    pub fn is_punct(self) -> bool {
+        match self {
+            SEMICOLON | COMMA | L_PAREN | R_PAREN | L_CURLY | R_CURLY | L_BRACK | R_BRACK
+            | L_ANGLE | R_ANGLE | AT | POUND | TILDE | QUESTION | DOLLAR | AMP | PIPE | PLUS
+            | STAR | SLASH | CARET | PERCENT | UNDERSCORE | DOT | DOT2 | DOT3 | DOT2EQ | COLON
+            | COLON2 | EQ | EQ2 | FAT_ARROW | BANG | NEQ | MINUS | THIN_ARROW | LTEQ | GTEQ
+            | PLUSEQ | MINUSEQ | PIPEEQ | AMPEQ | CARETEQ | SLASHEQ | STAREQ | PERCENTEQ | AMP2
+            | PIPE2 | SHL | SHR | SHLEQ | SHREQ => true,
+            _ => false,
+        }
+    }
+    pub fn is_literal(self) -> bool {
+        match self {
+            INT_NUMBER | FLOAT_NUMBER | CHAR | BYTE | STRING | RAW_STRING | BYTE_STRING
+            | RAW_BYTE_STRING => true,
+            _ => false,
+        }
+    }
+    pub fn from_keyword(ident: &str) -> Option<SyntaxKind> {
+        let kw = match ident {
+            "as" => AS_KW,
+            "async" => ASYNC_KW,
+            "await" => AWAIT_KW,
+            "box" => BOX_KW,
+            "break" => BREAK_KW,
+            "const" => CONST_KW,
+            "continue" => CONTINUE_KW,
+            "crate" => CRATE_KW,
+            "dyn" => DYN_KW,
+            "else" => ELSE_KW,
+            "enum" => ENUM_KW,
+            "extern" => EXTERN_KW,
+            "false" => FALSE_KW,
+            "fn" => FN_KW,
+            "for" => FOR_KW,
+            "if" => IF_KW,
+            "impl" => IMPL_KW,
+            "in" => IN_KW,
+            "let" => LET_KW,
+            "loop" => LOOP_KW,
+            "macro" => MACRO_KW,
+            "match" => MATCH_KW,
+            "mod" => MOD_KW,
+            "move" => MOVE_KW,
+            "mut" => MUT_KW,
+            "pub" => PUB_KW,
+            "ref" => REF_KW,
+            "return" => RETURN_KW,
+            "self" => SELF_KW,
+            "static" => STATIC_KW,
+            "struct" => STRUCT_KW,
+            "super" => SUPER_KW,
+            "trait" => TRAIT_KW,
+            "true" => TRUE_KW,
+            "try" => TRY_KW,
+            "type" => TYPE_KW,
+            "unsafe" => UNSAFE_KW,
+            "use" => USE_KW,
+            "where" => WHERE_KW,
+            "while" => WHILE_KW,
+            _ => return None,
+        };
+        Some(kw)
+    }
+    pub fn from_char(c: char) -> Option<SyntaxKind> {
+        let tok = match c {
+            ';' => SEMICOLON,
+            ',' => COMMA,
+            '(' => L_PAREN,
+            ')' => R_PAREN,
+            '{' => L_CURLY,
+            '}' => R_CURLY,
+            '[' => L_BRACK,
+            ']' => R_BRACK,
+            '<' => L_ANGLE,
+            '>' => R_ANGLE,
+            '@' => AT,
+            '#' => POUND,
+            '~' => TILDE,
+            '?' => QUESTION,
+            '$' => DOLLAR,
+            '&' => AMP,
+            '|' => PIPE,
+            '+' => PLUS,
+            '*' => STAR,
+            '/' => SLASH,
+            '^' => CARET,
+            '%' => PERCENT,
+            '_' => UNDERSCORE,
+            '.' => DOT,
+            ':' => COLON,
+            '=' => EQ,
+            '!' => BANG,
+            '-' => MINUS,
+            _ => return None,
+        };
+        Some(tok)
+    }
+}
+#[macro_export]
+macro_rules ! T { [ ; ] => { $ crate :: SyntaxKind :: SEMICOLON } ; [ , ] => { $ crate :: SyntaxKind :: COMMA } ; [ '(' ] => { $ crate :: SyntaxKind :: L_PAREN } ; [ ')' ] => { $ crate :: SyntaxKind :: R_PAREN } ; [ '{' ] => { $ crate :: SyntaxKind :: L_CURLY } ; [ '}' ] => { $ crate :: SyntaxKind :: R_CURLY } ; [ '[' ] => { $ crate :: SyntaxKind :: L_BRACK } ; [ ']' ] => { $ crate :: SyntaxKind :: R_BRACK } ; [ < ] => { $ crate :: SyntaxKind :: L_ANGLE } ; [ > ] => { $ crate :: SyntaxKind :: R_ANGLE } ; [ @ ] => { $ crate :: SyntaxKind :: AT } ; [ # ] => { $ crate :: SyntaxKind :: POUND } ; [ ~ ] => { $ crate :: SyntaxKind :: TILDE } ; [ ? ] => { $ crate :: SyntaxKind :: QUESTION } ; [ $ ] => { $ crate :: SyntaxKind :: DOLLAR } ; [ & ] => { $ crate :: SyntaxKind :: AMP } ; [ | ] => { $ crate :: SyntaxKind :: PIPE } ; [ + ] => { $ crate :: SyntaxKind :: PLUS } ; [ * ] => { $ crate :: SyntaxKind :: STAR } ; [ / ] => { $ crate :: SyntaxKind :: SLASH } ; [ ^ ] => { $ crate :: SyntaxKind :: CARET } ; [ % ] => { $ crate :: SyntaxKind :: PERCENT } ; [ _ ] => { $ crate :: SyntaxKind :: UNDERSCORE } ; [ . ] => { $ crate :: SyntaxKind :: DOT } ; [ .. ] => { $ crate :: SyntaxKind :: DOT2 } ; [ ... ] => { $ crate :: SyntaxKind :: DOT3 } ; [ ..= ] => { $ crate :: SyntaxKind :: DOT2EQ } ; [ : ] => { $ crate :: SyntaxKind :: COLON } ; [ :: ] => { $ crate :: SyntaxKind :: COLON2 } ; [ = ] => { $ crate :: SyntaxKind :: EQ } ; [ == ] => { $ crate :: SyntaxKind :: EQ2 } ; [ => ] => { $ crate :: SyntaxKind :: FAT_ARROW } ; [ ! ] => { $ crate :: SyntaxKind :: BANG } ; [ != ] => { $ crate :: SyntaxKind :: NEQ } ; [ - ] => { $ crate :: SyntaxKind :: MINUS } ; [ -> ] => { $ crate :: SyntaxKind :: THIN_ARROW } ; [ <= ] => { $ crate :: SyntaxKind :: LTEQ } ; [ >= ] => { $ crate :: SyntaxKind :: GTEQ } ; [ += ] => { $ crate :: SyntaxKind :: PLUSEQ } ; [ -= ] => { $ crate :: SyntaxKind :: MINUSEQ } ; [ |= ] => { $ crate :: SyntaxKind :: PIPEEQ } ; [ &= ] => { $ crate :: SyntaxKind :: AMPEQ } ; [ ^= ] => { $ crate :: SyntaxKind :: CARETEQ } ; [ /= ] => { $ crate :: SyntaxKind :: SLASHEQ } ; [ *= ] => { $ crate :: SyntaxKind :: STAREQ } ; [ %= ] => { $ crate :: SyntaxKind :: PERCENTEQ } ; [ && ] => { $ crate :: SyntaxKind :: AMP2 } ; [ || ] => { $ crate :: SyntaxKind :: PIPE2 } ; [ << ] => { $ crate :: SyntaxKind :: SHL } ; [ >> ] => { $ crate :: SyntaxKind :: SHR } ; [ <<= ] => { $ crate :: SyntaxKind :: SHLEQ } ; [ >>= ] => { $ crate :: SyntaxKind :: SHREQ } ; [ as ] => { $ crate :: SyntaxKind :: AS_KW } ; [ async ] => { $ crate :: SyntaxKind :: ASYNC_KW } ; [ await ] => { $ crate :: SyntaxKind :: AWAIT_KW } ; [ box ] => { $ crate :: SyntaxKind :: BOX_KW } ; [ break ] => { $ crate :: SyntaxKind :: BREAK_KW } ; [ const ] => { $ crate :: SyntaxKind :: CONST_KW } ; [ continue ] => { $ crate :: SyntaxKind :: CONTINUE_KW } ; [ crate ] => { $ crate :: SyntaxKind :: CRATE_KW } ; [ dyn ] => { $ crate :: SyntaxKind :: DYN_KW } ; [ else ] => { $ crate :: SyntaxKind :: ELSE_KW } ; [ enum ] => { $ crate :: SyntaxKind :: ENUM_KW } ; [ extern ] => { $ crate :: SyntaxKind :: EXTERN_KW } ; [ false ] => { $ crate :: SyntaxKind :: FALSE_KW } ; [ fn ] => { $ crate :: SyntaxKind :: FN_KW } ; [ for ] => { $ crate :: SyntaxKind :: FOR_KW } ; [ if ] => { $ crate :: SyntaxKind :: IF_KW } ; [ impl ] => { $ crate :: SyntaxKind :: IMPL_KW } ; [ in ] => { $ crate :: SyntaxKind :: IN_KW } ; [ let ] => { $ crate :: SyntaxKind :: LET_KW } ; [ loop ] => { $ crate :: SyntaxKind :: LOOP_KW } ; [ macro ] => { $ crate :: SyntaxKind :: MACRO_KW } ; [ match ] => { $ crate :: SyntaxKind :: MATCH_KW } ; [ mod ] => { $ crate :: SyntaxKind :: MOD_KW } ; [ move ] => { $ crate :: SyntaxKind :: MOVE_KW } ; [ mut ] => { $ crate :: SyntaxKind :: MUT_KW } ; [ pub ] => { $ crate :: SyntaxKind :: PUB_KW } ; [ ref ] => { $ crate :: SyntaxKind :: REF_KW } ; [ return ] => { $ crate :: SyntaxKind :: RETURN_KW } ; [ self ] => { $ crate :: SyntaxKind :: SELF_KW } ; [ static ] => { $ crate :: SyntaxKind :: STATIC_KW } ; [ struct ] => { $ crate :: SyntaxKind :: STRUCT_KW } ; [ super ] => { $ crate :: SyntaxKind :: SUPER_KW } ; [ trait ] => { $ crate :: SyntaxKind :: TRAIT_KW } ; [ true ] => { $ crate :: SyntaxKind :: TRUE_KW } ; [ try ] => { $ crate :: SyntaxKind :: TRY_KW } ; [ type ] => { $ crate :: SyntaxKind :: TYPE_KW } ; [ unsafe ] => { $ crate :: SyntaxKind :: UNSAFE_KW } ; [ use ] => { $ crate :: SyntaxKind :: USE_KW } ; [ where ] => { $ crate :: SyntaxKind :: WHERE_KW } ; [ while ] => { $ crate :: SyntaxKind :: WHILE_KW } ; [ auto ] => { $ crate :: SyntaxKind :: AUTO_KW } ; [ default ] => { $ crate :: SyntaxKind :: DEFAULT_KW } ; [ existential ] => { $ crate :: SyntaxKind :: EXISTENTIAL_KW } ; [ union ] => { $ crate :: SyntaxKind :: UNION_KW } ; [ raw ] => { $ crate :: SyntaxKind :: RAW_KW } ; [ lifetime ] => { $ crate :: SyntaxKind :: LIFETIME } ; [ ident ] => { $ crate :: SyntaxKind :: IDENT } ; [ shebang ] => { $ crate :: SyntaxKind :: SHEBANG } ; }

crates/parser/src/token_set.rs

@@ -0,0 +1,42 @@
+//! A bit-set of `SyntaxKind`s.
+
+use crate::SyntaxKind;
+
+/// A bit-set of `SyntaxKind`s
+#[derive(Clone, Copy)]
+pub(crate) struct TokenSet(u128);
+
+impl TokenSet {
+    pub(crate) const EMPTY: TokenSet = TokenSet(0);
+
+    pub(crate) const fn singleton(kind: SyntaxKind) -> TokenSet {
+        TokenSet(mask(kind))
+    }
+
+    pub(crate) const fn union(self, other: TokenSet) -> TokenSet {
+        TokenSet(self.0 | other.0)
+    }
+
+    pub(crate) fn contains(&self, kind: SyntaxKind) -> bool {
+        self.0 & mask(kind) != 0
+    }
+}
+
+const fn mask(kind: SyntaxKind) -> u128 {
+    1u128 << (kind as usize)
+}
+
+#[macro_export]
+macro_rules! token_set {
+    ($($t:expr),*) => { TokenSet::EMPTY$(.union(TokenSet::singleton($t)))* };
+    ($($t:expr),* ,) => { token_set!($($t),*) };
+}
+
+#[test]
+fn token_set_works_for_tokens() {
+    use crate::SyntaxKind::*;
+    let ts = token_set![EOF, SHEBANG];
+    assert!(ts.contains(EOF));
+    assert!(ts.contains(SHEBANG));
+    assert!(!ts.contains(PLUS));
+}