Encode ident rawness and literal kind separately in tt::Leaf

This commit is contained in:
Lukas Wirth 2024-07-07 17:47:38 +02:00
parent 5784915618
commit e846c04fbe
33 changed files with 860 additions and 412 deletions

View file

@ -49,58 +49,39 @@ fn spacing_to_internal(spacing: proc_macro::Spacing) -> Spacing {
#[allow(unused)]
fn spacing_to_external(spacing: Spacing) -> proc_macro::Spacing {
match spacing {
Spacing::Alone => proc_macro::Spacing::Alone,
Spacing::Alone | Spacing::JointHidden => proc_macro::Spacing::Alone,
Spacing::Joint => proc_macro::Spacing::Joint,
}
}
/// Invokes the callback with a `&[&str]` consisting of each part of the
/// literal's representation. This is done to allow the `ToString` and
/// `Display` implementations to borrow references to symbol values, and
/// both be optimized to reduce overhead.
fn literal_with_stringify_parts<S, R>(
literal: &bridge::Literal<S, Symbol>,
interner: SymbolInternerRef,
f: impl FnOnce(&[&str]) -> R,
) -> R {
/// Returns a string containing exactly `num` '#' characters.
/// Uses a 256-character source string literal which is always safe to
/// index with a `u8` index.
fn get_hashes_str(num: u8) -> &'static str {
const HASHES: &str = "\
################################################################\
################################################################\
################################################################\
################################################################\
";
const _: () = assert!(HASHES.len() == 256);
&HASHES[..num as usize]
}
{
let symbol = &*literal.symbol.text(interner);
let suffix = &*literal.suffix.map(|s| s.text(interner)).unwrap_or_default();
match literal.kind {
bridge::LitKind::Byte => f(&["b'", symbol, "'", suffix]),
bridge::LitKind::Char => f(&["'", symbol, "'", suffix]),
bridge::LitKind::Str => f(&["\"", symbol, "\"", suffix]),
bridge::LitKind::StrRaw(n) => {
let hashes = get_hashes_str(n);
f(&["r", hashes, "\"", symbol, "\"", hashes, suffix])
}
bridge::LitKind::ByteStr => f(&["b\"", symbol, "\"", suffix]),
bridge::LitKind::ByteStrRaw(n) => {
let hashes = get_hashes_str(n);
f(&["br", hashes, "\"", symbol, "\"", hashes, suffix])
}
bridge::LitKind::CStr => f(&["c\"", symbol, "\"", suffix]),
bridge::LitKind::CStrRaw(n) => {
let hashes = get_hashes_str(n);
f(&["cr", hashes, "\"", symbol, "\"", hashes, suffix])
}
bridge::LitKind::Integer | bridge::LitKind::Float | bridge::LitKind::ErrWithGuar => {
f(&[symbol, suffix])
}
}
fn literal_kind_to_external(kind: tt::LitKind) -> bridge::LitKind {
match kind {
tt::LitKind::Byte => bridge::LitKind::Byte,
tt::LitKind::Char => bridge::LitKind::Char,
tt::LitKind::Integer => bridge::LitKind::Integer,
tt::LitKind::Float => bridge::LitKind::Float,
tt::LitKind::Str => bridge::LitKind::Str,
tt::LitKind::StrRaw(r) => bridge::LitKind::StrRaw(r),
tt::LitKind::ByteStr => bridge::LitKind::ByteStr,
tt::LitKind::ByteStrRaw(r) => bridge::LitKind::ByteStrRaw(r),
tt::LitKind::CStr => bridge::LitKind::CStr,
tt::LitKind::CStrRaw(r) => bridge::LitKind::CStrRaw(r),
tt::LitKind::Err(_) => bridge::LitKind::ErrWithGuar,
}
}
fn literal_kind_to_internal(kind: bridge::LitKind) -> tt::LitKind {
match kind {
bridge::LitKind::Byte => tt::LitKind::Byte,
bridge::LitKind::Char => tt::LitKind::Char,
bridge::LitKind::Str => tt::LitKind::Str,
bridge::LitKind::StrRaw(r) => tt::LitKind::StrRaw(r),
bridge::LitKind::ByteStr => tt::LitKind::ByteStr,
bridge::LitKind::ByteStrRaw(r) => tt::LitKind::ByteStrRaw(r),
bridge::LitKind::CStr => tt::LitKind::CStr,
bridge::LitKind::CStrRaw(r) => tt::LitKind::CStrRaw(r),
bridge::LitKind::Integer => tt::LitKind::Integer,
bridge::LitKind::Float => tt::LitKind::Float,
bridge::LitKind::ErrWithGuar => tt::LitKind::Err(()),
}
}

View file

@ -15,7 +15,7 @@ use span::{Span, FIXUP_ERASED_FILE_AST_ID_MARKER};
use tt::{TextRange, TextSize};
use crate::server_impl::{
delim_to_external, delim_to_internal, literal_with_stringify_parts,
delim_to_external, delim_to_internal, literal_kind_to_external, literal_kind_to_internal,
token_stream::TokenStreamBuilder, Symbol, SymbolInternerRef, SYMBOL_INTERNER,
};
mod tt {
@ -171,20 +171,24 @@ impl server::TokenStream for RaSpanServer {
bridge::TokenTree::Ident(ident) => {
let text = ident.sym.text(self.interner);
let text =
if ident.is_raw { ::tt::SmolStr::from_iter(["r#", &text]) } else { text };
let ident: tt::Ident = tt::Ident { text, span: ident.span };
let ident: tt::Ident = tt::Ident {
text,
span: ident.span,
is_raw: if ident.is_raw { tt::IdentIsRaw::Yes } else { tt::IdentIsRaw::No },
};
let leaf = tt::Leaf::from(ident);
let tree = tt::TokenTree::from(leaf);
Self::TokenStream::from_iter(iter::once(tree))
}
bridge::TokenTree::Literal(literal) => {
let text = literal_with_stringify_parts(&literal, self.interner, |parts| {
::tt::SmolStr::from_iter(parts.iter().copied())
});
let literal = tt::Literal {
text: literal.symbol.text(self.interner),
suffix: literal.suffix.map(|it| Box::new(it.text(self.interner))),
span: literal.span,
kind: literal_kind_to_internal(literal.kind),
};
let literal = tt::Literal { text, span: literal.span };
let leaf: tt::Leaf = tt::Leaf::from(literal);
let tree = tt::TokenTree::from(leaf);
Self::TokenStream::from_iter(iter::once(tree))
@ -250,23 +254,18 @@ impl server::TokenStream for RaSpanServer {
.into_iter()
.map(|tree| match tree {
tt::TokenTree::Leaf(tt::Leaf::Ident(ident)) => {
bridge::TokenTree::Ident(match ident.text.strip_prefix("r#") {
Some(text) => bridge::Ident {
sym: Symbol::intern(self.interner, text),
is_raw: true,
span: ident.span,
},
None => bridge::Ident {
sym: Symbol::intern(self.interner, &ident.text),
is_raw: false,
span: ident.span,
},
bridge::TokenTree::Ident(bridge::Ident {
sym: Symbol::intern(self.interner, &ident.text),
is_raw: ident.is_raw.yes(),
span: ident.span,
})
}
tt::TokenTree::Leaf(tt::Leaf::Literal(lit)) => {
bridge::TokenTree::Literal(bridge::Literal {
span: lit.span,
..server::FreeFunctions::literal_from_str(self, &lit.text).unwrap()
kind: literal_kind_to_external(lit.kind),
symbol: Symbol::intern(self.interner, &lit.text),
suffix: lit.suffix.map(|it| Symbol::intern(self.interner, &it)),
})
}
tt::TokenTree::Leaf(tt::Leaf::Punct(punct)) => {

View file

@ -8,7 +8,7 @@ use std::{
use proc_macro::bridge::{self, server};
use crate::server_impl::{
delim_to_external, delim_to_internal, literal_with_stringify_parts,
delim_to_external, delim_to_internal, literal_kind_to_external, literal_kind_to_internal,
token_stream::TokenStreamBuilder, Symbol, SymbolInternerRef, SYMBOL_INTERNER,
};
mod tt {
@ -25,10 +25,8 @@ mod tt {
}
type Group = tt::Subtree;
type TokenTree = tt::TokenTree;
#[allow(unused)]
type Punct = tt::Punct;
type Spacing = tt::Spacing;
#[allow(unused)]
type Literal = tt::Literal;
type Span = tt::TokenId;
type TokenStream = crate::server_impl::TokenStream<Span>;
@ -162,20 +160,23 @@ impl server::TokenStream for TokenIdServer {
bridge::TokenTree::Ident(ident) => {
let text = ident.sym.text(self.interner);
let text =
if ident.is_raw { ::tt::SmolStr::from_iter(["r#", &text]) } else { text };
let ident: tt::Ident = tt::Ident { text, span: ident.span };
let ident: tt::Ident = tt::Ident {
text,
span: ident.span,
is_raw: if ident.is_raw { tt::IdentIsRaw::Yes } else { tt::IdentIsRaw::No },
};
let leaf = tt::Leaf::from(ident);
let tree = TokenTree::from(leaf);
Self::TokenStream::from_iter(iter::once(tree))
}
bridge::TokenTree::Literal(literal) => {
let text = literal_with_stringify_parts(&literal, self.interner, |parts| {
::tt::SmolStr::from_iter(parts.iter().copied())
});
let literal = tt::Literal { text, span: literal.span };
let literal = Literal {
text: literal.symbol.text(self.interner),
suffix: literal.suffix.map(|it| Box::new(it.text(self.interner))),
span: literal.span,
kind: literal_kind_to_internal(literal.kind),
};
let leaf = tt::Leaf::from(literal);
let tree = TokenTree::from(leaf);
@ -183,7 +184,7 @@ impl server::TokenStream for TokenIdServer {
}
bridge::TokenTree::Punct(p) => {
let punct = tt::Punct {
let punct = Punct {
char: p.ch as char,
spacing: if p.joint { Spacing::Joint } else { Spacing::Alone },
span: p.span,
@ -238,16 +239,17 @@ impl server::TokenStream for TokenIdServer {
.map(|tree| match tree {
tt::TokenTree::Leaf(tt::Leaf::Ident(ident)) => {
bridge::TokenTree::Ident(bridge::Ident {
sym: Symbol::intern(self.interner, ident.text.trim_start_matches("r#")),
is_raw: ident.text.starts_with("r#"),
sym: Symbol::intern(self.interner, &ident.text),
is_raw: ident.is_raw.yes(),
span: ident.span,
})
}
tt::TokenTree::Leaf(tt::Leaf::Literal(lit)) => {
bridge::TokenTree::Literal(bridge::Literal {
span: lit.span,
..server::FreeFunctions::literal_from_str(self, &lit.text)
.unwrap_or_else(|_| panic!("`{}`", lit.text))
kind: literal_kind_to_external(lit.kind),
symbol: Symbol::intern(self.interner, &lit.text),
suffix: lit.suffix.map(|it| Symbol::intern(self.interner, &it)),
})
}
tt::TokenTree::Leaf(tt::Leaf::Punct(punct)) => {
@ -383,10 +385,12 @@ mod tests {
tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident {
text: "struct".into(),
span: tt::TokenId(0),
is_raw: tt::IdentIsRaw::No,
})),
tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident {
text: "T".into(),
span: tt::TokenId(0),
is_raw: tt::IdentIsRaw::No,
})),
tt::TokenTree::Subtree(tt::Subtree {
delimiter: tt::Delimiter {
@ -411,6 +415,7 @@ mod tests {
kind: tt::DelimiterKind::Parenthesis,
},
token_trees: Box::new([tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident {
is_raw: tt::IdentIsRaw::No,
text: "a".into(),
span: tt::TokenId(0),
}))]),
@ -430,6 +435,7 @@ mod tests {
tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident {
text: "_".into(),
span: tt::TokenId(0),
is_raw: tt::IdentIsRaw::No,
}))
);
}

View file

@ -21,20 +21,20 @@ fn test_derive_error() {
assert_expand(
"DeriveError",
r#"struct S;"#,
expect![[r##"
expect![[r#"
SUBTREE $$ 1 1
IDENT compile_error 1
PUNCH ! [alone] 1
SUBTREE () 1 1
LITERAL "#[derive(DeriveError)] struct S ;"1
PUNCH ; [alone] 1"##]],
expect![[r##"
LITERAL Str #[derive(DeriveError)] struct S ; 1
PUNCH ; [alone] 1"#]],
expect![[r#"
SUBTREE $$ 42:2@0..100#0 42:2@0..100#0
IDENT compile_error 42:2@0..100#0
PUNCH ! [alone] 42:2@0..100#0
SUBTREE () 42:2@0..100#0 42:2@0..100#0
LITERAL "#[derive(DeriveError)] struct S ;"42:2@0..100#0
PUNCH ; [alone] 42:2@0..100#0"##]],
LITERAL Str #[derive(DeriveError)] struct S ; 42:2@0..100#0
PUNCH ; [alone] 42:2@0..100#0"#]],
);
}
@ -47,18 +47,18 @@ fn test_fn_like_macro_noop() {
SUBTREE $$ 1 1
IDENT ident 1
PUNCH , [alone] 1
LITERAL 01
LITERAL Integer 0 1
PUNCH , [alone] 1
LITERAL 11
LITERAL Integer 1 1
PUNCH , [alone] 1
SUBTREE [] 1 1"#]],
expect![[r#"
SUBTREE $$ 42:2@0..100#0 42:2@0..100#0
IDENT ident 42:2@0..5#0
PUNCH , [alone] 42:2@5..6#0
LITERAL 042:2@7..8#0
LITERAL Integer 0 42:2@7..8#0
PUNCH , [alone] 42:2@8..9#0
LITERAL 142:2@10..11#0
LITERAL Integer 1 42:2@10..11#0
PUNCH , [alone] 42:2@11..12#0
SUBTREE [] 42:2@13..14#0 42:2@14..15#0"#]],
);
@ -135,22 +135,22 @@ fn test_fn_like_mk_literals() {
r#""#,
expect![[r#"
SUBTREE $$ 1 1
LITERAL b"byte_string"1
LITERAL 'c'1
LITERAL "string"1
LITERAL 3.14f641
LITERAL 3.141
LITERAL 123i641
LITERAL 1231"#]],
LITERAL ByteStr byte_string 1
LITERAL Char c 1
LITERAL Str string 1
LITERAL Float 3.14f64 1
LITERAL Float 3.14 1
LITERAL Integer 123i64 1
LITERAL Integer 123 1"#]],
expect![[r#"
SUBTREE $$ 42:2@0..100#0 42:2@0..100#0
LITERAL b"byte_string"42:2@0..100#0
LITERAL 'c'42:2@0..100#0
LITERAL "string"42:2@0..100#0
LITERAL 3.14f6442:2@0..100#0
LITERAL 3.1442:2@0..100#0
LITERAL 123i6442:2@0..100#0
LITERAL 12342:2@0..100#0"#]],
LITERAL ByteStr byte_string 42:2@0..100#0
LITERAL Char c 42:2@0..100#0
LITERAL Str string 42:2@0..100#0
LITERAL Float 3.14f64 42:2@0..100#0
LITERAL Float 3.14 42:2@0..100#0
LITERAL Integer 123i64 42:2@0..100#0
LITERAL Integer 123 42:2@0..100#0"#]],
);
}
@ -175,50 +175,50 @@ fn test_fn_like_macro_clone_literals() {
assert_expand(
"fn_like_clone_tokens",
r###"1u16, 2_u32, -4i64, 3.14f32, "hello bridge", "suffixed"suffix, r##"raw"##, 'a', b'b', c"null""###,
expect![[r###"
expect![[r#"
SUBTREE $$ 1 1
LITERAL 1u161
LITERAL Integer 1u16 1
PUNCH , [alone] 1
LITERAL 2_u321
LITERAL Integer 2_u32 1
PUNCH , [alone] 1
PUNCH - [alone] 1
LITERAL 4i641
LITERAL Integer 4i64 1
PUNCH , [alone] 1
LITERAL 3.14f321
LITERAL Float 3.14f32 1
PUNCH , [alone] 1
LITERAL "hello bridge"1
LITERAL Str hello bridge 1
PUNCH , [alone] 1
LITERAL "suffixed"suffix1
LITERAL Str suffixedsuffix 1
PUNCH , [alone] 1
LITERAL r##"raw"##1
LITERAL StrRaw(2) raw 1
PUNCH , [alone] 1
LITERAL 'a'1
LITERAL Char a 1
PUNCH , [alone] 1
LITERAL b'b'1
LITERAL Byte b 1
PUNCH , [alone] 1
LITERAL c"null"1"###]],
expect![[r###"
LITERAL CStr null 1"#]],
expect![[r#"
SUBTREE $$ 42:2@0..100#0 42:2@0..100#0
LITERAL 1u1642:2@0..4#0
LITERAL Integer 1u16 42:2@0..4#0
PUNCH , [alone] 42:2@4..5#0
LITERAL 2_u3242:2@6..11#0
LITERAL Integer 2_u32 42:2@6..11#0
PUNCH , [alone] 42:2@11..12#0
PUNCH - [alone] 42:2@13..14#0
LITERAL 4i6442:2@14..18#0
LITERAL Integer 4i64 42:2@14..18#0
PUNCH , [alone] 42:2@18..19#0
LITERAL 3.14f3242:2@20..27#0
LITERAL Float 3.14f32 42:2@20..27#0
PUNCH , [alone] 42:2@27..28#0
LITERAL "hello bridge"42:2@29..43#0
LITERAL Str hello bridge 42:2@29..43#0
PUNCH , [alone] 42:2@43..44#0
LITERAL "suffixed"suffix42:2@45..61#0
LITERAL Str suffixedsuffix 42:2@45..61#0
PUNCH , [alone] 42:2@61..62#0
LITERAL r##"raw"##42:2@63..73#0
LITERAL StrRaw(2) raw 42:2@63..73#0
PUNCH , [alone] 42:2@73..74#0
LITERAL 'a'42:2@75..78#0
LITERAL Char a 42:2@75..78#0
PUNCH , [alone] 42:2@78..79#0
LITERAL b'b'42:2@80..84#0
LITERAL Byte b 42:2@80..84#0
PUNCH , [alone] 42:2@84..85#0
LITERAL c"null"42:2@86..93#0"###]],
LITERAL CStr null 42:2@86..93#0"#]],
);
}
@ -231,20 +231,20 @@ fn test_attr_macro() {
"attr_error",
r#"mod m {}"#,
r#"some arguments"#,
expect![[r##"
expect![[r#"
SUBTREE $$ 1 1
IDENT compile_error 1
PUNCH ! [alone] 1
SUBTREE () 1 1
LITERAL "#[attr_error(some arguments)] mod m {}"1
PUNCH ; [alone] 1"##]],
expect![[r##"
LITERAL Str #[attr_error(some arguments)] mod m {} 1
PUNCH ; [alone] 1"#]],
expect![[r#"
SUBTREE $$ 42:2@0..100#0 42:2@0..100#0
IDENT compile_error 42:2@0..100#0
PUNCH ! [alone] 42:2@0..100#0
SUBTREE () 42:2@0..100#0 42:2@0..100#0
LITERAL "#[attr_error(some arguments)] mod m {}"42:2@0..100#0
PUNCH ; [alone] 42:2@0..100#0"##]],
LITERAL Str #[attr_error(some arguments)] mod m {} 42:2@0..100#0
PUNCH ; [alone] 42:2@0..100#0"#]],
);
}