language-servers/rust-analyzer
1
0
Fork 0
mirror of https://github.com/rust-lang/rust-analyzer.git synced 2025-09-28 12:54:58 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

Auto merge of #16895 - Veykril:spans, r=Veykril

Browse source 
Resolve whether `$pat` is `$pat_param` or not via 🌟hygiene🌟

Before we just picked the edition from the macro def which is wrong, since a macro call can produce the fragment kind from a different definition site.
This commit is contained in:
bors 2024-03-21 11:04:41 +00:00
commit 83f9cc677f
28 changed files with 440 additions and 375 deletions
Download patch file Download diff file Expand all files Collapse all files

3
Cargo.lock generated
View file

@ -594,6 +594,7 @@ dependencies = [
"rustc-hash", "rustc-hash",
"scoped-tls", "scoped-tls",
"smallvec", "smallvec",
"span",
"stdx", "stdx",
"syntax", "syntax",
"test-fixture", "test-fixture",
@ -637,6 +638,7 @@ dependencies = [
"pulldown-cmark", "pulldown-cmark",
"pulldown-cmark-to-cmark", "pulldown-cmark-to-cmark",
"smallvec", "smallvec",
"span",
"stdx", "stdx",
"syntax", "syntax",
"test-fixture", "test-fixture",
@ -1380,6 +1382,7 @@ dependencies = [
"semver", "semver",
"serde", "serde",
"serde_json", "serde_json",
"span",
"stdx", "stdx",
"toolchain", "toolchain",
"tracing", "tracing",

73
crates/base-db/src/input.rs
View file

@ -6,11 +6,12 @@
//! actual IO. See `vfs` and `project_model` in the `rust-analyzer` crate for how //! actual IO. See `vfs` and `project_model` in the `rust-analyzer` crate for how
//! actual IO is done and lowered to input. //! actual IO is done and lowered to input.
use std::{fmt, mem, ops, str::FromStr}; use std::{fmt, mem, ops};
use cfg::CfgOptions; use cfg::CfgOptions;
use la_arena::{Arena, Idx, RawIdx}; use la_arena::{Arena, Idx, RawIdx};
use rustc_hash::{FxHashMap, FxHashSet}; use rustc_hash::{FxHashMap, FxHashSet};
use span::Edition;
use syntax::SmolStr; use syntax::SmolStr;
use triomphe::Arc; use triomphe::Arc;
use vfs::{file_set::FileSet, AbsPathBuf, AnchoredPath, FileId, VfsPath}; use vfs::{file_set::FileSet, AbsPathBuf, AnchoredPath, FileId, VfsPath};
@ -293,42 +294,11 @@ pub struct CrateData {
pub is_proc_macro: bool, pub is_proc_macro: bool,
} }
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub enum Edition {
Edition2015,
Edition2018,
Edition2021,
Edition2024,
}
impl Edition {
pub const CURRENT: Edition = Edition::Edition2021;
pub const DEFAULT: Edition = Edition::Edition2015;
}
#[derive(Default, Debug, Clone, PartialEq, Eq)] #[derive(Default, Debug, Clone, PartialEq, Eq)]
pub struct Env { pub struct Env {
entries: FxHashMap<String, String>, entries: FxHashMap<String, String>,
} }
impl Env {
pub fn new_for_test_fixture() -> Self {
Env {
entries: FxHashMap::from_iter([(
String::from("__ra_is_test_fixture"),
String::from("__ra_is_test_fixture"),
)]),
}
}
}
#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
pub enum DependencyKind {
Normal,
Dev,
Build,
}
#[derive(Debug, Clone, PartialEq, Eq, Hash)] #[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct Dependency { pub struct Dependency {
pub crate_id: CrateId, pub crate_id: CrateId,
@ -670,32 +640,6 @@ impl CrateData {
} }
} }
impl FromStr for Edition {
type Err = ParseEditionError;
fn from_str(s: &str) -> Result<Self, Self::Err> {
let res = match s {
"2015" => Edition::Edition2015,
"2018" => Edition::Edition2018,
"2021" => Edition::Edition2021,
"2024" => Edition::Edition2024,
_ => return Err(ParseEditionError { invalid_input: s.to_owned() }),
};
Ok(res)
}
}
impl fmt::Display for Edition {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(match self {
Edition::Edition2015 => "2015",
Edition::Edition2018 => "2018",
Edition::Edition2021 => "2021",
Edition::Edition2024 => "2024",
})
}
}
impl Extend<(String, String)> for Env { impl Extend<(String, String)> for Env {
fn extend<T: IntoIterator<Item = (String, String)>>(&mut self, iter: T) { fn extend<T: IntoIterator<Item = (String, String)>>(&mut self, iter: T) {
self.entries.extend(iter); self.entries.extend(iter);
@ -722,19 +666,6 @@ impl Env {
} }
} }
#[derive(Debug)]
pub struct ParseEditionError {
invalid_input: String,
}
impl fmt::Display for ParseEditionError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "invalid edition: {:?}", self.invalid_input)
}
}
impl std::error::Error for ParseEditionError {}
#[derive(Debug)] #[derive(Debug)]
pub struct CyclicDependenciesError { pub struct CyclicDependenciesError {
path: Vec<(CrateId, Option<CrateDisplayName>)>, path: Vec<(CrateId, Option<CrateDisplayName>)>,

6
crates/base-db/src/lib.rs
View file

@ -14,9 +14,9 @@ use triomphe::Arc;
pub use crate::{ pub use crate::{
change::FileChange, change::FileChange,
input::{ input::{
CrateData, CrateDisplayName, CrateGraph, CrateId, CrateName, CrateOrigin, Dependency, CrateData, CrateDisplayName, CrateGraph, CrateId, CrateName, CrateOrigin, Dependency, Env,
DependencyKind, Edition, Env, LangCrateOrigin, ProcMacroPaths, ReleaseChannel, SourceRoot, LangCrateOrigin, ProcMacroPaths, ReleaseChannel, SourceRoot, SourceRootId,
SourceRootId, TargetLayoutLoadResult, TargetLayoutLoadResult,
}, },
}; };
pub use salsa::{self, Cancelled}; pub use salsa::{self, Cancelled};

4
crates/hir-def/src/lib.rs
View file

@ -73,7 +73,7 @@ use std::{
use base_db::{ use base_db::{
impl_intern_key, impl_intern_key,
salsa::{self, impl_intern_value_trivial}, salsa::{self, impl_intern_value_trivial},
CrateId, Edition, CrateId,
}; };
use hir_expand::{ use hir_expand::{
builtin_attr_macro::BuiltinAttrExpander, builtin_attr_macro::BuiltinAttrExpander,
@ -90,7 +90,7 @@ use hir_expand::{
use item_tree::ExternBlock; use item_tree::ExternBlock;
use la_arena::Idx; use la_arena::Idx;
use nameres::DefMap; use nameres::DefMap;
use span::{AstIdNode, FileAstId, FileId, SyntaxContextId}; use span::{AstIdNode, Edition, FileAstId, FileId, SyntaxContextId};
use stdx::impl_from; use stdx::impl_from;
use syntax::{ast, AstNode}; use syntax::{ast, AstNode};

85
crates/hir-def/src/macro_expansion_tests/mbe.rs
View file

@ -1449,6 +1449,7 @@ ok!();
#[test] #[test]
fn test_new_std_matches() { fn test_new_std_matches() {
check( check(
//- edition:2021
r#" r#"
macro_rules! matches { macro_rules! matches {
($expression:expr, $pattern:pat $(if $guard:expr)? $(,)?) => { ($expression:expr, $pattern:pat $(if $guard:expr)? $(,)?) => {
@ -1480,6 +1481,90 @@ fn main() {
); );
} }
#[test]
fn test_hygienic_pat() {
check(
r#"
//- /new.rs crate:new deps:old edition:2015
old::make!();
fn main() {
matches!(0, 0 | 1 if true);
}
//- /old.rs crate:old edition:2021
#[macro_export]
macro_rules! make {
() => {
macro_rules! matches {
($expression:expr, $pattern:pat if $guard:expr ) => {
match $expression {
$pattern if $guard => true,
_ => false
}
};
}
}
}
"#,
expect![[r#"
macro_rules !matches {
($expression: expr, $pattern: pat if $guard: expr) = > {
match $expression {
$pattern if $guard = > true , _ = > false
}
}
;
}
fn main() {
match 0 {
0|1 if true =>true , _=>false
};
}
"#]],
);
check(
r#"
//- /new.rs crate:new deps:old edition:2021
old::make!();
fn main() {
matches/*+errors*/!(0, 0 | 1 if true);
}
//- /old.rs crate:old edition:2015
#[macro_export]
macro_rules! make {
() => {
macro_rules! matches {
($expression:expr, $pattern:pat if $guard:expr ) => {
match $expression {
$pattern if $guard => true,
_ => false
}
};
}
}
}
"#,
expect![[r#"
macro_rules !matches {
($expression: expr, $pattern: pat if $guard: expr) = > {
match $expression {
$pattern if $guard = > true , _ = > false
}
}
;
}
fn main() {
/* error: unexpected token in input *//* parse error: expected expression */
/* parse error: expected FAT_ARROW */
/* parse error: expected `,` */
/* parse error: expected pattern */
match 0 {
0 if $guard=>true , _=>false
};
}
"#]],
);
}
#[test] #[test]
fn test_dollar_crate_lhs_is_not_meta() { fn test_dollar_crate_lhs_is_not_meta() {
check( check(

4
crates/hir-def/src/nameres.rs
View file

@ -59,14 +59,14 @@ mod tests;
use std::ops::Deref; use std::ops::Deref;
use base_db::{CrateId, Edition, FileId}; use base_db::{CrateId, FileId};
use hir_expand::{ use hir_expand::{
name::Name, proc_macro::ProcMacroKind, ErasedAstId, HirFileId, InFile, MacroCallId, MacroDefId, name::Name, proc_macro::ProcMacroKind, ErasedAstId, HirFileId, InFile, MacroCallId, MacroDefId,
}; };
use itertools::Itertools; use itertools::Itertools;
use la_arena::Arena; use la_arena::Arena;
use rustc_hash::{FxHashMap, FxHashSet}; use rustc_hash::{FxHashMap, FxHashSet};
use span::{FileAstId, ROOT_ERASED_FILE_AST_ID}; use span::{Edition, FileAstId, ROOT_ERASED_FILE_AST_ID};
use stdx::format_to; use stdx::format_to;
use syntax::{ast, SmolStr}; use syntax::{ast, SmolStr};
use triomphe::Arc; use triomphe::Arc;

4
crates/hir-def/src/nameres/collector.rs
View file

@ -5,7 +5,7 @@
use std::{cmp::Ordering, iter, mem, ops::Not}; use std::{cmp::Ordering, iter, mem, ops::Not};
use base_db::{CrateId, Dependency, Edition, FileId}; use base_db::{CrateId, Dependency, FileId};
use cfg::{CfgExpr, CfgOptions}; use cfg::{CfgExpr, CfgOptions};
use either::Either; use either::Either;
use hir_expand::{ use hir_expand::{
@ -22,7 +22,7 @@ use itertools::{izip, Itertools};
use la_arena::Idx; use la_arena::Idx;
use limit::Limit; use limit::Limit;
use rustc_hash::{FxHashMap, FxHashSet}; use rustc_hash::{FxHashMap, FxHashSet};
use span::{ErasedFileAstId, FileAstId, Span, SyntaxContextId}; use span::{Edition, ErasedFileAstId, FileAstId, Span, SyntaxContextId};
use stdx::always; use stdx::always;
use syntax::{ast, SmolStr}; use syntax::{ast, SmolStr};
use triomphe::Arc; use triomphe::Arc;

2
crates/hir-def/src/nameres/path_resolution.rs
View file

@ -10,8 +10,8 @@
//! //!
//! `ReachedFixedPoint` signals about this. //! `ReachedFixedPoint` signals about this.
use base_db::Edition;
use hir_expand::{name::Name, Lookup}; use hir_expand::{name::Name, Lookup};
use span::Edition;
use triomphe::Arc; use triomphe::Arc;
use crate::{ use crate::{

4
crates/hir-expand/src/builtin_fn_macro.rs
View file

@ -1,11 +1,11 @@
//! Builtin macro //! Builtin macro
use base_db::{AnchoredPath, Edition, FileId}; use base_db::{AnchoredPath, FileId};
use cfg::CfgExpr; use cfg::CfgExpr;
use either::Either; use either::Either;
use itertools::Itertools; use itertools::Itertools;
use mbe::{parse_exprs_with_sep, parse_to_token_tree}; use mbe::{parse_exprs_with_sep, parse_to_token_tree};
use span::{Span, SpanAnchor, SyntaxContextId, ROOT_ERASED_FILE_AST_ID}; use span::{Edition, Span, SpanAnchor, SyntaxContextId, ROOT_ERASED_FILE_AST_ID};
use syntax::ast::{self, AstToken}; use syntax::ast::{self, AstToken};
use crate::{ use crate::{

38
crates/hir-expand/src/declarative.rs
View file

@ -1,8 +1,8 @@
//! Compiled declarative macro expanders (`macro_rules!`` and `macro`) //! Compiled declarative macro expanders (`macro_rules!`` and `macro`)
use std::sync::OnceLock; use std::sync::OnceLock;
use base_db::{CrateId, Edition, VersionReq}; use base_db::{CrateId, VersionReq};
use span::{MacroCallId, Span}; use span::{MacroCallId, Span, SyntaxContextId};
use syntax::{ast, AstNode}; use syntax::{ast, AstNode};
use triomphe::Arc; use triomphe::Arc;
@ -10,13 +10,13 @@ use crate::{
attrs::RawAttrs, attrs::RawAttrs,
db::ExpandDatabase, db::ExpandDatabase,
hygiene::{apply_mark, Transparency}, hygiene::{apply_mark, Transparency},
tt, AstId, ExpandError, ExpandResult, tt, AstId, ExpandError, ExpandResult, Lookup,
}; };
/// Old-style `macro_rules` or the new macros 2.0 /// Old-style `macro_rules` or the new macros 2.0
#[derive(Debug, Clone, Eq, PartialEq)] #[derive(Debug, Clone, Eq, PartialEq)]
pub struct DeclarativeMacroExpander { pub struct DeclarativeMacroExpander {
pub mac: mbe::DeclarativeMacro<span::Span>, pub mac: mbe::DeclarativeMacro,
pub transparency: Transparency, pub transparency: Transparency,
} }
@ -94,8 +94,6 @@ impl DeclarativeMacroExpander {
def_crate: CrateId, def_crate: CrateId,
id: AstId<ast::Macro>, id: AstId<ast::Macro>,
) -> Arc<DeclarativeMacroExpander> { ) -> Arc<DeclarativeMacroExpander> {
let crate_data = &db.crate_graph()[def_crate];
let is_2021 = crate_data.edition >= Edition::Edition2021;
let (root, map) = crate::db::parse_with_map(db, id.file_id); let (root, map) = crate::db::parse_with_map(db, id.file_id);
let root = root.syntax_node(); let root = root.syntax_node();
@ -133,6 +131,16 @@ impl DeclarativeMacroExpander {
) )
}); });
let edition = |ctx: SyntaxContextId| {
let crate_graph = db.crate_graph();
if ctx.is_root() {
crate_graph[def_crate].edition
} else {
let data = db.lookup_intern_syntax_context(ctx);
// UNWRAP-SAFETY: Only the root context has no outer expansion
crate_graph[data.outer_expn.unwrap().lookup(db).def.krate].edition
}
};
let (mac, transparency) = match id.to_ptr(db).to_node(&root) { let (mac, transparency) = match id.to_ptr(db).to_node(&root) {
ast::Macro::MacroRules(macro_rules) => ( ast::Macro::MacroRules(macro_rules) => (
match macro_rules.token_tree() { match macro_rules.token_tree() {
@ -145,12 +153,11 @@ impl DeclarativeMacroExpander {
), ),
); );
mbe::DeclarativeMacro::parse_macro_rules(&tt, is_2021, new_meta_vars) mbe::DeclarativeMacro::parse_macro_rules(&tt, edition, new_meta_vars)
} }
None => mbe::DeclarativeMacro::from_err( None => mbe::DeclarativeMacro::from_err(mbe::ParseError::Expected(
mbe::ParseError::Expected("expected a token tree".into()), "expected a token tree".into(),
is_2021, )),
),
}, },
transparency(&macro_rules).unwrap_or(Transparency::SemiTransparent), transparency(&macro_rules).unwrap_or(Transparency::SemiTransparent),
), ),
@ -163,12 +170,11 @@ impl DeclarativeMacroExpander {
map.span_for_range(macro_def.macro_token().unwrap().text_range()), map.span_for_range(macro_def.macro_token().unwrap().text_range()),
); );
mbe::DeclarativeMacro::parse_macro2(&tt, is_2021, new_meta_vars) mbe::DeclarativeMacro::parse_macro2(&tt, edition, new_meta_vars)
} }
None => mbe::DeclarativeMacro::from_err( None => mbe::DeclarativeMacro::from_err(mbe::ParseError::Expected(
mbe::ParseError::Expected("expected a token tree".into()), "expected a token tree".into(),
is_2021, )),
),
}, },
transparency(&macro_def).unwrap_or(Transparency::Opaque), transparency(&macro_def).unwrap_or(Transparency::Opaque),
), ),

9
crates/hir-expand/src/lib.rs
View file

@ -30,10 +30,11 @@ use triomphe::Arc;
use std::{fmt, hash::Hash}; use std::{fmt, hash::Hash};
use base_db::{salsa::impl_intern_value_trivial, CrateId, Edition, FileId}; use base_db::{salsa::impl_intern_value_trivial, CrateId, FileId};
use either::Either; use either::Either;
use span::{ use span::{
ErasedFileAstId, FileRange, HirFileIdRepr, Span, SpanAnchor, SyntaxContextData, SyntaxContextId, Edition, ErasedFileAstId, FileRange, HirFileIdRepr, Span, SpanAnchor, SyntaxContextData,
SyntaxContextId,
}; };
use syntax::{ use syntax::{
ast::{self, AstNode}, ast::{self, AstNode},
@ -53,11 +54,9 @@ use crate::{
pub use crate::files::{AstId, ErasedAstId, InFile, InMacroFile, InRealFile}; pub use crate::files::{AstId, ErasedAstId, InFile, InMacroFile, InRealFile};
pub use mbe::ValueResult; pub use mbe::{DeclarativeMacro, ValueResult};
pub use span::{HirFileId, MacroCallId, MacroFileId}; pub use span::{HirFileId, MacroCallId, MacroFileId};
pub type DeclarativeMacro = ::mbe::DeclarativeMacro<tt::Span>;
pub mod tt { pub mod tt {
pub use span::Span; pub use span::Span;
pub use tt::{DelimiterKind, Spacing}; pub use tt::{DelimiterKind, Spacing};

1
crates/hir-ty/Cargo.toml
View file

@ -47,6 +47,7 @@ hir-expand.workspace = true
base-db.workspace = true base-db.workspace = true
syntax.workspace = true syntax.workspace = true
limit.workspace = true limit.workspace = true
span.workspace = true
[dev-dependencies] [dev-dependencies]
expect-test = "1.4.0" expect-test = "1.4.0"

3
crates/hir-ty/src/method_resolution.rs
View file

@ -4,7 +4,7 @@
//! and the corresponding code mostly in rustc_hir_analysis/check/method/probe.rs. //! and the corresponding code mostly in rustc_hir_analysis/check/method/probe.rs.
use std::ops::ControlFlow; use std::ops::ControlFlow;
use base_db::{CrateId, Edition}; use base_db::CrateId;
use chalk_ir::{cast::Cast, Mutability, TyKind, UniverseIndex, WhereClause}; use chalk_ir::{cast::Cast, Mutability, TyKind, UniverseIndex, WhereClause};
use hir_def::{ use hir_def::{
data::{adt::StructFlags, ImplData}, data::{adt::StructFlags, ImplData},
@ -15,6 +15,7 @@ use hir_def::{
use hir_expand::name::Name; use hir_expand::name::Name;
use rustc_hash::{FxHashMap, FxHashSet}; use rustc_hash::{FxHashMap, FxHashSet};
use smallvec::{smallvec, SmallVec}; use smallvec::{smallvec, SmallVec};
use span::Edition;
use stdx::never; use stdx::never;
use triomphe::Arc; use triomphe::Arc;

3
crates/hir/src/lib.rs
View file

@ -38,7 +38,7 @@ mod display;
use std::{iter, mem::discriminant, ops::ControlFlow}; use std::{iter, mem::discriminant, ops::ControlFlow};
use arrayvec::ArrayVec; use arrayvec::ArrayVec;
use base_db::{CrateDisplayName, CrateId, CrateOrigin, Edition, FileId}; use base_db::{CrateDisplayName, CrateId, CrateOrigin, FileId};
use either::Either; use either::Either;
use hir_def::{ use hir_def::{
body::{BodyDiagnostic, SyntheticSyntax}, body::{BodyDiagnostic, SyntheticSyntax},
@ -79,6 +79,7 @@ use hir_ty::{
use itertools::Itertools; use itertools::Itertools;
use nameres::diagnostics::DefDiagnosticKind; use nameres::diagnostics::DefDiagnosticKind;
use rustc_hash::FxHashSet; use rustc_hash::FxHashSet;
use span::Edition;
use stdx::{impl_from, never}; use stdx::{impl_from, never};
use syntax::{ use syntax::{
ast::{self, HasAttrs as _, HasName}, ast::{self, HasAttrs as _, HasName},

1
crates/ide/Cargo.toml
View file

@ -36,6 +36,7 @@ ide-ssr.workspace = true
profile.workspace = true profile.workspace = true
stdx.workspace = true stdx.workspace = true
syntax.workspace = true syntax.workspace = true
span.workspace = true
text-edit.workspace = true text-edit.workspace = true
# ide should depend only on the top-level `hir` package. if you need # ide should depend only on the top-level `hir` package. if you need
# something from some `hir-xxx` subpackage, reexport the API via `hir`. # something from some `hir-xxx` subpackage, reexport the API via `hir`.

5
crates/ide/src/lib.rs
View file

@ -119,8 +119,8 @@ pub use ide_completion::{
}; };
pub use ide_db::{ pub use ide_db::{
base_db::{ base_db::{
Cancelled, CrateGraph, CrateId, Edition, FileChange, FileId, FilePosition, FileRange, Cancelled, CrateGraph, CrateId, FileChange, FileId, FilePosition, FileRange, SourceRoot,
SourceRoot, SourceRootId, SourceRootId,
}, },
documentation::Documentation, documentation::Documentation,
label::Label, label::Label,
@ -135,6 +135,7 @@ pub use ide_diagnostics::{
Diagnostic, DiagnosticCode, DiagnosticsConfig, ExprFillDefaultMode, Severity, Diagnostic, DiagnosticCode, DiagnosticsConfig, ExprFillDefaultMode, Severity,
}; };
pub use ide_ssr::SsrError; pub use ide_ssr::SsrError;
pub use span::Edition;
pub use syntax::{TextRange, TextSize}; pub use syntax::{TextRange, TextSize};
pub use text_edit::{Indel, TextEdit}; pub use text_edit::{Indel, TextEdit};

20
crates/mbe/src/benchmark.rs
View file

@ -23,7 +23,11 @@ fn benchmark_parse_macro_rules() {
let _pt = bench("mbe parse macro rules"); let _pt = bench("mbe parse macro rules");
rules rules
.values() .values()
.map(|it| DeclarativeMacro::parse_macro_rules(it, true, true).rules.len()) .map(|it| {
DeclarativeMacro::parse_macro_rules(it, |_| span::Edition::CURRENT, true)
.rules
.len()
})
.sum() .sum()
}; };
assert_eq!(hash, 1144); assert_eq!(hash, 1144);
@ -51,10 +55,12 @@ fn benchmark_expand_macro_rules() {
assert_eq!(hash, 69413); assert_eq!(hash, 69413);
} }
fn macro_rules_fixtures() -> FxHashMap<String, DeclarativeMacro<Span>> { fn macro_rules_fixtures() -> FxHashMap<String, DeclarativeMacro> {
macro_rules_fixtures_tt() macro_rules_fixtures_tt()
.into_iter() .into_iter()
.map(|(id, tt)| (id, DeclarativeMacro::parse_macro_rules(&tt, true, true))) .map(|(id, tt)| {
(id, DeclarativeMacro::parse_macro_rules(&tt, |_| span::Edition::CURRENT, true))
})
.collect() .collect()
} }
@ -80,7 +86,7 @@ fn macro_rules_fixtures_tt() -> FxHashMap<String, tt::Subtree<Span>> {
/// Generate random invocation fixtures from rules /// Generate random invocation fixtures from rules
fn invocation_fixtures( fn invocation_fixtures(
rules: &FxHashMap<String, DeclarativeMacro<Span>>, rules: &FxHashMap<String, DeclarativeMacro>,
) -> Vec<(String, tt::Subtree<Span>)> { ) -> Vec<(String, tt::Subtree<Span>)> {
let mut seed = 123456789; let mut seed = 123456789;
let mut res = Vec::new(); let mut res = Vec::new();
@ -128,11 +134,7 @@ fn invocation_fixtures(
} }
return res; return res;
fn collect_from_op( fn collect_from_op(op: &Op, token_trees: &mut Vec<tt::TokenTree<Span>>, seed: &mut usize) {
op: &Op<Span>,
token_trees: &mut Vec<tt::TokenTree<Span>>,
seed: &mut usize,
) {
return match op { return match op {
Op::Var { kind, .. } => match kind.as_ref() { Op::Var { kind, .. } => match kind.as_ref() {
Some(MetaVarKind::Ident) => token_trees.push(make_ident("foo")), Some(MetaVarKind::Ident) => token_trees.push(make_ident("foo")),

45
crates/mbe/src/expander.rs
View file

@ -6,22 +6,21 @@ mod matcher;
mod transcriber; mod transcriber;
use rustc_hash::FxHashMap; use rustc_hash::FxHashMap;
use span::Span;
use syntax::SmolStr; use syntax::SmolStr;
use tt::Span;
use crate::{parser::MetaVarKind, ExpandError, ExpandResult}; use crate::{parser::MetaVarKind, ExpandError, ExpandResult};
pub(crate) fn expand_rules<S: Span>( pub(crate) fn expand_rules(
rules: &[crate::Rule<S>], rules: &[crate::Rule],
input: &tt::Subtree<S>, input: &tt::Subtree<Span>,
marker: impl Fn(&mut S) + Copy, marker: impl Fn(&mut Span) + Copy,
is_2021: bool,
new_meta_vars: bool, new_meta_vars: bool,
call_site: S, call_site: Span,
) -> ExpandResult<tt::Subtree<S>> { ) -> ExpandResult<tt::Subtree<Span>> {
let mut match_: Option<(matcher::Match<S>, &crate::Rule<S>)> = None; let mut match_: Option<(matcher::Match, &crate::Rule)> = None;
for rule in rules { for rule in rules {
let new_match = matcher::match_(&rule.lhs, input, is_2021); let new_match = matcher::match_(&rule.lhs, input);
if new_match.err.is_none() { if new_match.err.is_none() {
// If we find a rule that applies without errors, we're done. // If we find a rule that applies without errors, we're done.
@ -110,30 +109,24 @@ pub(crate) fn expand_rules<S: Span>(
/// In other words, `Bindings` is a *multi* mapping from `SmolStr` to /// In other words, `Bindings` is a *multi* mapping from `SmolStr` to
/// `tt::TokenTree`, where the index to select a particular `TokenTree` among /// `tt::TokenTree`, where the index to select a particular `TokenTree` among
/// many is not a plain `usize`, but a `&[usize]`. /// many is not a plain `usize`, but a `&[usize]`.
#[derive(Debug, Clone, PartialEq, Eq)] #[derive(Debug, Default, Clone, PartialEq, Eq)]
struct Bindings<S> { struct Bindings {
inner: FxHashMap<SmolStr, Binding<S>>, inner: FxHashMap<SmolStr, Binding>,
}
impl<S> Default for Bindings<S> {
fn default() -> Self {
Self { inner: Default::default() }
}
} }
#[derive(Debug, Clone, PartialEq, Eq)] #[derive(Debug, Clone, PartialEq, Eq)]
enum Binding<S> { enum Binding {
Fragment(Fragment<S>), Fragment(Fragment),
Nested(Vec<Binding<S>>), Nested(Vec<Binding>),
Empty, Empty,
Missing(MetaVarKind), Missing(MetaVarKind),
} }
#[derive(Debug, Clone, PartialEq, Eq)] #[derive(Debug, Clone, PartialEq, Eq)]
enum Fragment<S> { enum Fragment {
Empty, Empty,
/// token fragments are just copy-pasted into the output /// token fragments are just copy-pasted into the output
Tokens(tt::TokenTree<S>), Tokens(tt::TokenTree<Span>),
/// Expr ast fragments are surrounded with `()` on insertion to preserve /// Expr ast fragments are surrounded with `()` on insertion to preserve
/// precedence. Note that this impl is different from the one currently in /// precedence. Note that this impl is different from the one currently in
/// `rustc` -- `rustc` doesn't translate fragments into token trees at all. /// `rustc` -- `rustc` doesn't translate fragments into token trees at all.
@ -141,7 +134,7 @@ enum Fragment<S> {
/// At one point in time, we tried to use "fake" delimiters here à la /// At one point in time, we tried to use "fake" delimiters here à la
/// proc-macro delimiter=none. As we later discovered, "none" delimiters are /// proc-macro delimiter=none. As we later discovered, "none" delimiters are
/// tricky to handle in the parser, and rustc doesn't handle those either. /// tricky to handle in the parser, and rustc doesn't handle those either.
Expr(tt::Subtree<S>), Expr(tt::Subtree<Span>),
/// There are roughly two types of paths: paths in expression context, where a /// There are roughly two types of paths: paths in expression context, where a
/// separator `::` between an identifier and its following generic argument list /// separator `::` between an identifier and its following generic argument list
/// is mandatory, and paths in type context, where `::` can be omitted. /// is mandatory, and paths in type context, where `::` can be omitted.
@ -151,5 +144,5 @@ enum Fragment<S> {
/// and is trasncribed as an expression-context path, verbatim transcription /// and is trasncribed as an expression-context path, verbatim transcription
/// would cause a syntax error. We need to fix it up just before transcribing; /// would cause a syntax error. We need to fix it up just before transcribing;
/// see `transcriber::fix_up_and_push_path_tt()`. /// see `transcriber::fix_up_and_push_path_tt()`.
Path(tt::Subtree<S>), Path(tt::Subtree<Span>),
} }

160
crates/mbe/src/expander/matcher.rs
View file

@ -62,8 +62,9 @@
use std::rc::Rc; use std::rc::Rc;
use smallvec::{smallvec, SmallVec}; use smallvec::{smallvec, SmallVec};
use span::Span;
use syntax::SmolStr; use syntax::SmolStr;
use tt::{DelimSpan, Span}; use tt::DelimSpan;
use crate::{ use crate::{
expander::{Binding, Bindings, ExpandResult, Fragment}, expander::{Binding, Bindings, ExpandResult, Fragment},
@ -72,7 +73,7 @@ use crate::{
ExpandError, MetaTemplate, ValueResult, ExpandError, MetaTemplate, ValueResult,
}; };
impl<S: Span> Bindings<S> { impl Bindings {
fn push_optional(&mut self, name: &SmolStr) { fn push_optional(&mut self, name: &SmolStr) {
self.inner.insert(name.clone(), Binding::Fragment(Fragment::Empty)); self.inner.insert(name.clone(), Binding::Fragment(Fragment::Empty));
} }
@ -81,14 +82,14 @@ impl<S: Span> Bindings<S> {
self.inner.insert(name.clone(), Binding::Empty); self.inner.insert(name.clone(), Binding::Empty);
} }
fn bindings(&self) -> impl Iterator<Item = &Binding<S>> { fn bindings(&self) -> impl Iterator<Item = &Binding> {
self.inner.values() self.inner.values()
} }
} }
#[derive(Clone, Debug, PartialEq, Eq)] #[derive(Clone, Default, Debug, PartialEq, Eq)]
pub(super) struct Match<S> { pub(super) struct Match {
pub(super) bindings: Bindings<S>, pub(super) bindings: Bindings,
/// We currently just keep the first error and count the rest to compare matches. /// We currently just keep the first error and count the rest to compare matches.
pub(super) err: Option<ExpandError>, pub(super) err: Option<ExpandError>,
pub(super) err_count: usize, pub(super) err_count: usize,
@ -98,19 +99,7 @@ pub(super) struct Match<S> {
pub(super) bound_count: usize, pub(super) bound_count: usize,
} }
impl<S> Default for Match<S> { impl Match {
fn default() -> Self {
Self {
bindings: Default::default(),
err: Default::default(),
err_count: Default::default(),
unmatched_tts: Default::default(),
bound_count: Default::default(),
}
}
}
impl<S> Match<S> {
fn add_err(&mut self, err: ExpandError) { fn add_err(&mut self, err: ExpandError) {
let prev_err = self.err.take(); let prev_err = self.err.take();
self.err = prev_err.or(Some(err)); self.err = prev_err.or(Some(err));
@ -119,16 +108,12 @@ impl<S> Match<S> {
} }
/// Matching errors are added to the `Match`. /// Matching errors are added to the `Match`.
pub(super) fn match_<S: Span>( pub(super) fn match_(pattern: &MetaTemplate, input: &tt::Subtree<Span>) -> Match {
pattern: &MetaTemplate<S>, let mut res = match_loop(pattern, input);
input: &tt::Subtree<S>,
is_2021: bool,
) -> Match<S> {
let mut res = match_loop(pattern, input, is_2021);
res.bound_count = count(res.bindings.bindings()); res.bound_count = count(res.bindings.bindings());
return res; return res;
fn count<'a, S: 'a>(bindings: impl Iterator<Item = &'a Binding<S>>) -> usize { fn count<'a>(bindings: impl Iterator<Item = &'a Binding>) -> usize {
bindings bindings
.map(|it| match it { .map(|it| match it {
Binding::Fragment(_) => 1, Binding::Fragment(_) => 1,
@ -141,10 +126,10 @@ pub(super) fn match_<S: Span>(
} }
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
enum BindingKind<S> { enum BindingKind {
Empty(SmolStr), Empty(SmolStr),
Optional(SmolStr), Optional(SmolStr),
Fragment(SmolStr, Fragment<S>), Fragment(SmolStr, Fragment),
Missing(SmolStr, MetaVarKind), Missing(SmolStr, MetaVarKind),
Nested(usize, usize), Nested(usize, usize),
} }
@ -158,18 +143,13 @@ enum LinkNode<T> {
Parent { idx: usize, len: usize }, Parent { idx: usize, len: usize },
} }
struct BindingsBuilder<S> { #[derive(Default)]
nodes: Vec<Vec<LinkNode<Rc<BindingKind<S>>>>>, struct BindingsBuilder {
nodes: Vec<Vec<LinkNode<Rc<BindingKind>>>>,
nested: Vec<Vec<LinkNode<usize>>>, nested: Vec<Vec<LinkNode<usize>>>,
} }
impl<S> Default for BindingsBuilder<S> { impl BindingsBuilder {
fn default() -> Self {
Self { nodes: Default::default(), nested: Default::default() }
}
}
impl<S: Span> BindingsBuilder<S> {
fn alloc(&mut self) -> BindingsIdx { fn alloc(&mut self) -> BindingsIdx {
let idx = self.nodes.len(); let idx = self.nodes.len();
self.nodes.push(Vec::new()); self.nodes.push(Vec::new());
@ -206,7 +186,7 @@ impl<S: Span> BindingsBuilder<S> {
self.nodes[idx.0].push(LinkNode::Node(Rc::new(BindingKind::Optional(var.clone())))); self.nodes[idx.0].push(LinkNode::Node(Rc::new(BindingKind::Optional(var.clone()))));
} }
fn push_fragment(&mut self, idx: &mut BindingsIdx, var: &SmolStr, fragment: Fragment<S>) { fn push_fragment(&mut self, idx: &mut BindingsIdx, var: &SmolStr, fragment: Fragment) {
self.nodes[idx.0] self.nodes[idx.0]
.push(LinkNode::Node(Rc::new(BindingKind::Fragment(var.clone(), fragment)))); .push(LinkNode::Node(Rc::new(BindingKind::Fragment(var.clone(), fragment))));
} }
@ -227,11 +207,11 @@ impl<S: Span> BindingsBuilder<S> {
idx.0 = new_idx; idx.0 = new_idx;
} }
fn build(self, idx: &BindingsIdx) -> Bindings<S> { fn build(self, idx: &BindingsIdx) -> Bindings {
self.build_inner(&self.nodes[idx.0]) self.build_inner(&self.nodes[idx.0])
} }
fn build_inner(&self, link_nodes: &[LinkNode<Rc<BindingKind<S>>>]) -> Bindings<S> { fn build_inner(&self, link_nodes: &[LinkNode<Rc<BindingKind>>]) -> Bindings {
let mut bindings = Bindings::default(); let mut bindings = Bindings::default();
let mut nodes = Vec::new(); let mut nodes = Vec::new();
self.collect_nodes(link_nodes, &mut nodes); self.collect_nodes(link_nodes, &mut nodes);
@ -281,7 +261,7 @@ impl<S: Span> BindingsBuilder<S> {
&'a self, &'a self,
id: usize, id: usize,
len: usize, len: usize,
nested_refs: &mut Vec<&'a [LinkNode<Rc<BindingKind<S>>>]>, nested_refs: &mut Vec<&'a [LinkNode<Rc<BindingKind>>]>,
) { ) {
self.nested[id].iter().take(len).for_each(|it| match it { self.nested[id].iter().take(len).for_each(|it| match it {
LinkNode::Node(id) => nested_refs.push(&self.nodes[*id]), LinkNode::Node(id) => nested_refs.push(&self.nodes[*id]),
@ -289,7 +269,7 @@ impl<S: Span> BindingsBuilder<S> {
}); });
} }
fn collect_nested(&self, idx: usize, nested_idx: usize, nested: &mut Vec<Bindings<S>>) { fn collect_nested(&self, idx: usize, nested_idx: usize, nested: &mut Vec<Bindings>) {
let last = &self.nodes[idx]; let last = &self.nodes[idx];
let mut nested_refs: Vec<&[_]> = Vec::new(); let mut nested_refs: Vec<&[_]> = Vec::new();
self.nested[nested_idx].iter().for_each(|it| match *it { self.nested[nested_idx].iter().for_each(|it| match *it {
@ -300,7 +280,7 @@ impl<S: Span> BindingsBuilder<S> {
nested.extend(nested_refs.into_iter().map(|iter| self.build_inner(iter))); nested.extend(nested_refs.into_iter().map(|iter| self.build_inner(iter)));
} }
fn collect_nodes_ref<'a>(&'a self, id: usize, len: usize, nodes: &mut Vec<&'a BindingKind<S>>) { fn collect_nodes_ref<'a>(&'a self, id: usize, len: usize, nodes: &mut Vec<&'a BindingKind>) {
self.nodes[id].iter().take(len).for_each(|it| match it { self.nodes[id].iter().take(len).for_each(|it| match it {
LinkNode::Node(it) => nodes.push(it), LinkNode::Node(it) => nodes.push(it),
LinkNode::Parent { idx, len } => self.collect_nodes_ref(*idx, *len, nodes), LinkNode::Parent { idx, len } => self.collect_nodes_ref(*idx, *len, nodes),
@ -309,8 +289,8 @@ impl<S: Span> BindingsBuilder<S> {
fn collect_nodes<'a>( fn collect_nodes<'a>(
&'a self, &'a self,
link_nodes: &'a [LinkNode<Rc<BindingKind<S>>>], link_nodes: &'a [LinkNode<Rc<BindingKind>>],
nodes: &mut Vec<&'a BindingKind<S>>, nodes: &mut Vec<&'a BindingKind>,
) { ) {
link_nodes.iter().for_each(|it| match it { link_nodes.iter().for_each(|it| match it {
LinkNode::Node(it) => nodes.push(it), LinkNode::Node(it) => nodes.push(it),
@ -320,22 +300,22 @@ impl<S: Span> BindingsBuilder<S> {
} }
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
struct MatchState<'t, S> { struct MatchState<'t> {
/// The position of the "dot" in this matcher /// The position of the "dot" in this matcher
dot: OpDelimitedIter<'t, S>, dot: OpDelimitedIter<'t>,
/// Token subtree stack /// Token subtree stack
/// When matching against matchers with nested delimited submatchers (e.g., `pat ( pat ( .. ) /// When matching against matchers with nested delimited submatchers (e.g., `pat ( pat ( .. )
/// pat ) pat`), we need to keep track of the matchers we are descending into. This stack does /// pat ) pat`), we need to keep track of the matchers we are descending into. This stack does
/// that where the bottom of the stack is the outermost matcher. /// that where the bottom of the stack is the outermost matcher.
stack: SmallVec<[OpDelimitedIter<'t, S>; 4]>, stack: SmallVec<[OpDelimitedIter<'t>; 4]>,
/// The "parent" matcher position if we are in a repetition. That is, the matcher position just /// The "parent" matcher position if we are in a repetition. That is, the matcher position just
/// before we enter the repetition. /// before we enter the repetition.
up: Option<Box<MatchState<'t, S>>>, up: Option<Box<MatchState<'t>>>,
/// The separator if we are in a repetition. /// The separator if we are in a repetition.
sep: Option<Separator<S>>, sep: Option<Separator>,
/// The KleeneOp of this sequence if we are in a repetition. /// The KleeneOp of this sequence if we are in a repetition.
sep_kind: Option<RepeatKind>, sep_kind: Option<RepeatKind>,
@ -347,7 +327,7 @@ struct MatchState<'t, S> {
bindings: BindingsIdx, bindings: BindingsIdx,
/// Cached result of meta variable parsing /// Cached result of meta variable parsing
meta_result: Option<(TtIter<'t, S>, ExpandResult<Option<Fragment<S>>>)>, meta_result: Option<(TtIter<'t, Span>, ExpandResult<Option<Fragment>>)>,
/// Is error occurred in this state, will `poised` to "parent" /// Is error occurred in this state, will `poised` to "parent"
is_error: bool, is_error: bool,
@ -372,18 +352,17 @@ struct MatchState<'t, S> {
/// - `bb_items`: the set of items that are waiting for the black-box parser. /// - `bb_items`: the set of items that are waiting for the black-box parser.
/// - `error_items`: the set of items in errors, used for error-resilient parsing /// - `error_items`: the set of items in errors, used for error-resilient parsing
#[inline] #[inline]
fn match_loop_inner<'t, S: Span>( fn match_loop_inner<'t>(
src: TtIter<'t, S>, src: TtIter<'t, Span>,
stack: &[TtIter<'t, S>], stack: &[TtIter<'t, Span>],
res: &mut Match<S>, res: &mut Match,
bindings_builder: &mut BindingsBuilder<S>, bindings_builder: &mut BindingsBuilder,
cur_items: &mut SmallVec<[MatchState<'t, S>; 1]>, cur_items: &mut SmallVec<[MatchState<'t>; 1]>,
bb_items: &mut SmallVec<[MatchState<'t, S>; 1]>, bb_items: &mut SmallVec<[MatchState<'t>; 1]>,
next_items: &mut Vec<MatchState<'t, S>>, next_items: &mut Vec<MatchState<'t>>,
eof_items: &mut SmallVec<[MatchState<'t, S>; 1]>, eof_items: &mut SmallVec<[MatchState<'t>; 1]>,
error_items: &mut SmallVec<[MatchState<'t, S>; 1]>, error_items: &mut SmallVec<[MatchState<'t>; 1]>,
is_2021: bool, delim_span: tt::DelimSpan<Span>,
delim_span: tt::DelimSpan<S>,
) { ) {
macro_rules! try_push { macro_rules! try_push {
($items: expr, $it:expr) => { ($items: expr, $it:expr) => {
@ -494,7 +473,7 @@ fn match_loop_inner<'t, S: Span>(
OpDelimited::Op(Op::Var { kind, name, .. }) => { OpDelimited::Op(Op::Var { kind, name, .. }) => {
if let &Some(kind) = kind { if let &Some(kind) = kind {
let mut fork = src.clone(); let mut fork = src.clone();
let match_res = match_meta_var(kind, &mut fork, is_2021, delim_span); let match_res = match_meta_var(kind, &mut fork, delim_span);
match match_res.err { match match_res.err {
None => { None => {
// Some meta variables are optional (e.g. vis) // Some meta variables are optional (e.g. vis)
@ -607,10 +586,10 @@ fn match_loop_inner<'t, S: Span>(
} }
} }
fn match_loop<S: Span>(pattern: &MetaTemplate<S>, src: &tt::Subtree<S>, is_2021: bool) -> Match<S> { fn match_loop(pattern: &MetaTemplate, src: &tt::Subtree<Span>) -> Match {
let span = src.delimiter.delim_span(); let span = src.delimiter.delim_span();
let mut src = TtIter::new(src); let mut src = TtIter::new(src);
let mut stack: SmallVec<[TtIter<'_, S>; 1]> = SmallVec::new(); let mut stack: SmallVec<[TtIter<'_, Span>; 1]> = SmallVec::new();
let mut res = Match::default(); let mut res = Match::default();
let mut error_recover_item = None; let mut error_recover_item = None;
@ -647,7 +626,6 @@ fn match_loop<S: Span>(pattern: &MetaTemplate<S>, src: &tt::Subtree<S>, is_2021:
&mut next_items, &mut next_items,
&mut eof_items, &mut eof_items,
&mut error_items, &mut error_items,
is_2021,
span, span,
); );
stdx::always!(cur_items.is_empty()); stdx::always!(cur_items.is_empty());
@ -758,12 +736,11 @@ fn match_loop<S: Span>(pattern: &MetaTemplate<S>, src: &tt::Subtree<S>, is_2021:
} }
} }
fn match_meta_var<S: Span>( fn match_meta_var(
kind: MetaVarKind, kind: MetaVarKind,
input: &mut TtIter<'_, S>, input: &mut TtIter<'_, Span>,
is_2021: bool, delim_span: DelimSpan<Span>,
delim_span: DelimSpan<S>, ) -> ExpandResult<Option<Fragment>> {
) -> ExpandResult<Option<Fragment<S>>> {
let fragment = match kind { let fragment = match kind {
MetaVarKind::Path => { MetaVarKind::Path => {
return input.expect_fragment(parser::PrefixEntryPoint::Path).map(|it| { return input.expect_fragment(parser::PrefixEntryPoint::Path).map(|it| {
@ -771,8 +748,7 @@ fn match_meta_var<S: Span>(
}); });
} }
MetaVarKind::Ty => parser::PrefixEntryPoint::Ty, MetaVarKind::Ty => parser::PrefixEntryPoint::Ty,
MetaVarKind::Pat if is_2021 => parser::PrefixEntryPoint::PatTop, MetaVarKind::Pat => parser::PrefixEntryPoint::PatTop,
MetaVarKind::Pat => parser::PrefixEntryPoint::Pat,
MetaVarKind::PatParam => parser::PrefixEntryPoint::Pat, MetaVarKind::PatParam => parser::PrefixEntryPoint::Pat,
MetaVarKind::Stmt => parser::PrefixEntryPoint::Stmt, MetaVarKind::Stmt => parser::PrefixEntryPoint::Stmt,
MetaVarKind::Block => parser::PrefixEntryPoint::Block, MetaVarKind::Block => parser::PrefixEntryPoint::Block,
@ -846,7 +822,7 @@ fn match_meta_var<S: Span>(
input.expect_fragment(fragment).map(|it| it.map(Fragment::Tokens)) input.expect_fragment(fragment).map(|it| it.map(Fragment::Tokens))
} }
fn collect_vars<S: Span>(collector_fun: &mut impl FnMut(SmolStr), pattern: &MetaTemplate<S>) { fn collect_vars(collector_fun: &mut impl FnMut(SmolStr), pattern: &MetaTemplate) {
for op in pattern.iter() { for op in pattern.iter() {
match op { match op {
Op::Var { name, .. } => collector_fun(name.clone()), Op::Var { name, .. } => collector_fun(name.clone()),
@ -859,11 +835,11 @@ fn collect_vars<S: Span>(collector_fun: &mut impl FnMut(SmolStr), pattern: &Meta
} }
} }
} }
impl<S: Span> MetaTemplate<S> { impl MetaTemplate {
fn iter_delimited_with(&self, delimiter: tt::Delimiter<S>) -> OpDelimitedIter<'_, S> { fn iter_delimited_with(&self, delimiter: tt::Delimiter<Span>) -> OpDelimitedIter<'_> {
OpDelimitedIter { inner: &self.0, idx: 0, delimited: delimiter } OpDelimitedIter { inner: &self.0, idx: 0, delimited: delimiter }
} }
fn iter_delimited(&self, span: tt::DelimSpan<S>) -> OpDelimitedIter<'_, S> { fn iter_delimited(&self, span: tt::DelimSpan<Span>) -> OpDelimitedIter<'_> {
OpDelimitedIter { OpDelimitedIter {
inner: &self.0, inner: &self.0,
idx: 0, idx: 0,
@ -873,27 +849,27 @@ impl<S: Span> MetaTemplate<S> {
} }
#[derive(Debug, Clone, Copy)] #[derive(Debug, Clone, Copy)]
enum OpDelimited<'a, S> { enum OpDelimited<'a> {
Op(&'a Op<S>), Op(&'a Op),
Open, Open,
Close, Close,
} }
#[derive(Debug, Clone, Copy)] #[derive(Debug, Clone, Copy)]
struct OpDelimitedIter<'a, S> { struct OpDelimitedIter<'a> {
inner: &'a [Op<S>], inner: &'a [Op],
delimited: tt::Delimiter<S>, delimited: tt::Delimiter<Span>,
idx: usize, idx: usize,
} }
impl<'a, S: Span> OpDelimitedIter<'a, S> { impl<'a> OpDelimitedIter<'a> {
fn is_eof(&self) -> bool { fn is_eof(&self) -> bool {
let len = self.inner.len() let len = self.inner.len()
+ if self.delimited.kind != tt::DelimiterKind::Invisible { 2 } else { 0 }; + if self.delimited.kind != tt::DelimiterKind::Invisible { 2 } else { 0 };
self.idx >= len self.idx >= len
} }
fn peek(&self) -> Option<OpDelimited<'a, S>> { fn peek(&self) -> Option<OpDelimited<'a>> {
match self.delimited.kind { match self.delimited.kind {
tt::DelimiterKind::Invisible => self.inner.get(self.idx).map(OpDelimited::Op), tt::DelimiterKind::Invisible => self.inner.get(self.idx).map(OpDelimited::Op),
_ => match self.idx { _ => match self.idx {
@ -909,8 +885,8 @@ impl<'a, S: Span> OpDelimitedIter<'a, S> {
} }
} }
impl<'a, S: Span> Iterator for OpDelimitedIter<'a, S> { impl<'a> Iterator for OpDelimitedIter<'a> {
type Item = OpDelimited<'a, S>; type Item = OpDelimited<'a>;
fn next(&mut self) -> Option<Self::Item> { fn next(&mut self) -> Option<Self::Item> {
let res = self.peek(); let res = self.peek();
@ -926,8 +902,8 @@ impl<'a, S: Span> Iterator for OpDelimitedIter<'a, S> {
} }
} }
impl<S: Span> TtIter<'_, S> { impl TtIter<'_, Span> {
fn expect_separator(&mut self, separator: &Separator<S>) -> bool { fn expect_separator(&mut self, separator: &Separator) -> bool {
let mut fork = self.clone(); let mut fork = self.clone();
let ok = match separator { let ok = match separator {
Separator::Ident(lhs) => match fork.expect_ident_or_underscore() { Separator::Ident(lhs) => match fork.expect_ident_or_underscore() {
@ -957,7 +933,7 @@ impl<S: Span> TtIter<'_, S> {
ok ok
} }
fn expect_tt(&mut self) -> Result<tt::TokenTree<S>, ()> { fn expect_tt(&mut self) -> Result<tt::TokenTree<Span>, ()> {
if let Some(tt::TokenTree::Leaf(tt::Leaf::Punct(punct))) = self.peek_n(0) { if let Some(tt::TokenTree::Leaf(tt::Leaf::Punct(punct))) = self.peek_n(0) {
if punct.char == '\'' { if punct.char == '\'' {
self.expect_lifetime() self.expect_lifetime()
@ -976,7 +952,7 @@ impl<S: Span> TtIter<'_, S> {
} }
} }
fn expect_lifetime(&mut self) -> Result<tt::TokenTree<S>, ()> { fn expect_lifetime(&mut self) -> Result<tt::TokenTree<Span>, ()> {
let punct = self.expect_single_punct()?; let punct = self.expect_single_punct()?;
if punct.char != '\'' { if punct.char != '\'' {
return Err(()); return Err(());
@ -997,7 +973,7 @@ impl<S: Span> TtIter<'_, S> {
.into()) .into())
} }
fn eat_char(&mut self, c: char) -> Option<tt::TokenTree<S>> { fn eat_char(&mut self, c: char) -> Option<tt::TokenTree<Span>> {
let mut fork = self.clone(); let mut fork = self.clone();
match fork.expect_char(c) { match fork.expect_char(c) {
Ok(_) => { Ok(_) => {

99
crates/mbe/src/expander/transcriber.rs
View file

@ -1,8 +1,9 @@
//! Transcriber takes a template, like `fn $ident() {}`, a set of bindings like //! Transcriber takes a template, like `fn $ident() {}`, a set of bindings like
//! `$ident => foo`, interpolates variables in the template, to get `fn foo() {}` //! `$ident => foo`, interpolates variables in the template, to get `fn foo() {}`
use span::Span;
use syntax::SmolStr; use syntax::SmolStr;
use tt::{Delimiter, Span}; use tt::Delimiter;
use crate::{ use crate::{
expander::{Binding, Bindings, Fragment}, expander::{Binding, Bindings, Fragment},
@ -10,8 +11,8 @@ use crate::{
CountError, ExpandError, ExpandResult, MetaTemplate, CountError, ExpandError, ExpandResult, MetaTemplate,
}; };
impl<S: Span> Bindings<S> { impl Bindings {
fn get(&self, name: &str) -> Result<&Binding<S>, ExpandError> { fn get(&self, name: &str) -> Result<&Binding, ExpandError> {
match self.inner.get(name) { match self.inner.get(name) {
Some(binding) => Ok(binding), Some(binding) => Ok(binding),
None => Err(ExpandError::UnresolvedBinding(Box::new(Box::from(name)))), None => Err(ExpandError::UnresolvedBinding(Box::new(Box::from(name)))),
@ -21,10 +22,10 @@ impl<S: Span> Bindings<S> {
fn get_fragment( fn get_fragment(
&self, &self,
name: &str, name: &str,
mut span: S, mut span: Span,
nesting: &mut [NestingState], nesting: &mut [NestingState],
marker: impl Fn(&mut S), marker: impl Fn(&mut Span),
) -> Result<Fragment<S>, ExpandError> { ) -> Result<Fragment, ExpandError> {
macro_rules! binding_err { macro_rules! binding_err {
($($arg:tt)*) => { ExpandError::binding_error(format!($($arg)*)) }; ($($arg:tt)*) => { ExpandError::binding_error(format!($($arg)*)) };
} }
@ -134,15 +135,15 @@ impl<S: Span> Bindings<S> {
} }
} }
pub(super) fn transcribe<S: Span>( pub(super) fn transcribe(
template: &MetaTemplate<S>, template: &MetaTemplate,
bindings: &Bindings<S>, bindings: &Bindings,
marker: impl Fn(&mut S) + Copy, marker: impl Fn(&mut Span) + Copy,
new_meta_vars: bool, new_meta_vars: bool,
call_site: S, call_site: Span,
) -> ExpandResult<tt::Subtree<S>> { ) -> ExpandResult<tt::Subtree<Span>> {
let mut ctx = ExpandCtx { bindings, nesting: Vec::new(), new_meta_vars, call_site }; let mut ctx = ExpandCtx { bindings, nesting: Vec::new(), new_meta_vars, call_site };
let mut arena: Vec<tt::TokenTree<S>> = Vec::new(); let mut arena: Vec<tt::TokenTree<Span>> = Vec::new();
expand_subtree(&mut ctx, template, None, &mut arena, marker) expand_subtree(&mut ctx, template, None, &mut arena, marker)
} }
@ -158,20 +159,20 @@ struct NestingState {
} }
#[derive(Debug)] #[derive(Debug)]
struct ExpandCtx<'a, S> { struct ExpandCtx<'a> {
bindings: &'a Bindings<S>, bindings: &'a Bindings,
nesting: Vec<NestingState>, nesting: Vec<NestingState>,
new_meta_vars: bool, new_meta_vars: bool,
call_site: S, call_site: Span,
} }
fn expand_subtree<S: Span>( fn expand_subtree(
ctx: &mut ExpandCtx<'_, S>, ctx: &mut ExpandCtx<'_>,
template: &MetaTemplate<S>, template: &MetaTemplate,
delimiter: Option<Delimiter<S>>, delimiter: Option<Delimiter<Span>>,
arena: &mut Vec<tt::TokenTree<S>>, arena: &mut Vec<tt::TokenTree<Span>>,
marker: impl Fn(&mut S) + Copy, marker: impl Fn(&mut Span) + Copy,
) -> ExpandResult<tt::Subtree<S>> { ) -> ExpandResult<tt::Subtree<Span>> {
// remember how many elements are in the arena now - when returning, we want to drain exactly how many elements we added. This way, the recursive uses of the arena get their own "view" of the arena, but will reuse the allocation // remember how many elements are in the arena now - when returning, we want to drain exactly how many elements we added. This way, the recursive uses of the arena get their own "view" of the arena, but will reuse the allocation
let start_elements = arena.len(); let start_elements = arena.len();
let mut err = None; let mut err = None;
@ -332,12 +333,12 @@ fn expand_subtree<S: Span>(
} }
} }
fn expand_var<S: Span>( fn expand_var(
ctx: &mut ExpandCtx<'_, S>, ctx: &mut ExpandCtx<'_>,
v: &SmolStr, v: &SmolStr,
id: S, id: Span,
marker: impl Fn(&mut S), marker: impl Fn(&mut Span),
) -> ExpandResult<Fragment<S>> { ) -> ExpandResult<Fragment> {
// We already handle $crate case in mbe parser // We already handle $crate case in mbe parser
debug_assert!(v != "crate"); debug_assert!(v != "crate");
@ -378,15 +379,15 @@ fn expand_var<S: Span>(
} }
} }
fn expand_repeat<S: Span>( fn expand_repeat(
ctx: &mut ExpandCtx<'_, S>, ctx: &mut ExpandCtx<'_>,
template: &MetaTemplate<S>, template: &MetaTemplate,
kind: RepeatKind, kind: RepeatKind,
separator: &Option<Separator<S>>, separator: &Option<Separator>,
arena: &mut Vec<tt::TokenTree<S>>, arena: &mut Vec<tt::TokenTree<Span>>,
marker: impl Fn(&mut S) + Copy, marker: impl Fn(&mut Span) + Copy,
) -> ExpandResult<Fragment<S>> { ) -> ExpandResult<Fragment> {
let mut buf: Vec<tt::TokenTree<S>> = Vec::new(); let mut buf: Vec<tt::TokenTree<Span>> = Vec::new();
ctx.nesting.push(NestingState { idx: 0, at_end: false, hit: false }); ctx.nesting.push(NestingState { idx: 0, at_end: false, hit: false });
// Dirty hack to make macro-expansion terminate. // Dirty hack to make macro-expansion terminate.
// This should be replaced by a proper macro-by-example implementation // This should be replaced by a proper macro-by-example implementation
@ -478,11 +479,7 @@ fn expand_repeat<S: Span>(
ExpandResult { value: Fragment::Tokens(tt), err } ExpandResult { value: Fragment::Tokens(tt), err }
} }
fn push_fragment<S: Span>( fn push_fragment(ctx: &ExpandCtx<'_>, buf: &mut Vec<tt::TokenTree<Span>>, fragment: Fragment) {
ctx: &ExpandCtx<'_, S>,
buf: &mut Vec<tt::TokenTree<S>>,
fragment: Fragment<S>,
) {
match fragment { match fragment {
Fragment::Tokens(tt::TokenTree::Subtree(tt)) => push_subtree(buf, tt), Fragment::Tokens(tt::TokenTree::Subtree(tt)) => push_subtree(buf, tt),
Fragment::Expr(sub) => { Fragment::Expr(sub) => {
@ -494,7 +491,7 @@ fn push_fragment<S: Span>(
} }
} }
fn push_subtree<S>(buf: &mut Vec<tt::TokenTree<S>>, tt: tt::Subtree<S>) { fn push_subtree(buf: &mut Vec<tt::TokenTree<Span>>, tt: tt::Subtree<Span>) {
match tt.delimiter.kind { match tt.delimiter.kind {
tt::DelimiterKind::Invisible => buf.extend(Vec::from(tt.token_trees)), tt::DelimiterKind::Invisible => buf.extend(Vec::from(tt.token_trees)),
_ => buf.push(tt.into()), _ => buf.push(tt.into()),
@ -504,10 +501,10 @@ fn push_subtree<S>(buf: &mut Vec<tt::TokenTree<S>>, tt: tt::Subtree<S>) {
/// Inserts the path separator `::` between an identifier and its following generic /// Inserts the path separator `::` between an identifier and its following generic
/// argument list, and then pushes into the buffer. See [`Fragment::Path`] for why /// argument list, and then pushes into the buffer. See [`Fragment::Path`] for why
/// we need this fixup. /// we need this fixup.
fn fix_up_and_push_path_tt<S: Span>( fn fix_up_and_push_path_tt(
ctx: &ExpandCtx<'_, S>, ctx: &ExpandCtx<'_>,
buf: &mut Vec<tt::TokenTree<S>>, buf: &mut Vec<tt::TokenTree<Span>>,
subtree: tt::Subtree<S>, subtree: tt::Subtree<Span>,
) { ) {
stdx::always!(matches!(subtree.delimiter.kind, tt::DelimiterKind::Invisible)); stdx::always!(matches!(subtree.delimiter.kind, tt::DelimiterKind::Invisible));
let mut prev_was_ident = false; let mut prev_was_ident = false;
@ -546,11 +543,7 @@ fn fix_up_and_push_path_tt<S: Span>(
/// Handles `${count(t, depth)}`. `our_depth` is the recursion depth and `count_depth` is the depth /// Handles `${count(t, depth)}`. `our_depth` is the recursion depth and `count_depth` is the depth
/// defined by the metavar expression. /// defined by the metavar expression.
fn count<S>( fn count(binding: &Binding, depth_curr: usize, depth_max: usize) -> Result<usize, CountError> {
binding: &Binding<S>,
depth_curr: usize,
depth_max: usize,
) -> Result<usize, CountError> {
match binding { match binding {
Binding::Nested(bs) => { Binding::Nested(bs) => {
if depth_curr == depth_max { if depth_curr == depth_max {
@ -564,8 +557,8 @@ fn count<S>(
} }
} }
fn count_old<S>( fn count_old(
binding: &Binding<S>, binding: &Binding,
our_depth: usize, our_depth: usize,
count_depth: Option<usize>, count_depth: Option<usize>,
) -> Result<usize, CountError> { ) -> Result<usize, CountError> {

64
crates/mbe/src/lib.rs
View file

@ -17,8 +17,8 @@ mod tt_iter;
#[cfg(test)] #[cfg(test)]
mod benchmark; mod benchmark;
use span::{Edition, Span, SyntaxContextId};
use stdx::impl_from; use stdx::impl_from;
use tt::Span;
use std::fmt; use std::fmt;
@ -127,32 +127,29 @@ impl fmt::Display for CountError {
/// `tt::TokenTree`, but there's a crucial difference: in macro rules, `$ident` /// `tt::TokenTree`, but there's a crucial difference: in macro rules, `$ident`
/// and `$()*` have special meaning (see `Var` and `Repeat` data structures) /// and `$()*` have special meaning (see `Var` and `Repeat` data structures)
#[derive(Clone, Debug, PartialEq, Eq)] #[derive(Clone, Debug, PartialEq, Eq)]
pub struct DeclarativeMacro<S> { pub struct DeclarativeMacro {
rules: Box<[Rule<S>]>, rules: Box<[Rule]>,
// This is used for correctly determining the behavior of the pat fragment
// FIXME: This should be tracked by hygiene of the fragment identifier!
is_2021: bool,
err: Option<Box<ParseError>>, err: Option<Box<ParseError>>,
} }
#[derive(Clone, Debug, PartialEq, Eq)] #[derive(Clone, Debug, PartialEq, Eq)]
struct Rule<S> { struct Rule {
lhs: MetaTemplate<S>, lhs: MetaTemplate,
rhs: MetaTemplate<S>, rhs: MetaTemplate,
} }
impl<S: Span> DeclarativeMacro<S> { impl DeclarativeMacro {
pub fn from_err(err: ParseError, is_2021: bool) -> DeclarativeMacro<S> { pub fn from_err(err: ParseError) -> DeclarativeMacro {
DeclarativeMacro { rules: Box::default(), is_2021, err: Some(Box::new(err)) } DeclarativeMacro { rules: Box::default(), err: Some(Box::new(err)) }
} }
/// The old, `macro_rules! m {}` flavor. /// The old, `macro_rules! m {}` flavor.
pub fn parse_macro_rules( pub fn parse_macro_rules(
tt: &tt::Subtree<S>, tt: &tt::Subtree<Span>,
is_2021: bool, edition: impl Copy + Fn(SyntaxContextId) -> Edition,
// FIXME: Remove this once we drop support for rust 1.76 (defaults to true then) // FIXME: Remove this once we drop support for rust 1.76 (defaults to true then)
new_meta_vars: bool, new_meta_vars: bool,
) -> DeclarativeMacro<S> { ) -> DeclarativeMacro {
// Note: this parsing can be implemented using mbe machinery itself, by // Note: this parsing can be implemented using mbe machinery itself, by
// matching against `$($lhs:tt => $rhs:tt);*` pattern, but implementing // matching against `$($lhs:tt => $rhs:tt);*` pattern, but implementing
// manually seems easier. // manually seems easier.
@ -161,7 +158,7 @@ impl<S: Span> DeclarativeMacro<S> {
let mut err = None; let mut err = None;
while src.len() > 0 { while src.len() > 0 {
let rule = match Rule::parse(&mut src, true, new_meta_vars) { let rule = match Rule::parse(edition, &mut src, true, new_meta_vars) {
Ok(it) => it, Ok(it) => it,
Err(e) => { Err(e) => {
err = Some(Box::new(e)); err = Some(Box::new(e));
@ -184,16 +181,16 @@ impl<S: Span> DeclarativeMacro<S> {
} }
} }
DeclarativeMacro { rules: rules.into_boxed_slice(), is_2021, err } DeclarativeMacro { rules: rules.into_boxed_slice(), err }
} }
/// The new, unstable `macro m {}` flavor. /// The new, unstable `macro m {}` flavor.
pub fn parse_macro2( pub fn parse_macro2(
tt: &tt::Subtree<S>, tt: &tt::Subtree<Span>,
is_2021: bool, edition: impl Copy + Fn(SyntaxContextId) -> Edition,
// FIXME: Remove this once we drop support for rust 1.76 (defaults to true then) // FIXME: Remove this once we drop support for rust 1.76 (defaults to true then)
new_meta_vars: bool, new_meta_vars: bool,
) -> DeclarativeMacro<S> { ) -> DeclarativeMacro {
let mut src = TtIter::new(tt); let mut src = TtIter::new(tt);
let mut rules = Vec::new(); let mut rules = Vec::new();
let mut err = None; let mut err = None;
@ -201,7 +198,7 @@ impl<S: Span> DeclarativeMacro<S> {
if tt::DelimiterKind::Brace == tt.delimiter.kind { if tt::DelimiterKind::Brace == tt.delimiter.kind {
cov_mark::hit!(parse_macro_def_rules); cov_mark::hit!(parse_macro_def_rules);
while src.len() > 0 { while src.len() > 0 {
let rule = match Rule::parse(&mut src, true, new_meta_vars) { let rule = match Rule::parse(edition, &mut src, true, new_meta_vars) {
Ok(it) => it, Ok(it) => it,
Err(e) => { Err(e) => {
err = Some(Box::new(e)); err = Some(Box::new(e));
@ -220,7 +217,7 @@ impl<S: Span> DeclarativeMacro<S> {
} }
} else { } else {
cov_mark::hit!(parse_macro_def_simple); cov_mark::hit!(parse_macro_def_simple);
match Rule::parse(&mut src, false, new_meta_vars) { match Rule::parse(edition, &mut src, false, new_meta_vars) {
Ok(rule) => { Ok(rule) => {
if src.len() != 0 { if src.len() != 0 {
err = Some(Box::new(ParseError::expected("remaining tokens in macro def"))); err = Some(Box::new(ParseError::expected("remaining tokens in macro def")));
@ -240,7 +237,7 @@ impl<S: Span> DeclarativeMacro<S> {
} }
} }
DeclarativeMacro { rules: rules.into_boxed_slice(), is_2021, err } DeclarativeMacro { rules: rules.into_boxed_slice(), err }
} }
pub fn err(&self) -> Option<&ParseError> { pub fn err(&self) -> Option<&ParseError> {
@ -249,18 +246,19 @@ impl<S: Span> DeclarativeMacro<S> {
pub fn expand( pub fn expand(
&self, &self,
tt: &tt::Subtree<S>, tt: &tt::Subtree<Span>,
marker: impl Fn(&mut S) + Copy, marker: impl Fn(&mut Span) + Copy,
new_meta_vars: bool, new_meta_vars: bool,
call_site: S, call_site: Span,
) -> ExpandResult<tt::Subtree<S>> { ) -> ExpandResult<tt::Subtree<Span>> {
expander::expand_rules(&self.rules, tt, marker, self.is_2021, new_meta_vars, call_site) expander::expand_rules(&self.rules, tt, marker, new_meta_vars, call_site)
} }
} }
impl<S: Span> Rule<S> { impl Rule {
fn parse( fn parse(
src: &mut TtIter<'_, S>, edition: impl Copy + Fn(SyntaxContextId) -> Edition,
src: &mut TtIter<'_, Span>,
expect_arrow: bool, expect_arrow: bool,
new_meta_vars: bool, new_meta_vars: bool,
) -> Result<Self, ParseError> { ) -> Result<Self, ParseError> {
@ -271,14 +269,14 @@ impl<S: Span> Rule<S> {
} }
let rhs = src.expect_subtree().map_err(|()| ParseError::expected("expected subtree"))?; let rhs = src.expect_subtree().map_err(|()| ParseError::expected("expected subtree"))?;
let lhs = MetaTemplate::parse_pattern(lhs)?; let lhs = MetaTemplate::parse_pattern(edition, lhs)?;
let rhs = MetaTemplate::parse_template(rhs, new_meta_vars)?; let rhs = MetaTemplate::parse_template(edition, rhs, new_meta_vars)?;
Ok(crate::Rule { lhs, rhs }) Ok(crate::Rule { lhs, rhs })
} }
} }
fn validate<S: Span>(pattern: &MetaTemplate<S>) -> Result<(), ParseError> { fn validate(pattern: &MetaTemplate) -> Result<(), ParseError> {
for op in pattern.iter() { for op in pattern.iter() {
match op { match op {
Op::Subtree { tokens, .. } => validate(tokens)?, Op::Subtree { tokens, .. } => validate(tokens)?,

100
crates/mbe/src/parser.rs
View file

@ -2,8 +2,8 @@
//! trees. //! trees.
use smallvec::{smallvec, SmallVec}; use smallvec::{smallvec, SmallVec};
use span::{Edition, Span, SyntaxContextId};
use syntax::SmolStr; use syntax::SmolStr;
use tt::Span;
use crate::{tt_iter::TtIter, ParseError}; use crate::{tt_iter::TtIter, ParseError};
@ -21,30 +21,39 @@ use crate::{tt_iter::TtIter, ParseError};
/// Stuff to the right is a [`MetaTemplate`] template which is used to produce /// Stuff to the right is a [`MetaTemplate`] template which is used to produce
/// output. /// output.
#[derive(Clone, Debug, PartialEq, Eq)] #[derive(Clone, Debug, PartialEq, Eq)]
pub(crate) struct MetaTemplate<S>(pub(crate) Box<[Op<S>]>); pub(crate) struct MetaTemplate(pub(crate) Box<[Op]>);
impl<S: Span> MetaTemplate<S> { impl MetaTemplate {
pub(crate) fn parse_pattern(pattern: &tt::Subtree<S>) -> Result<Self, ParseError> { pub(crate) fn parse_pattern(
MetaTemplate::parse(pattern, Mode::Pattern, false) edition: impl Copy + Fn(SyntaxContextId) -> Edition,
pattern: &tt::Subtree<Span>,
) -> Result<Self, ParseError> {
MetaTemplate::parse(edition, pattern, Mode::Pattern, false)
} }
pub(crate) fn parse_template( pub(crate) fn parse_template(
template: &tt::Subtree<S>, edition: impl Copy + Fn(SyntaxContextId) -> Edition,
template: &tt::Subtree<Span>,
new_meta_vars: bool, new_meta_vars: bool,
) -> Result<Self, ParseError> { ) -> Result<Self, ParseError> {
MetaTemplate::parse(template, Mode::Template, new_meta_vars) MetaTemplate::parse(edition, template, Mode::Template, new_meta_vars)
} }
pub(crate) fn iter(&self) -> impl Iterator<Item = &Op<S>> { pub(crate) fn iter(&self) -> impl Iterator<Item = &Op> {
self.0.iter() self.0.iter()
} }
fn parse(tt: &tt::Subtree<S>, mode: Mode, new_meta_vars: bool) -> Result<Self, ParseError> { fn parse(
edition: impl Copy + Fn(SyntaxContextId) -> Edition,
tt: &tt::Subtree<Span>,
mode: Mode,
new_meta_vars: bool,
) -> Result<Self, ParseError> {
let mut src = TtIter::new(tt); let mut src = TtIter::new(tt);
let mut res = Vec::new(); let mut res = Vec::new();
while let Some(first) = src.peek_n(0) { while let Some(first) = src.peek_n(0) {
let op = next_op(first, &mut src, mode, new_meta_vars)?; let op = next_op(edition, first, &mut src, mode, new_meta_vars)?;
res.push(op); res.push(op);
} }
@ -53,15 +62,15 @@ impl<S: Span> MetaTemplate<S> {
} }
#[derive(Clone, Debug, PartialEq, Eq)] #[derive(Clone, Debug, PartialEq, Eq)]
pub(crate) enum Op<S> { pub(crate) enum Op {
Var { Var {
name: SmolStr, name: SmolStr,
kind: Option<MetaVarKind>, kind: Option<MetaVarKind>,
id: S, id: Span,
}, },
Ignore { Ignore {
name: SmolStr, name: SmolStr,
id: S, id: Span,
}, },
Index { Index {
depth: usize, depth: usize,
@ -75,17 +84,17 @@ pub(crate) enum Op<S> {
depth: Option<usize>, depth: Option<usize>,
}, },
Repeat { Repeat {
tokens: MetaTemplate<S>, tokens: MetaTemplate,
kind: RepeatKind, kind: RepeatKind,
separator: Option<Separator<S>>, separator: Option<Separator>,
}, },
Subtree { Subtree {
tokens: MetaTemplate<S>, tokens: MetaTemplate,
delimiter: tt::Delimiter<S>, delimiter: tt::Delimiter<Span>,
}, },
Literal(tt::Literal<S>), Literal(tt::Literal<Span>),
Punct(SmallVec<[tt::Punct<S>; 3]>), Punct(SmallVec<[tt::Punct<Span>; 3]>),
Ident(tt::Ident<S>), Ident(tt::Ident<Span>),
} }
#[derive(Copy, Clone, Debug, PartialEq, Eq)] #[derive(Copy, Clone, Debug, PartialEq, Eq)]
@ -114,15 +123,15 @@ pub(crate) enum MetaVarKind {
} }
#[derive(Clone, Debug, Eq)] #[derive(Clone, Debug, Eq)]
pub(crate) enum Separator<S> { pub(crate) enum Separator {
Literal(tt::Literal<S>), Literal(tt::Literal<Span>),
Ident(tt::Ident<S>), Ident(tt::Ident<Span>),
Puncts(SmallVec<[tt::Punct<S>; 3]>), Puncts(SmallVec<[tt::Punct<Span>; 3]>),
} }
// Note that when we compare a Separator, we just care about its textual value. // Note that when we compare a Separator, we just care about its textual value.
impl<S> PartialEq for Separator<S> { impl PartialEq for Separator {
fn eq(&self, other: &Separator<S>) -> bool { fn eq(&self, other: &Separator) -> bool {
use Separator::*; use Separator::*;
match (self, other) { match (self, other) {
@ -144,12 +153,13 @@ enum Mode {
Template, Template,
} }
fn next_op<S: Span>( fn next_op(
first_peeked: &tt::TokenTree<S>, edition: impl Copy + Fn(SyntaxContextId) -> Edition,
src: &mut TtIter<'_, S>, first_peeked: &tt::TokenTree<Span>,
src: &mut TtIter<'_, Span>,
mode: Mode, mode: Mode,
new_meta_vars: bool, new_meta_vars: bool,
) -> Result<Op<S>, ParseError> { ) -> Result<Op, ParseError> {
let res = match first_peeked { let res = match first_peeked {
tt::TokenTree::Leaf(tt::Leaf::Punct(p @ tt::Punct { char: '$', .. })) => { tt::TokenTree::Leaf(tt::Leaf::Punct(p @ tt::Punct { char: '$', .. })) => {
src.next().expect("first token already peeked"); src.next().expect("first token already peeked");
@ -162,7 +172,7 @@ fn next_op<S: Span>(
tt::TokenTree::Subtree(subtree) => match subtree.delimiter.kind { tt::TokenTree::Subtree(subtree) => match subtree.delimiter.kind {
tt::DelimiterKind::Parenthesis => { tt::DelimiterKind::Parenthesis => {
let (separator, kind) = parse_repeat(src)?; let (separator, kind) = parse_repeat(src)?;
let tokens = MetaTemplate::parse(subtree, mode, new_meta_vars)?; let tokens = MetaTemplate::parse(edition, subtree, mode, new_meta_vars)?;
Op::Repeat { tokens, separator, kind } Op::Repeat { tokens, separator, kind }
} }
tt::DelimiterKind::Brace => match mode { tt::DelimiterKind::Brace => match mode {
@ -189,13 +199,13 @@ fn next_op<S: Span>(
Op::Ident(tt::Ident { text: "$crate".into(), span: ident.span }) Op::Ident(tt::Ident { text: "$crate".into(), span: ident.span })
} }
tt::Leaf::Ident(ident) => { tt::Leaf::Ident(ident) => {
let kind = eat_fragment_kind(src, mode)?; let kind = eat_fragment_kind(edition, src, mode)?;
let name = ident.text.clone(); let name = ident.text.clone();
let id = ident.span; let id = ident.span;
Op::Var { name, kind, id } Op::Var { name, kind, id }
} }
tt::Leaf::Literal(lit) if is_boolean_literal(lit) => { tt::Leaf::Literal(lit) if is_boolean_literal(lit) => {
let kind = eat_fragment_kind(src, mode)?; let kind = eat_fragment_kind(edition, src, mode)?;
let name = lit.text.clone(); let name = lit.text.clone();
let id = lit.span; let id = lit.span;
Op::Var { name, kind, id } Op::Var { name, kind, id }
@ -233,15 +243,16 @@ fn next_op<S: Span>(
tt::TokenTree::Subtree(subtree) => { tt::TokenTree::Subtree(subtree) => {
src.next().expect("first token already peeked"); src.next().expect("first token already peeked");
let tokens = MetaTemplate::parse(subtree, mode, new_meta_vars)?; let tokens = MetaTemplate::parse(edition, subtree, mode, new_meta_vars)?;
Op::Subtree { tokens, delimiter: subtree.delimiter } Op::Subtree { tokens, delimiter: subtree.delimiter }
} }
}; };
Ok(res) Ok(res)
} }
fn eat_fragment_kind<S: Span>( fn eat_fragment_kind(
src: &mut TtIter<'_, S>, edition: impl Copy + Fn(SyntaxContextId) -> Edition,
src: &mut TtIter<'_, Span>,
mode: Mode, mode: Mode,
) -> Result<Option<MetaVarKind>, ParseError> { ) -> Result<Option<MetaVarKind>, ParseError> {
if let Mode::Pattern = mode { if let Mode::Pattern = mode {
@ -252,7 +263,10 @@ fn eat_fragment_kind<S: Span>(
let kind = match ident.text.as_str() { let kind = match ident.text.as_str() {
"path" => MetaVarKind::Path, "path" => MetaVarKind::Path,
"ty" => MetaVarKind::Ty, "ty" => MetaVarKind::Ty,
"pat" => MetaVarKind::Pat, "pat" => match edition(ident.span.ctx) {
Edition::Edition2015 | Edition::Edition2018 => MetaVarKind::PatParam,
Edition::Edition2021 | Edition::Edition2024 => MetaVarKind::Pat,
},
"pat_param" => MetaVarKind::PatParam, "pat_param" => MetaVarKind::PatParam,
"stmt" => MetaVarKind::Stmt, "stmt" => MetaVarKind::Stmt,
"block" => MetaVarKind::Block, "block" => MetaVarKind::Block,
@ -271,13 +285,11 @@ fn eat_fragment_kind<S: Span>(
Ok(None) Ok(None)
} }
fn is_boolean_literal<S>(lit: &tt::Literal<S>) -> bool { fn is_boolean_literal(lit: &tt::Literal<Span>) -> bool {
matches!(lit.text.as_str(), "true" | "false") matches!(lit.text.as_str(), "true" | "false")
} }
fn parse_repeat<S: Span>( fn parse_repeat(src: &mut TtIter<'_, Span>) -> Result<(Option<Separator>, RepeatKind), ParseError> {
src: &mut TtIter<'_, S>,
) -> Result<(Option<Separator<S>>, RepeatKind), ParseError> {
let mut separator = Separator::Puncts(SmallVec::new()); let mut separator = Separator::Puncts(SmallVec::new());
for tt in src { for tt in src {
let tt = match tt { let tt = match tt {
@ -314,7 +326,7 @@ fn parse_repeat<S: Span>(
Err(ParseError::InvalidRepeat) Err(ParseError::InvalidRepeat)
} }
fn parse_metavar_expr<S: Span>(new_meta_vars: bool, src: &mut TtIter<'_, S>) -> Result<Op<S>, ()> { fn parse_metavar_expr(new_meta_vars: bool, src: &mut TtIter<'_, Span>) -> Result<Op, ()> {
let func = src.expect_ident()?; let func = src.expect_ident()?;
let args = src.expect_subtree()?; let args = src.expect_subtree()?;
@ -352,7 +364,7 @@ fn parse_metavar_expr<S: Span>(new_meta_vars: bool, src: &mut TtIter<'_, S>) ->
Ok(op) Ok(op)
} }
fn parse_depth<S: Span>(src: &mut TtIter<'_, S>) -> Result<usize, ()> { fn parse_depth(src: &mut TtIter<'_, Span>) -> Result<usize, ()> {
if src.len() == 0 { if src.len() == 0 {
Ok(0) Ok(0)
} else if let tt::Leaf::Literal(lit) = src.expect_literal()? { } else if let tt::Leaf::Literal(lit) = src.expect_literal()? {
@ -363,7 +375,7 @@ fn parse_depth<S: Span>(src: &mut TtIter<'_, S>) -> Result<usize, ()> {
} }
} }
fn try_eat_comma<S: Span>(src: &mut TtIter<'_, S>) -> bool { fn try_eat_comma(src: &mut TtIter<'_, Span>) -> bool {
if let Some(tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct { char: ',', .. }))) = src.peek_n(0) { if let Some(tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct { char: ',', .. }))) = src.peek_n(0) {
let _ = src.next(); let _ = src.next();
return true; return true;

1
crates/project-model/Cargo.toml
View file

@ -25,6 +25,7 @@ itertools.workspace = true
# local deps # local deps
base-db.workspace = true base-db.workspace = true
span.workspace = true
cfg.workspace = true cfg.workspace = true
paths = { workspace = true, features = ["serde1"] } paths = { workspace = true, features = ["serde1"] }
stdx.workspace = true stdx.workspace = true

2
crates/project-model/src/cargo_workspace.rs
View file

@ -4,13 +4,13 @@ use std::ops;
use std::str::from_utf8; use std::str::from_utf8;
use anyhow::Context; use anyhow::Context;
use base_db::Edition;
use cargo_metadata::{CargoOpt, MetadataCommand}; use cargo_metadata::{CargoOpt, MetadataCommand};
use la_arena::{Arena, Idx}; use la_arena::{Arena, Idx};
use paths::{AbsPath, AbsPathBuf, Utf8PathBuf}; use paths::{AbsPath, AbsPathBuf, Utf8PathBuf};
use rustc_hash::{FxHashMap, FxHashSet}; use rustc_hash::{FxHashMap, FxHashSet};
use serde::Deserialize; use serde::Deserialize;
use serde_json::from_value; use serde_json::from_value;
use span::Edition;
use toolchain::Tool; use toolchain::Tool;
use crate::{utf8_stdout, InvocationLocation, ManifestPath, Sysroot}; use crate::{utf8_stdout, InvocationLocation, ManifestPath, Sysroot};

3
crates/project-model/src/project_json.rs
View file

@ -49,11 +49,12 @@
//! user explores them belongs to that extension (it's totally valid to change //! user explores them belongs to that extension (it's totally valid to change
//! rust-project.json over time via configuration request!) //! rust-project.json over time via configuration request!)
use base_db::{CrateDisplayName, CrateId, CrateName, Dependency, Edition}; use base_db::{CrateDisplayName, CrateId, CrateName, Dependency};
use la_arena::RawIdx; use la_arena::RawIdx;
use paths::{AbsPath, AbsPathBuf, Utf8PathBuf}; use paths::{AbsPath, AbsPathBuf, Utf8PathBuf};
use rustc_hash::FxHashMap; use rustc_hash::FxHashMap;
use serde::{de, Deserialize}; use serde::{de, Deserialize};
use span::Edition;
use crate::cfg_flag::CfgFlag; use crate::cfg_flag::CfgFlag;

5
crates/project-model/src/workspace.rs
View file

@ -6,13 +6,14 @@ use std::{collections::VecDeque, fmt, fs, iter, str::FromStr, sync};
use anyhow::{format_err, Context}; use anyhow::{format_err, Context};
use base_db::{ use base_db::{
CrateDisplayName, CrateGraph, CrateId, CrateName, CrateOrigin, Dependency, Edition, Env, CrateDisplayName, CrateGraph, CrateId, CrateName, CrateOrigin, Dependency, Env, FileId,
FileId, LangCrateOrigin, ProcMacroPaths, TargetLayoutLoadResult, LangCrateOrigin, ProcMacroPaths, TargetLayoutLoadResult,
}; };
use cfg::{CfgAtom, CfgDiff, CfgOptions}; use cfg::{CfgAtom, CfgDiff, CfgOptions};
use paths::{AbsPath, AbsPathBuf}; use paths::{AbsPath, AbsPathBuf};
use rustc_hash::{FxHashMap, FxHashSet}; use rustc_hash::{FxHashMap, FxHashSet};
use semver::Version; use semver::Version;
use span::Edition;
use stdx::always; use stdx::always;
use toolchain::Tool; use toolchain::Tool;
use triomphe::Arc; use triomphe::Arc;

50
crates/span/src/lib.rs
View file

@ -16,6 +16,56 @@ pub use self::{
pub use syntax::{TextRange, TextSize}; pub use syntax::{TextRange, TextSize};
pub use vfs::FileId; pub use vfs::FileId;
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub enum Edition {
Edition2015,
Edition2018,
Edition2021,
Edition2024,
}
impl Edition {
pub const CURRENT: Edition = Edition::Edition2021;
pub const DEFAULT: Edition = Edition::Edition2015;
}
#[derive(Debug)]
pub struct ParseEditionError {
invalid_input: String,
}
impl std::error::Error for ParseEditionError {}
impl fmt::Display for ParseEditionError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "invalid edition: {:?}", self.invalid_input)
}
}
impl std::str::FromStr for Edition {
type Err = ParseEditionError;
fn from_str(s: &str) -> Result<Self, Self::Err> {
let res = match s {
"2015" => Edition::Edition2015,
"2018" => Edition::Edition2018,
"2021" => Edition::Edition2021,
"2024" => Edition::Edition2024,
_ => return Err(ParseEditionError { invalid_input: s.to_owned() }),
};
Ok(res)
}
}
impl fmt::Display for Edition {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(match self {
Edition::Edition2015 => "2015",
Edition::Edition2018 => "2018",
Edition::Edition2021 => "2021",
Edition::Edition2024 => "2024",
})
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub struct FilePosition { pub struct FilePosition {
pub file_id: FileId, pub file_id: FileId,

21
crates/test-fixture/src/lib.rs
View file

@ -2,8 +2,8 @@
use std::{iter, mem, ops::Not, str::FromStr, sync}; use std::{iter, mem, ops::Not, str::FromStr, sync};
use base_db::{ use base_db::{
CrateDisplayName, CrateGraph, CrateId, CrateName, CrateOrigin, Dependency, Edition, Env, CrateDisplayName, CrateGraph, CrateId, CrateName, CrateOrigin, Dependency, Env, FileChange,
FileChange, FileSet, LangCrateOrigin, SourceDatabaseExt, SourceRoot, Version, VfsPath, FileSet, LangCrateOrigin, SourceDatabaseExt, SourceRoot, Version, VfsPath,
}; };
use cfg::CfgOptions; use cfg::CfgOptions;
use hir_expand::{ use hir_expand::{
@ -14,7 +14,7 @@ use hir_expand::{
}, },
}; };
use rustc_hash::FxHashMap; use rustc_hash::FxHashMap;
use span::{FileId, FilePosition, FileRange, Span}; use span::{Edition, FileId, FilePosition, FileRange, Span};
use test_utils::{ use test_utils::{
extract_range_or_offset, Fixture, FixtureWithProjectMeta, RangeOrOffset, CURSOR_MARKER, extract_range_or_offset, Fixture, FixtureWithProjectMeta, RangeOrOffset, CURSOR_MARKER,
ESCAPED_CURSOR_MARKER, ESCAPED_CURSOR_MARKER,
@ -137,7 +137,10 @@ impl ChangeFixture {
let mut crate_deps = Vec::new(); let mut crate_deps = Vec::new();
let mut default_crate_root: Option<FileId> = None; let mut default_crate_root: Option<FileId> = None;
let mut default_cfg = CfgOptions::default(); let mut default_cfg = CfgOptions::default();
let mut default_env = Env::new_for_test_fixture(); let mut default_env = Env::from_iter([(
String::from("__ra_is_test_fixture"),
String::from("__ra_is_test_fixture"),
)]);
let mut file_set = FileSet::default(); let mut file_set = FileSet::default();
let mut current_source_root_kind = SourceRootKind::Local; let mut current_source_root_kind = SourceRootKind::Local;
@ -262,7 +265,10 @@ impl ChangeFixture {
None, None,
Default::default(), Default::default(),
Default::default(), Default::default(),
Env::new_for_test_fixture(), Env::from_iter([(
String::from("__ra_is_test_fixture"),
String::from("__ra_is_test_fixture"),
)]),
false, false,
CrateOrigin::Lang(LangCrateOrigin::Core), CrateOrigin::Lang(LangCrateOrigin::Core),
); );
@ -298,7 +304,10 @@ impl ChangeFixture {
None, None,
Default::default(), Default::default(),
Default::default(), Default::default(),
Env::new_for_test_fixture(), Env::from_iter([(
String::from("__ra_is_test_fixture"),
String::from("__ra_is_test_fixture"),
)]),
true, true,
CrateOrigin::Local { repo: None, name: None }, CrateOrigin::Local { repo: None, name: None },
); );