Skip to content

Commit e18e86a

Browse files
VeykrilVeykril
committed
temp
1 parent 31f8dc7 commit e18e86a

File tree

6 files changed

+819
-669
lines changed

6 files changed

+819
-669
lines changed

‎crates/cfg/src/cfg_expr.rs

Lines changed: 25 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -45,8 +45,8 @@ impl From<CfgAtom> for CfgExpr {
4545

4646
impl CfgExpr {
4747
#[cfg(feature = "tt")]
48-
pub fn parse<S>(tt: &tt::Subtree<S>) -> CfgExpr {
49-
next_cfg_expr(&mut tt.token_trees.iter()).unwrap_or(CfgExpr::Invalid)
48+
pub fn parse<S>(tt: &tt::TokenStream<S>) -> CfgExpr {
49+
next_cfg_expr(tt.trees()).unwrap_or(CfgExpr::Invalid)
5050
}
5151

5252
/// Fold the cfg by querying all basic `Atom` and `KeyValue` predicates.
@@ -66,48 +66,52 @@ impl CfgExpr {
6666
}
6767

6868
#[cfg(feature = "tt")]
69-
fn next_cfg_expr<S>(it: &mut std::slice::Iter<'_, tt::TokenTree<S>>) -> Option<CfgExpr> {
69+
fn next_cfg_expr<S>(mut it: tt::RefTokenTreeCursor<'_, S>) -> Option<CfgExpr> {
7070
use intern::sym;
7171

7272
let name = match it.next() {
7373
None => return None,
74-
Some(tt::TokenTree::Leaf(tt::Leaf::Ident(ident))) => ident.sym.clone(),
74+
Some(tt::TokenTree::Token(tt::Token { kind: tt::TokenKind::Ident(ident, _), .. }, _)) => {
75+
ident.clone()
76+
}
7577
Some(_) => return Some(CfgExpr::Invalid),
7678
};
7779

7880
// Peek
79-
let ret = match it.as_slice().first() {
80-
Some(tt::TokenTree::Leaf(tt::Leaf::Punct(punct))) if punct.char == '=' => {
81-
match it.as_slice().get(1) {
82-
Some(tt::TokenTree::Leaf(tt::Leaf::Literal(literal))) => {
83-
it.next();
84-
it.next();
85-
CfgAtom::KeyValue { key: name, value: literal.symbol.clone() }.into()
81+
let ret = match it.look_ahead(0) {
82+
Some(tt::TokenTree::Token(tt::Token { kind: tt::TokenKind::Eq, .. }, _)) => {
83+
match it.look_ahead(1) {
84+
Some(tt::TokenTree::Token(
85+
tt::Token { kind: tt::TokenKind::Literal(lit), .. },
86+
_,
87+
)) => {
88+
let res = CfgAtom::KeyValue { key: name, value: lit.symbol.clone() }.into();
89+
_ = it.next();
90+
_ = it.next();
91+
res
8692
}
8793
_ => return Some(CfgExpr::Invalid),
8894
}
8995
}
90-
Some(tt::TokenTree::Subtree(subtree)) => {
91-
it.next();
92-
let mut sub_it = subtree.token_trees.iter();
93-
let mut subs = std::iter::from_fn(|| next_cfg_expr(&mut sub_it));
94-
match name {
96+
Some(tt::TokenTree::Delimited(_, _, _, stream)) => {
97+
let mut subs = std::iter::from_fn(|| next_cfg_expr(stream.trees()));
98+
let res = match name {
9599
s if s == sym::all => CfgExpr::All(subs.collect()),
96100
s if s == sym::any => CfgExpr::Any(subs.collect()),
97101
s if s == sym::not => {
98102
CfgExpr::Not(Box::new(subs.next().unwrap_or(CfgExpr::Invalid)))
99103
}
100104
_ => CfgExpr::Invalid,
101-
}
105+
};
106+
it.next();
107+
res
102108
}
103109
_ => CfgAtom::Flag(name).into(),
104110
};
105111

106112
// Eat comma separator
107-
if let Some(tt::TokenTree::Leaf(tt::Leaf::Punct(punct))) = it.as_slice().first() {
108-
if punct.char == ',' {
109-
it.next();
110-
}
113+
if let Some(tt::TokenTree::Token(tt::Token { kind: tt::TokenKind::Comma, .. }, _)) = it.next() {
114+
it.next();
111115
}
112116
Some(ret)
113117
}

‎crates/mbe/src/lib.rs

Lines changed: 29 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -16,15 +16,14 @@ mod tests;
1616

1717
use span::{Edition, Span, SyntaxContextId};
1818
use syntax_bridge::to_parser_input;
19-
use tt::iter::TtIter;
20-
use tt::DelimSpan;
19+
use tt::{DelimSpan, RefTokenCursor, Token, TokenCursor, TokenKind, TokenStream};
2120

2221
use std::fmt;
2322
use std::sync::Arc;
2423

2524
use crate::parser::{MetaTemplate, MetaVarKind, Op};
2625

27-
pub use tt::{Delimiter, DelimiterKind, Punct};
26+
pub use tt::Delimiter;
2827

2928
#[derive(Debug, PartialEq, Eq, Clone)]
3029
pub enum ParseError {
@@ -141,30 +140,32 @@ impl DeclarativeMacro {
141140

142141
/// The old, `macro_rules! m {}` flavor.
143142
pub fn parse_macro_rules(
144-
tt: &tt::Subtree<Span>,
143+
stream: &TokenStream<Span>,
145144
ctx_edition: impl Copy + Fn(SyntaxContextId) -> Edition,
146145
) -> DeclarativeMacro {
147146
// Note: this parsing can be implemented using mbe machinery itself, by
148147
// matching against `$($lhs:tt => $rhs:tt);*` pattern, but implementing
149148
// manually seems easier.
150-
let mut src = TtIter::new(tt);
149+
let mut cursor = RefTokenCursor::new(stream);
151150
let mut rules = Vec::new();
152151
let mut err = None;
153152

154-
while src.len() > 0 {
155-
let rule = match Rule::parse(ctx_edition, &mut src) {
153+
while let Some((token, _)) = cursor.next() {
154+
let rule = match Rule::parse(ctx_edition, &mut cursor) {
156155
Ok(it) => it,
157156
Err(e) => {
158157
err = Some(Box::new(e));
159158
break;
160159
}
161160
};
162161
rules.push(rule);
163-
if let Err(()) = src.expect_char(';') {
164-
if src.len() > 0 {
162+
match cursor.next() {
163+
Some((Token { kind: TokenKind::Semi, .. }, _)) => (),
164+
Some((Token { span, .. }, _)) => {
165165
err = Some(Box::new(ParseError::expected("expected `;`")));
166+
break;
166167
}
167-
break;
168+
None => break,
168169
}
169170
}
170171

@@ -180,8 +181,8 @@ impl DeclarativeMacro {
180181

181182
/// The new, unstable `macro m {}` flavor.
182183
pub fn parse_macro2(
183-
args: Option<&tt::Subtree<Span>>,
184-
body: &tt::Subtree<Span>,
184+
args: Option<&tt::TokenStream<Span>>,
185+
body: &tt::TokenStream<Span>,
185186
ctx_edition: impl Copy + Fn(SyntaxContextId) -> Edition,
186187
) -> DeclarativeMacro {
187188
let mut rules = Vec::new();
@@ -203,23 +204,25 @@ impl DeclarativeMacro {
203204
}
204205
} else {
205206
cov_mark::hit!(parse_macro_def_rules);
206-
let mut src = TtIter::new(body);
207-
while src.len() > 0 {
208-
let rule = match Rule::parse(ctx_edition, &mut src) {
207+
let mut cursor = RefTokenCursor::new(body);
208+
while let Some((token, _)) = cursor.next() {
209+
let rule = match Rule::parse(ctx_edition, &mut cursor) {
209210
Ok(it) => it,
210211
Err(e) => {
211212
err = Some(Box::new(e));
212213
break;
213214
}
214215
};
215216
rules.push(rule);
216-
if let Err(()) = src.expect_any_char(&[';', ',']) {
217-
if src.len() > 0 {
217+
match cursor.next() {
218+
Some((Token { kind: TokenKind::Semi | TokenKind::Comma, .. }, _)) => (),
219+
Some((Token { span, .. }, _)) => {
218220
err = Some(Box::new(ParseError::expected(
219221
"expected `;` or `,` to delimit rules",
220222
)));
223+
break;
221224
}
222-
break;
225+
None => break,
223226
}
224227
}
225228
}
@@ -244,19 +247,19 @@ impl DeclarativeMacro {
244247

245248
pub fn expand(
246249
&self,
247-
tt: &tt::Subtree<Span>,
250+
tt: &tt::TokenStream<Span>,
248251
marker: impl Fn(&mut Span) + Copy,
249252
call_site: Span,
250253
def_site_edition: Edition,
251-
) -> ExpandResult<(tt::Subtree<Span>, MatchedArmIndex)> {
254+
) -> ExpandResult<(tt::TokenStream<Span>, MatchedArmIndex)> {
252255
expander::expand_rules(&self.rules, tt, marker, call_site, def_site_edition)
253256
}
254257
}
255258

256259
impl Rule {
257260
fn parse(
258261
edition: impl Copy + Fn(SyntaxContextId) -> Edition,
259-
src: &mut TtIter<'_, Span>,
262+
src: &mut RefTokenCursor<'_, Span>,
260263
) -> Result<Self, ParseError> {
261264
let lhs = src.expect_subtree().map_err(|()| ParseError::expected("expected subtree"))?;
262265
src.expect_char('=').map_err(|()| ParseError::expected("expected `=`"))?;
@@ -353,31 +356,26 @@ impl<T: Default, E> From<Result<T, E>> for ValueResult<T, E> {
353356
}
354357

355358
pub fn expect_fragment(
356-
tt_iter: &mut TtIter<'_, Span>,
359+
cursor: &mut RefTokenCursor<'_, Span>,
357360
entry_point: ::parser::PrefixEntryPoint,
358361
edition: ::parser::Edition,
359362
delim_span: DelimSpan<Span>,
360363
) -> ExpandResult<Option<tt::TokenTree<Span>>> {
361364
use ::parser;
362-
let buffer = tt::buffer::TokenBuffer::from_tokens(tt_iter.as_slice());
363-
let parser_input = to_parser_input(edition, &buffer);
365+
let parser_input = to_parser_input(edition, cursor.clone());
364366
let tree_traversal = entry_point.parse(&parser_input, edition);
365-
let mut cursor = buffer.begin();
366367
let mut error = false;
367368
for step in tree_traversal.iter() {
368369
match step {
369370
parser::Step::Token { kind, mut n_input_tokens } => {
370-
if kind == ::parser::SyntaxKind::LIFETIME_IDENT {
371-
n_input_tokens = 2;
372-
}
373371
for _ in 0..n_input_tokens {
374-
cursor = cursor.bump_subtree();
372+
cursor.next();
375373
}
376374
}
377375
parser::Step::FloatSplit { .. } => {
378376
// FIXME: We need to split the tree properly here, but mutating the token trees
379377
// in the buffer is somewhat tricky to pull off.
380-
cursor = cursor.bump_subtree();
378+
cursor.next();
381379
}
382380
parser::Step::Enter { .. } | parser::Step::Exit => (),
383381
parser::Step::Error { .. } => error = true,
@@ -402,7 +400,7 @@ pub fn expect_fragment(
402400
curr = curr.bump();
403401
}
404402

405-
*tt_iter = TtIter::new_iter(tt_iter.as_slice()[res.len()..].iter());
403+
*cursor = TtIter::new_iter(cursor.as_slice()[res.len()..].iter());
406404
let res = match &*res {
407405
[] | [_] => res.pop(),
408406
[first, ..] => Some(tt::TokenTree::Subtree(tt::Subtree {

0 commit comments

Comments
 (0)