Skip to content

Commit 016e443

Browse files
VeykrilVeykril
committed
temp
1 parent 6ca0444 commit 016e443

File tree

6 files changed

+820
-673
lines changed

6 files changed

+820
-673
lines changed

‎crates/cfg/src/cfg_expr.rs

Lines changed: 25 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -45,8 +45,8 @@ impl From<CfgAtom> for CfgExpr {
4545

4646
impl CfgExpr {
4747
#[cfg(feature = "tt")]
48-
pub fn parse<S>(tt: &tt::Subtree<S>) -> CfgExpr {
49-
next_cfg_expr(&mut tt.token_trees.iter()).unwrap_or(CfgExpr::Invalid)
48+
pub fn parse<S>(tt: &tt::TokenStream<S>) -> CfgExpr {
49+
next_cfg_expr(tt.trees()).unwrap_or(CfgExpr::Invalid)
5050
}
5151

5252
/// Fold the cfg by querying all basic `Atom` and `KeyValue` predicates.
@@ -66,48 +66,52 @@ impl CfgExpr {
6666
}
6767

6868
#[cfg(feature = "tt")]
69-
fn next_cfg_expr<S>(it: &mut std::slice::Iter<'_, tt::TokenTree<S>>) -> Option<CfgExpr> {
69+
fn next_cfg_expr<S>(mut it: tt::RefTokenTreeCursor<'_, S>) -> Option<CfgExpr> {
7070
use intern::sym;
7171

7272
let name = match it.next() {
7373
None => return None,
74-
Some(tt::TokenTree::Leaf(tt::Leaf::Ident(ident))) => ident.sym.clone(),
74+
Some(tt::TokenTree::Token(tt::Token { kind: tt::TokenKind::Ident(ident, _), .. }, _)) => {
75+
ident.clone()
76+
}
7577
Some(_) => return Some(CfgExpr::Invalid),
7678
};
7779

7880
// Peek
79-
let ret = match it.as_slice().first() {
80-
Some(tt::TokenTree::Leaf(tt::Leaf::Punct(punct))) if punct.char == '=' => {
81-
match it.as_slice().get(1) {
82-
Some(tt::TokenTree::Leaf(tt::Leaf::Literal(literal))) => {
83-
it.next();
84-
it.next();
85-
CfgAtom::KeyValue { key: name, value: literal.symbol.clone() }.into()
81+
let ret = match it.look_ahead(0) {
82+
Some(tt::TokenTree::Token(tt::Token { kind: tt::TokenKind::Eq, .. }, _)) => {
83+
match it.look_ahead(1) {
84+
Some(tt::TokenTree::Token(
85+
tt::Token { kind: tt::TokenKind::Literal(lit), .. },
86+
_,
87+
)) => {
88+
let res = CfgAtom::KeyValue { key: name, value: lit.symbol.clone() }.into();
89+
_ = it.next();
90+
_ = it.next();
91+
res
8692
}
8793
_ => return Some(CfgExpr::Invalid),
8894
}
8995
}
90-
Some(tt::TokenTree::Subtree(subtree)) => {
91-
it.next();
92-
let mut sub_it = subtree.token_trees.iter();
93-
let mut subs = std::iter::from_fn(|| next_cfg_expr(&mut sub_it));
94-
match name {
96+
Some(tt::TokenTree::Delimited(_, _, _, stream)) => {
97+
let mut subs = std::iter::from_fn(|| next_cfg_expr(stream.trees()));
98+
let res = match name {
9599
s if s == sym::all => CfgExpr::All(subs.collect()),
96100
s if s == sym::any => CfgExpr::Any(subs.collect()),
97101
s if s == sym::not => {
98102
CfgExpr::Not(Box::new(subs.next().unwrap_or(CfgExpr::Invalid)))
99103
}
100104
_ => CfgExpr::Invalid,
101-
}
105+
};
106+
it.next();
107+
res
102108
}
103109
_ => CfgAtom::Flag(name).into(),
104110
};
105111

106112
// Eat comma separator
107-
if let Some(tt::TokenTree::Leaf(tt::Leaf::Punct(punct))) = it.as_slice().first() {
108-
if punct.char == ',' {
109-
it.next();
110-
}
113+
if let Some(tt::TokenTree::Token(tt::Token { kind: tt::TokenKind::Comma, .. }, _)) = it.next() {
114+
it.next();
111115
}
112116
Some(ret)
113117
}

‎crates/mbe/src/lib.rs

Lines changed: 29 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -23,15 +23,14 @@ mod tests;
2323

2424
use span::{Edition, Span, SyntaxContextId};
2525
use syntax_bridge::to_parser_input;
26-
use tt::iter::TtIter;
27-
use tt::DelimSpan;
26+
use tt::{DelimSpan, RefTokenCursor, Token, TokenCursor, TokenKind, TokenStream};
2827

2928
use std::fmt;
3029
use std::sync::Arc;
3130

3231
use crate::parser::{MetaTemplate, MetaVarKind, Op};
3332

34-
pub use tt::{Delimiter, DelimiterKind, Punct};
33+
pub use tt::Delimiter;
3534

3635
#[derive(Debug, PartialEq, Eq, Clone)]
3736
pub enum ParseError {
@@ -148,30 +147,32 @@ impl DeclarativeMacro {
148147

149148
/// The old, `macro_rules! m {}` flavor.
150149
pub fn parse_macro_rules(
151-
tt: &tt::Subtree<Span>,
150+
stream: &TokenStream<Span>,
152151
ctx_edition: impl Copy + Fn(SyntaxContextId) -> Edition,
153152
) -> DeclarativeMacro {
154153
// Note: this parsing can be implemented using mbe machinery itself, by
155154
// matching against `$($lhs:tt => $rhs:tt);*` pattern, but implementing
156155
// manually seems easier.
157-
let mut src = TtIter::new(tt);
156+
let mut cursor = RefTokenCursor::new(stream);
158157
let mut rules = Vec::new();
159158
let mut err = None;
160159

161-
while src.len() > 0 {
162-
let rule = match Rule::parse(ctx_edition, &mut src) {
160+
while let Some((token, _)) = cursor.next() {
161+
let rule = match Rule::parse(ctx_edition, &mut cursor) {
163162
Ok(it) => it,
164163
Err(e) => {
165164
err = Some(Box::new(e));
166165
break;
167166
}
168167
};
169168
rules.push(rule);
170-
if let Err(()) = src.expect_char(';') {
171-
if src.len() > 0 {
169+
match cursor.next() {
170+
Some((Token { kind: TokenKind::Semi, .. }, _)) => (),
171+
Some((Token { span, .. }, _)) => {
172172
err = Some(Box::new(ParseError::expected("expected `;`")));
173+
break;
173174
}
174-
break;
175+
None => break,
175176
}
176177
}
177178

@@ -187,8 +188,8 @@ impl DeclarativeMacro {
187188

188189
/// The new, unstable `macro m {}` flavor.
189190
pub fn parse_macro2(
190-
args: Option<&tt::Subtree<Span>>,
191-
body: &tt::Subtree<Span>,
191+
args: Option<&tt::TokenStream<Span>>,
192+
body: &tt::TokenStream<Span>,
192193
ctx_edition: impl Copy + Fn(SyntaxContextId) -> Edition,
193194
) -> DeclarativeMacro {
194195
let mut rules = Vec::new();
@@ -210,23 +211,25 @@ impl DeclarativeMacro {
210211
}
211212
} else {
212213
cov_mark::hit!(parse_macro_def_rules);
213-
let mut src = TtIter::new(body);
214-
while src.len() > 0 {
215-
let rule = match Rule::parse(ctx_edition, &mut src) {
214+
let mut cursor = RefTokenCursor::new(body);
215+
while let Some((token, _)) = cursor.next() {
216+
let rule = match Rule::parse(ctx_edition, &mut cursor) {
216217
Ok(it) => it,
217218
Err(e) => {
218219
err = Some(Box::new(e));
219220
break;
220221
}
221222
};
222223
rules.push(rule);
223-
if let Err(()) = src.expect_any_char(&[';', ',']) {
224-
if src.len() > 0 {
224+
match cursor.next() {
225+
Some((Token { kind: TokenKind::Semi | TokenKind::Comma, .. }, _)) => (),
226+
Some((Token { span, .. }, _)) => {
225227
err = Some(Box::new(ParseError::expected(
226228
"expected `;` or `,` to delimit rules",
227229
)));
230+
break;
228231
}
229-
break;
232+
None => break,
230233
}
231234
}
232235
}
@@ -251,19 +254,19 @@ impl DeclarativeMacro {
251254

252255
pub fn expand(
253256
&self,
254-
tt: &tt::Subtree<Span>,
257+
tt: &tt::TokenStream<Span>,
255258
marker: impl Fn(&mut Span) + Copy,
256259
call_site: Span,
257260
def_site_edition: Edition,
258-
) -> ExpandResult<(tt::Subtree<Span>, MatchedArmIndex)> {
261+
) -> ExpandResult<(tt::TokenStream<Span>, MatchedArmIndex)> {
259262
expander::expand_rules(&self.rules, tt, marker, call_site, def_site_edition)
260263
}
261264
}
262265

263266
impl Rule {
264267
fn parse(
265268
edition: impl Copy + Fn(SyntaxContextId) -> Edition,
266-
src: &mut TtIter<'_, Span>,
269+
src: &mut RefTokenCursor<'_, Span>,
267270
) -> Result<Self, ParseError> {
268271
let lhs = src.expect_subtree().map_err(|()| ParseError::expected("expected subtree"))?;
269272
src.expect_char('=').map_err(|()| ParseError::expected("expected `=`"))?;
@@ -360,31 +363,26 @@ impl<T: Default, E> From<Result<T, E>> for ValueResult<T, E> {
360363
}
361364

362365
pub fn expect_fragment(
363-
tt_iter: &mut TtIter<'_, Span>,
366+
cursor: &mut RefTokenCursor<'_, Span>,
364367
entry_point: ::parser::PrefixEntryPoint,
365368
edition: ::parser::Edition,
366369
delim_span: DelimSpan<Span>,
367370
) -> ExpandResult<Option<tt::TokenTree<Span>>> {
368371
use ::parser;
369-
let buffer = tt::buffer::TokenBuffer::from_tokens(tt_iter.as_slice());
370-
let parser_input = to_parser_input(edition, &buffer);
372+
let parser_input = to_parser_input(edition, cursor.clone());
371373
let tree_traversal = entry_point.parse(&parser_input, edition);
372-
let mut cursor = buffer.begin();
373374
let mut error = false;
374375
for step in tree_traversal.iter() {
375376
match step {
376377
parser::Step::Token { kind, mut n_input_tokens } => {
377-
if kind == ::parser::SyntaxKind::LIFETIME_IDENT {
378-
n_input_tokens = 2;
379-
}
380378
for _ in 0..n_input_tokens {
381-
cursor = cursor.bump_subtree();
379+
cursor.next();
382380
}
383381
}
384382
parser::Step::FloatSplit { .. } => {
385383
// FIXME: We need to split the tree properly here, but mutating the token trees
386384
// in the buffer is somewhat tricky to pull off.
387-
cursor = cursor.bump_subtree();
385+
cursor.next();
388386
}
389387
parser::Step::Enter { .. } | parser::Step::Exit => (),
390388
parser::Step::Error { .. } => error = true,
@@ -409,7 +407,7 @@ pub fn expect_fragment(
409407
curr = curr.bump();
410408
}
411409

412-
*tt_iter = TtIter::new_iter(tt_iter.as_slice()[res.len()..].iter());
410+
*cursor = TtIter::new_iter(cursor.as_slice()[res.len()..].iter());
413411
let res = match &*res {
414412
[] | [_] => res.pop(),
415413
[first, ..] => Some(tt::TokenTree::Subtree(tt::Subtree {

0 commit comments

Comments
 (0)