@@ -23,7 +23,6 @@ use rustc_data_structures::sync::{self, Lrc};
2323use rustc_macros:: { Decodable , Encodable , HashStable_Generic } ;
2424use rustc_serialize:: { Decodable , Encodable } ;
2525use rustc_span:: { sym, Span , SpanDecoder , SpanEncoder , Symbol , DUMMY_SP } ;
26- use smallvec:: { smallvec, SmallVec } ;
2726
2827use std:: borrow:: Cow ;
2928use std:: { cmp, fmt, iter} ;
@@ -180,42 +179,33 @@ impl AttrTokenStream {
180179 AttrTokenStream ( Lrc :: new ( tokens) )
181180 }
182181
183- /// Converts this `AttrTokenStream` to a plain `TokenStream `.
182+ /// Converts this `AttrTokenStream` to a plain `Vec<TokenTree> `.
184183 /// During conversion, `AttrTokenTree::Attributes` get 'flattened'
185184 /// back to a `TokenStream` of the form `outer_attr attr_target`.
186185 /// If there are inner attributes, they are inserted into the proper
187186 /// place in the attribute target tokens.
188- pub fn to_tokenstream ( & self ) -> TokenStream {
189- let trees: Vec < _ > = self
190- . 0
191- . iter ( )
192- . flat_map ( |tree| match & tree {
187+ pub fn to_token_trees ( & self ) -> Vec < TokenTree > {
188+ let mut res = Vec :: with_capacity ( self . 0 . len ( ) ) ;
189+ for tree in self . 0 . iter ( ) {
190+ match tree {
193191 AttrTokenTree :: Token ( inner, spacing) => {
194- smallvec ! [ TokenTree :: Token ( inner. clone( ) , * spacing) ] . into_iter ( )
192+ res . push ( TokenTree :: Token ( inner. clone ( ) , * spacing) ) ;
195193 }
196194 AttrTokenTree :: Delimited ( span, spacing, delim, stream) => {
197- smallvec ! [ TokenTree :: Delimited (
195+ res . push ( TokenTree :: Delimited (
198196 * span,
199197 * spacing,
200198 * delim,
201- stream. to_tokenstream( )
202- ) , ]
203- . into_iter ( )
199+ TokenStream :: new ( stream. to_token_trees ( ) ) ,
200+ ) )
204201 }
205202 AttrTokenTree :: Attributes ( data) => {
206203 let idx = data
207204 . attrs
208205 . partition_point ( |attr| matches ! ( attr. style, crate :: AttrStyle :: Outer ) ) ;
209206 let ( outer_attrs, inner_attrs) = data. attrs . split_at ( idx) ;
210207
211- let mut target_tokens: Vec < _ > = data
212- . tokens
213- . to_attr_token_stream ( )
214- . to_tokenstream ( )
215- . 0
216- . iter ( )
217- . cloned ( )
218- . collect ( ) ;
208+ let mut target_tokens = data. tokens . to_attr_token_stream ( ) . to_token_trees ( ) ;
219209 if !inner_attrs. is_empty ( ) {
220210 let mut found = false ;
221211 // Check the last two trees (to account for a trailing semi)
@@ -251,17 +241,14 @@ impl AttrTokenStream {
251241 "Failed to find trailing delimited group in: {target_tokens:?}"
252242 ) ;
253243 }
254- let mut flat: SmallVec < [ _ ; 1 ] > =
255- SmallVec :: with_capacity ( target_tokens. len ( ) + outer_attrs. len ( ) ) ;
256244 for attr in outer_attrs {
257- flat . extend ( attr. tokens ( ) . 0 . iter ( ) . cloned ( ) ) ;
245+ res . extend ( attr. tokens ( ) . 0 . iter ( ) . cloned ( ) ) ;
258246 }
259- flat. extend ( target_tokens) ;
260- flat. into_iter ( )
247+ res. extend ( target_tokens) ;
261248 }
262- } )
263- . collect ( ) ;
264- TokenStream :: new ( trees )
249+ }
250+ }
251+ res
265252 }
266253}
267254
@@ -409,8 +396,8 @@ impl PartialEq<TokenStream> for TokenStream {
409396}
410397
411398impl TokenStream {
412- pub fn new ( streams : Vec < TokenTree > ) -> TokenStream {
413- TokenStream ( Lrc :: new ( streams ) )
399+ pub fn new ( tts : Vec < TokenTree > ) -> TokenStream {
400+ TokenStream ( Lrc :: new ( tts ) )
414401 }
415402
416403 pub fn is_empty ( & self ) -> bool {
@@ -461,7 +448,7 @@ impl TokenStream {
461448 AttributesData { attrs : attrs. iter ( ) . cloned ( ) . collect ( ) , tokens : tokens. clone ( ) } ;
462449 AttrTokenStream :: new ( vec ! [ AttrTokenTree :: Attributes ( attr_data) ] )
463450 } ;
464- attr_stream. to_tokenstream ( )
451+ TokenStream :: new ( attr_stream. to_token_trees ( ) )
465452 }
466453
467454 pub fn from_nonterminal_ast ( nt : & Nonterminal ) -> TokenStream {
0 commit comments