Use TokenStreams directly instead of Vec<TokenStream>

Also buffer the output to a String (and postprocess slightly to lessen
issues caused by having a single line with hundreds thousands of bytes)
giving a 10x speedup.

Signed-off-by: Daniel Egger <daniel@eggers-club.de>

Author: therealprof

src/generate/device.rs

@@ -17,8 +17,8 @@ pub fn render(
     nightly: bool,
     generic_mod: bool,
     device_x: &mut String,
-) -> Result<Vec<TokenStream>> {
-    let mut out = vec![];
+) -> Result<TokenStream> {
+    let mut out = TokenStream::new();
 
     let doc = format!(
         "Peripheral access API for {0} microcontrollers \
@@ -30,20 +30,20 @@ pub fn render(
     );
 
     if target == Target::Msp430 {
-        out.push(quote! {
+        out.extend(quote! {
             #![feature(abi_msp430_interrupt)]
         });
     }
 
     if target != Target::None && target != Target::CortexM && target != Target::RISCV {
-        out.push(quote! {
+        out.extend(quote! {
             #![cfg_attr(feature = "rt", feature(global_asm))]
             #![cfg_attr(feature = "rt", feature(use_extern_macros))]
             #![cfg_attr(feature = "rt", feature(used))]
         });
     }
 
-    out.push(quote! {
+    out.extend(quote! {
         #![doc = #doc]
         // Deny a subset of warnings
         #![deny(const_err)]
@@ -73,14 +73,14 @@ pub fn render(
 
     match target {
         Target::CortexM => {
-            out.push(quote! {
+            out.extend(quote! {
                 extern crate cortex_m;
                 #[cfg(feature = "rt")]
                 extern crate cortex_m_rt;
             });
         }
         Target::Msp430 => {
-            out.push(quote! {
+            out.extend(quote! {
                 extern crate msp430;
                 #[cfg(feature = "rt")]
                 extern crate msp430_rt;
@@ -89,7 +89,7 @@ pub fn render(
             });
         }
         Target::RISCV => {
-            out.push(quote! {
+            out.extend(quote! {
                 extern crate riscv;
                 #[cfg(feature = "rt")]
                 extern crate riscv_rt;
@@ -98,7 +98,7 @@ pub fn render(
         Target::None => {}
     }
 
-    out.push(quote! {
+    out.extend(quote! {
         extern crate bare_metal;
         extern crate vcell;
 
@@ -112,7 +112,7 @@ pub fn render(
     if let Some(cpu) = d.cpu.as_ref() {
         let bits = util::unsuffixed(u64::from(cpu.nvic_priority_bits));
 
-        out.push(quote! {
+        out.extend(quote! {
             ///Number available in the NVIC for configuring priority
             pub const NVIC_PRIO_BITS: u8 = #bits;
         });
@@ -136,7 +136,7 @@ pub fn render(
     let mut fields = vec![];
     let mut exprs = vec![];
     if target == Target::CortexM {
-        out.push(quote! {
+        out.extend(quote! {
             pub use cortex_m::peripheral::Peripherals as CorePeripherals;
             #[cfg(feature = "rt")]
             pub use cortex_m_rt::interrupt;
@@ -145,13 +145,13 @@ pub fn render(
         });
 
         if fpu_present {
-            out.push(quote! {
+            out.extend(quote! {
                 pub use cortex_m::peripheral::{
                     CBP, CPUID, DCB, DWT, FPB, FPU, ITM, MPU, NVIC, SCB, SYST, TPIU,
                 };
             });
         } else {
-            out.push(quote! {
+            out.extend(quote! {
                 pub use cortex_m::peripheral::{
                     CBP, CPUID, DCB, DWT, FPB, ITM, MPU, NVIC, SCB, SYST, TPIU,
                 };
@@ -165,7 +165,7 @@ pub fn render(
     } else {
         let tokens = syn::parse_file(generic_file).unwrap().into_token_stream();
 
-        out.push(quote! {
+        out.extend(quote! {
             #[allow(unused_imports)]
             use generic::*;
             ///Common register and bit access and modify traits
@@ -227,7 +227,7 @@ pub fn render(
         }
     });
 
-    out.push(quote! {
+    out.extend(quote! {
         // NOTE `no_mangle` is used here to prevent linking different minor versions of the device
         // crate as that would let you `take` the device peripherals more than once (one per minor
         // version)

src/generate/interrupt.rs

@@ -14,7 +14,7 @@ pub fn render(
     target: Target,
     peripherals: &[Peripheral],
     device_x: &mut String,
-) -> Result<Vec<TokenStream>> {
+) -> Result<TokenStream> {
     let interrupts = peripherals
         .iter()
         .flat_map(|p| p.interrupt.iter())
@@ -24,7 +24,7 @@ pub fn render(
     let mut interrupts = interrupts.into_iter().map(|(_, v)| v).collect::<Vec<_>>();
     interrupts.sort_by_key(|i| i.value);
 
-    let mut root = vec![];
+    let mut root = TokenStream::new();
     let mut arms = vec![];
     let mut from_arms = vec![];
     let mut elements = vec![];
@@ -80,7 +80,7 @@ pub fn render(
                 writeln!(device_x, "PROVIDE({} = DefaultHandler);", name).unwrap();
             }
 
-            root.push(quote! {
+            root.extend(quote! {
                 #[cfg(feature = "rt")]
                 extern "C" {
                     #(fn #names();)*
@@ -169,7 +169,7 @@ pub fn render(
     };
 
     if target == Target::CortexM {
-        root.push(interrupt_enum);
+        root.extend(interrupt_enum);
     } else {
         mod_items.push(quote! {
             #interrupt_enum
@@ -283,14 +283,14 @@ pub fn render(
     }
 
     if !interrupts.is_empty() && target != Target::CortexM {
-        root.push(quote! {
+        root.extend(quote! {
             #[doc(hidden)]
             pub mod interrupt {
                 #(#mod_items)*
             }
         });
 
-        root.push(quote! {
+        root.extend(quote! {
             pub use self::interrupt::Interrupt;
         });
     }

src/generate/peripheral.rs

@@ -20,8 +20,8 @@ pub fn render(
     all_peripherals: &[Peripheral],
     defaults: &RegisterProperties,
     nightly: bool,
-) -> Result<Vec<TokenStream>> {
-    let mut out = vec![];
+) -> Result<TokenStream> {
+    let mut out = TokenStream::new();
 
     let p_derivedfrom = p_original.derived_from.as_ref().and_then(|s| {
         all_peripherals.iter().find(|x| x.name == *s)
@@ -50,7 +50,7 @@ pub fn render(
     };
 
     // Insert the peripheral structure
-    out.push(quote! {
+    out.extend(quote! {
         #[doc = #description]
         pub struct #name_pc { _marker: PhantomData<*const ()> }
 
@@ -137,19 +137,18 @@ pub fn render(
     // No `struct RegisterBlock` can be generated
     if registers.is_empty() && clusters.is_empty() {
         // Drop the definition of the peripheral
-        out.pop();
-        return Ok(out);
+        return Ok(TokenStream::new());
     }
 
     let defaults = p.default_register_properties.derive_from(defaults);
 
     // Push any register or cluster blocks into the output
-    let mut mod_items = vec![];
-    mod_items.push(register_or_cluster_block(ercs, &defaults, None, nightly)?);
+    let mut mod_items = TokenStream::new();
+    mod_items.extend(register_or_cluster_block(ercs, &defaults, None, nightly)?);
 
     // Push all cluster related information into the peripheral module
     for c in &clusters {
-        mod_items.push(cluster_block(c, &defaults, p, all_peripherals, nightly)?);
+        mod_items.extend(cluster_block(c, &defaults, p, all_peripherals, nightly)?);
     }
 
     // Push all regsiter realted information into the peripheral module
@@ -169,18 +168,18 @@ pub fn render(
     let open = Punct::new('{', Spacing::Alone);
     let close = Punct::new('}', Spacing::Alone);
 
-    out.push(quote! {
+    out.extend(quote! {
         #[doc = #description]
         pub mod #name_sc #open
     });
 
     for item in mod_items {
-        out.push(quote! {
+        out.extend(quote! {
             #item
         });
     }
 
-    out.push(quote! {
+    out.extend(quote! {
        #close
     });
 
@@ -710,7 +709,7 @@ fn cluster_block(
     all_peripherals: &[Peripheral],
     nightly: bool,
 ) -> Result<TokenStream> {
-    let mut mod_items: Vec<TokenStream> = vec![];
+    let mut mod_items = TokenStream::new();
 
     // name_sc needs to take into account array type.
     let description = util::escape_brackets(util::respace(&c.description).as_ref());
@@ -743,7 +742,7 @@ fn cluster_block(
     // Generate the sub-cluster blocks.
     let clusters = util::only_clusters(&c.children);
     for c in &clusters {
-        mod_items.push(cluster_block(c, &defaults, p, all_peripherals, nightly)?);
+        mod_items.extend(cluster_block(c, &defaults, p, all_peripherals, nightly)?);
     }
 
     Ok(quote! {
@@ -752,7 +751,7 @@ fn cluster_block(
         ///Register block
         #[doc = #description]
         pub mod #name_sc {
-            #(#mod_items)*
+            #mod_items
         }
     })
 }

src/generate/register.rs

@@ -15,7 +15,7 @@ pub fn render(
     peripheral: &Peripheral,
     all_peripherals: &[Peripheral],
     defs: &RegisterProperties,
-) -> Result<Vec<TokenStream>> {
+) -> Result<TokenStream> {
     let access = util::access_of(register);
     let name = util::name_of(register);
     let span = Span::call_site();
@@ -150,15 +150,15 @@ pub fn render(
         });
     }
 
-    let mut out = vec![];
+    let mut out = TokenStream::new();
     let methods = methods.iter().map(|s| format!("[`{0}`](crate::generic::Reg::{0})", s)).collect::<Vec<_>>();
     let mut doc = format!("{}\n\nThis register you can {}. See [API](https://docs.rs/svd2rust/#read--modify--write-api).",
                         &description, methods.join(", "));
 
     if name_sc != "cfg" {
         doc += format!("\n\nFor information about available fields see [{0}]({0}) module", &name_sc).as_str();
     }
-    out.push(quote! {
+    out.extend(quote! {
         #[doc = #doc]
         pub type #name_pc = crate::Reg<#rty, #_name_pc>;
 
@@ -169,31 +169,31 @@ pub fn render(
 
     if can_read {
         let doc = format!("`read()` method returns [{0}::R]({0}::R) reader structure", &name_sc);
-        out.push(quote! {
+        out.extend(quote! {
             #[doc = #doc]
             impl crate::Readable for #name_pc {}
         });
     }
     if can_write {
         let doc = format!("`write(|w| ..)` method takes [{0}::W]({0}::W) writer structure", &name_sc);
-        out.push(quote! {
+        out.extend(quote! {
             #[doc = #doc]
             impl crate::Writable for #name_pc {}
         });
     }
 
-    out.push(quote! {
+    out.extend(quote! {
         #[doc = #description]
         pub mod #name_sc #open
     });
 
     for item in mod_items {
-        out.push(quote! {
+        out.extend(quote! {
             #item
         });
     }
 
-    out.push(quote! {
+    out.extend(quote! {
         #close
     });
 

src/lib.rs

@@ -482,7 +482,7 @@ pub fn generate(xml: &str, target: Target, nightly: bool) -> Result<Generation>
         &mut lib_rs,
         "{}",
         quote! {
-            #(#items)*
+            #items
         }
     )
     .or(Err(SvdError::Fmt))?;

src/main.rs

@@ -15,6 +15,7 @@ mod util;
 use std::fs::File;
 use std::io::Write;
 use std::process;
+use std::fmt::Write as _;
 
 use clap::{App, Arg};
 
@@ -102,9 +103,10 @@ fn run() -> Result<()> {
     let items = generate::device::render(&device, target, nightly, generic_mod, &mut device_x)?;
     let mut file = File::create("lib.rs").expect("Couldn't create lib.rs file");
 
-    for item in items {
-        writeln!(file, "{}", item).expect("Could not write item to lib.rs");
-    }
+    let mut data = String::new();
+    write!(data, "{}", items).expect("Could not output code");
+    let data = data.replace("] ", "]\n");
+    file.write_all(data.as_ref()).expect("Could not write code to lib.rs");
 
     if target == Target::CortexM {
         writeln!(File::create("device.x").unwrap(), "{}", device_x).unwrap();