Keep multiple files per work-product

In the older version, a `.o` and ` .bc` file were separate
work-products.  This newer version keeps, for each codegen-unit, a set
of files of different kinds. We assume that if any kinds are available
then all the kinds we need are available, since the precise set of
switches will depend on attributes and command-line switches.

Should probably test this: the effect of changing attributes in
particular might not be successfully tracked?

Author: nikomatsakis

src/librustc/dep_graph/dep_node.rs

@@ -246,7 +246,5 @@ impl<D: Clone + Debug> DepNode<D> {
 /// the need to be mapped or unmapped. (This ensures we can serialize
 /// them even in the absence of a tcx.)
 #[derive(Clone, Debug, PartialEq, Eq, Hash, RustcEncodable, RustcDecodable)]
-pub enum WorkProductId {
-    PartitionObjectFile(String), // see (*TransPartition) below
-}
+pub struct WorkProductId(pub String);
 

src/librustc/dep_graph/graph.rs

@@ -10,6 +10,7 @@
 
 use hir::def_id::DefId;
 use rustc_data_structures::fnv::FnvHashMap;
+use session::config::OutputType;
 use std::cell::{Ref, RefCell};
 use std::rc::Rc;
 use std::sync::Arc;
@@ -157,11 +158,11 @@ impl DepGraph {
 /// previous hash. If it matches up, we can reuse the object file.
 #[derive(Clone, Debug, RustcEncodable, RustcDecodable)]
 pub struct WorkProduct {
-    /// extra hash used to decide if work-product is still suitable;
+    /// Extra hash used to decide if work-product is still suitable;
     /// note that this is *not* a hash of the work-product itself.
     /// See documentation on `WorkProduct` type for an example.
     pub input_hash: u64,
 
-    /// filename storing this work-product (found in the incr. comp. directory)
-    pub file_name: String,
+    /// Saved files associated with this CGU
+    pub saved_files: Vec<(OutputType, String)>,
 }

src/librustc/session/config.rs

@@ -61,7 +61,7 @@ pub enum DebugInfoLevel {
     FullDebugInfo,
 }
 
-#[derive(Clone, Copy, PartialEq, Eq, Hash)]
+#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, RustcEncodable, RustcDecodable)]
 pub enum OutputType {
     Bitcode,
     Assembly,
@@ -105,6 +105,17 @@ impl OutputType {
             OutputType::DepInfo => "dep-info",
         }
     }
+
+    pub fn extension(&self) -> &'static str {
+        match *self {
+            OutputType::Bitcode => "bc",
+            OutputType::Assembly => "s",
+            OutputType::LlvmAssembly => "ll",
+            OutputType::Object => "o",
+            OutputType::DepInfo => "d",
+            OutputType::Exe => "",
+        }
+    }
 }
 
 #[derive(Clone)]
@@ -215,15 +226,7 @@ impl OutputFilenames {
                      flavor: OutputType,
                      codegen_unit_name: Option<&str>)
                      -> PathBuf {
-        let extension = match flavor {
-            OutputType::Bitcode => "bc",
-            OutputType::Assembly => "s",
-            OutputType::LlvmAssembly => "ll",
-            OutputType::Object => "o",
-            OutputType::DepInfo => "d",
-            OutputType::Exe => "",
-        };
-
+        let extension = flavor.extension();
         self.temp_path_ext(extension, codegen_unit_name)
     }
 

src/librustc_incremental/persist/load.rs

@@ -260,11 +260,20 @@ fn reconcile_work_products<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>,
             debug!("reconcile_work_products: dep-node for {:?} is dirty", swp);
             delete_dirty_work_product(tcx, swp);
         } else {
-            let path = in_incr_comp_dir(tcx.sess, &swp.work_product.file_name).unwrap();
-            if path.exists() {
+            let all_files_exist =
+                swp.work_product
+                   .saved_files
+                   .iter()
+                   .all(|&(_, ref file_name)| {
+                       let path = in_incr_comp_dir(tcx.sess, &file_name).unwrap();
+                       path.exists()
+                   });
+            if all_files_exist {
+                debug!("reconcile_work_products: all files for {:?} exist", swp);
                 tcx.dep_graph.insert_previous_work_product(&swp.id, swp.work_product);
             } else {
-                debug!("reconcile_work_products: file for {:?} does not exist", swp);
+                debug!("reconcile_work_products: some file for {:?} does not exist", swp);
+                delete_dirty_work_product(tcx, swp);
             }
         }
     }
@@ -273,13 +282,15 @@ fn reconcile_work_products<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>,
 fn delete_dirty_work_product(tcx: TyCtxt,
                              swp: SerializedWorkProduct) {
     debug!("delete_dirty_work_product({:?})", swp);
-    let path = in_incr_comp_dir(tcx.sess, &swp.work_product.file_name).unwrap();
-    match fs::remove_file(&path) {
-        Ok(()) => { }
-        Err(err) => {
-            tcx.sess.warn(
-                &format!("file-system error deleting outdated file `{}`: {}",
-                         path.display(), err));
+    for &(_, ref file_name) in &swp.work_product.saved_files {
+        let path = in_incr_comp_dir(tcx.sess, file_name).unwrap();
+        match fs::remove_file(&path) {
+            Ok(()) => { }
+            Err(err) => {
+                tcx.sess.warn(
+                    &format!("file-system error deleting outdated file `{}`: {}",
+                             path.display(), err));
+            }
         }
     }
 }

src/librustc_incremental/persist/work_product.rs

@@ -13,47 +13,51 @@
 use persist::util::*;
 use rustc::dep_graph::{WorkProduct, WorkProductId};
 use rustc::session::Session;
+use rustc::session::config::OutputType;
 use rustc::util::fs::link_or_copy;
-use std::fs;
-use std::path::Path;
+use std::path::PathBuf;
 use std::sync::Arc;
 
 pub fn save_trans_partition(sess: &Session,
-                            partition_name: &str,
+                            cgu_name: &str,
                             partition_hash: u64,
-                            path_to_obj_file: &Path) {
-    debug!("save_trans_partition({:?},{},{})",
-           partition_name,
+                            files: &[(OutputType, PathBuf)]) {
+    debug!("save_trans_partition({:?},{},{:?})",
+           cgu_name,
            partition_hash,
-           path_to_obj_file.display());
+           files);
     if sess.opts.incremental.is_none() {
         return;
     }
-    let id = Arc::new(WorkProductId::PartitionObjectFile(partition_name.to_string()));
-    let file_name = format!("cgu-{}", partition_name);
-    let path_in_incr_dir = in_incr_comp_dir(sess, &file_name).unwrap();
+    let work_product_id = Arc::new(WorkProductId(cgu_name.to_string()));
 
-    // try to delete the file if it already exists
-    //
-    // FIXME(#34955) we can be smarter here -- if we are re-using, no need to do anything
-    if path_in_incr_dir.exists() {
-        let _ = fs::remove_file(&path_in_incr_dir);
-    }
+    let saved_files: Option<Vec<_>> =
+        files.iter()
+             .map(|&(kind, ref path)| {
+                 let file_name = format!("cgu-{}.{}", cgu_name, kind.extension());
+                 let path_in_incr_dir = in_incr_comp_dir(sess, &file_name).unwrap();
+                 match link_or_copy(path, &path_in_incr_dir) {
+                     Ok(_) => Some((kind, file_name)),
+                     Err(err) => {
+                         sess.warn(&format!("error copying object file `{}` \
+                                             to incremental directory as `{}`: {}",
+                                            path.display(),
+                                            path_in_incr_dir.display(),
+                                            err));
+                         None
+                     }
+                 }
+             })
+             .collect();
+    let saved_files = match saved_files {
+        Some(v) => v,
+        None => return,
+    };
 
-    match link_or_copy(path_to_obj_file, &path_in_incr_dir) {
-        Ok(_) => {
-            let work_product = WorkProduct {
-                input_hash: partition_hash,
-                file_name: file_name,
-            };
-            sess.dep_graph.insert_work_product(&id, work_product);
-        }
-        Err(err) => {
-            sess.warn(&format!("error copying object file `{}` \
-                                to incremental directory as `{}`: {}",
-                               path_to_obj_file.display(),
-                               path_in_incr_dir.display(),
-                               err));
-        }
-    }
+    let work_product = WorkProduct {
+        input_hash: partition_hash,
+        saved_files: saved_files,
+    };
+
+    sess.dep_graph.insert_work_product(&work_product_id, work_product);
 }

src/librustc_trans/back/write.rs

@@ -337,6 +337,8 @@ struct CodegenContext<'a> {
     remark: Passes,
     // Worker thread number
     worker: usize,
+    // Directory where incremental data is stored (if any)
+    incremental: Option<PathBuf>,
 }
 
 impl<'a> CodegenContext<'a> {
@@ -347,6 +349,7 @@ impl<'a> CodegenContext<'a> {
             plugin_passes: sess.plugin_llvm_passes.borrow().clone(),
             remark: sess.opts.cg.remark.clone(),
             worker: 0,
+            incremental: sess.opts.incremental.clone(),
         }
     }
 }
@@ -612,7 +615,7 @@ unsafe fn optimize_and_codegen(cgcx: &CodegenContext,
 
     if copy_bc_to_obj {
         debug!("copying bitcode {:?} to obj {:?}", bc_out, obj_out);
-        if let Err(e) = fs::copy(&bc_out, &obj_out) {
+        if let Err(e) = link_or_copy(&bc_out, &obj_out) {
             cgcx.handler.err(&format!("failed to copy bitcode to object file: {}", e));
         }
     }
@@ -754,9 +757,19 @@ pub fn run_passes(sess: &Session,
 
     // If in incr. comp. mode, preserve the `.o` files for potential re-use
     for mtrans in trans.modules.iter() {
-        let path_to_obj = crate_output.temp_path(OutputType::Object, Some(&mtrans.name));
-        debug!("wrote module {:?} to {:?}", mtrans.name, path_to_obj);
-        save_trans_partition(sess, &mtrans.name, mtrans.symbol_name_hash, &path_to_obj);
+        let mut files = vec![];
+
+        if modules_config.emit_obj {
+            let path = crate_output.temp_path(OutputType::Object, Some(&mtrans.name));
+            files.push((OutputType::Object, path));
+        }
+
+        if modules_config.emit_bc {
+            let path = crate_output.temp_path(OutputType::Bitcode, Some(&mtrans.name));
+            files.push((OutputType::Bitcode, path));
+        }
+
+        save_trans_partition(sess, &mtrans.name, mtrans.symbol_name_hash, &files);
     }
 
     // All codegen is finished.
@@ -941,20 +954,24 @@ fn execute_work_item(cgcx: &CodegenContext,
                                      work_item.config,
                                      work_item.output_names);
             }
-            ModuleSource::Preexisting(ref buf) => {
-                let obj_out = work_item.output_names.temp_path(OutputType::Object,
-                                                               Some(&work_item.mtrans.name));
-                debug!("copying pre-existing module `{}` from {} to {}",
-                       work_item.mtrans.name,
-                       buf.display(),
-                       obj_out.display());
-                match link_or_copy(buf, &obj_out) {
-                    Ok(()) => { }
-                    Err(err) => {
-                        cgcx.handler.err(&format!("unable to copy {} to {}: {}",
-                                                  buf.display(),
-                                                  obj_out.display(),
-                                                  err));
+            ModuleSource::Preexisting(wp) => {
+                let incremental = cgcx.incremental.as_ref().unwrap();
+                let name = &work_item.mtrans.name;
+                for (kind, saved_file) in wp.saved_files {
+                    let obj_out = work_item.output_names.temp_path(kind, Some(name));
+                    let source_file = incremental.join(&saved_file);
+                    debug!("copying pre-existing module `{}` from {:?} to {}",
+                           work_item.mtrans.name,
+                           source_file,
+                           obj_out.display());
+                    match link_or_copy(&source_file, &obj_out) {
+                        Ok(()) => { }
+                        Err(err) => {
+                            cgcx.handler.err(&format!("unable to copy {} to {}: {}",
+                                                      source_file.display(),
+                                                      obj_out.display(),
+                                                      err));
+                        }
                     }
                 }
             }
@@ -994,6 +1011,8 @@ fn run_work_multithreaded(sess: &Session,
         let mut tx = Some(tx);
         futures.push(rx);
 
+        let incremental = sess.opts.incremental.clone();
+
         thread::Builder::new().name(format!("codegen-{}", i)).spawn(move || {
             let diag_handler = Handler::with_emitter(true, false, box diag_emitter);
 
@@ -1005,6 +1024,7 @@ fn run_work_multithreaded(sess: &Session,
                 plugin_passes: plugin_passes,
                 remark: remark,
                 worker: i,
+                incremental: incremental,
             };
 
             loop {

src/librustc_trans/base.rs

@@ -43,10 +43,9 @@ use rustc::ty::subst::{self, Substs};
 use rustc::traits;
 use rustc::ty::{self, Ty, TyCtxt, TypeFoldable};
 use rustc::ty::adjustment::CustomCoerceUnsized;
-use rustc::dep_graph::DepNode;
+use rustc::dep_graph::{DepNode, WorkProduct};
 use rustc::hir::map as hir_map;
 use rustc::util::common::time;
-use rustc_incremental::in_incr_comp_dir;
 use rustc::mir::mir_map::MirMap;
 use rustc_data_structures::graph::OUTGOING;
 use session::config::{self, NoDebugInfo, FullDebugInfo};
@@ -103,7 +102,6 @@ use std::cell::{Cell, RefCell};
 use std::collections::HashMap;
 use std::ptr;
 use std::rc::Rc;
-use std::path::PathBuf;
 use std::str;
 use std::{i8, i16, i32, i64};
 use syntax_pos::{Span, DUMMY_SP};
@@ -2721,7 +2719,7 @@ pub fn trans_crate<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>,
 fn trans_reuse_previous_work_products(tcx: TyCtxt,
                                       codegen_units: &[CodegenUnit],
                                       symbol_map: &SymbolMap)
-                                      -> Vec<Option<PathBuf>> {
+                                      -> Vec<Option<WorkProduct>> {
     debug!("trans_reuse_previous_work_products()");
     codegen_units
         .iter()
@@ -2735,7 +2733,7 @@ fn trans_reuse_previous_work_products(tcx: TyCtxt,
             if let Some(work_product) = tcx.dep_graph.previous_work_product(&id) {
                 if work_product.input_hash == hash {
                     debug!("trans_reuse_previous_work_products: reusing {:?}", work_product);
-                    return Some(in_incr_comp_dir(tcx.sess, &work_product.file_name).unwrap());
+                    return Some(work_product);
                 } else {
                     debug!("trans_reuse_previous_work_products: \
                             not reusing {:?} because hash changed to {:?}",