Add support for shell argfiles

Author: djkoloski

Cargo.lock

@@ -3750,6 +3750,7 @@ dependencies = [
  "rustc_trait_selection",
  "rustc_ty_utils",
  "serde_json",
+ "shlex",
  "time",
  "tracing",
  "windows",

compiler/rustc_driver_impl/Cargo.toml

@@ -50,6 +50,7 @@ rustc_target = { path = "../rustc_target" }
 rustc_trait_selection = { path = "../rustc_trait_selection" }
 rustc_ty_utils = { path = "../rustc_ty_utils" }
 serde_json = "1.0.59"
+shlex = "1.0"
 time = { version = "0.3", default-features = false, features = ["alloc", "formatting"] }
 tracing = { version = "0.1.35" }
 # tidy-alphabetical-end

compiler/rustc_driver_impl/src/args.rs

@@ -5,39 +5,97 @@ use std::io;
 
 use rustc_session::EarlyErrorHandler;
 
-fn arg_expand(arg: String) -> Result<Vec<String>, Error> {
-    if let Some(path) = arg.strip_prefix('@') {
-        let file = match fs::read_to_string(path) {
-            Ok(file) => file,
-            Err(ref err) if err.kind() == io::ErrorKind::InvalidData => {
-                return Err(Error::Utf8Error(Some(path.to_string())));
+/// Expands argfiles in command line arguments.
+#[derive(Default)]
+struct Expander {
+    shell_argfiles: bool,
+    next_is_unstable_option: bool,
+    expanded: Vec<String>,
+}
+
+impl Expander {
+    /// Handles the next argument. If the argument is an argfile, it is expanded
+    /// inline.
+    fn arg(&mut self, arg: &str) -> Result<(), Error> {
+        if let Some(argfile) = arg.strip_prefix('@') {
+            match argfile.split_once(':') {
+                Some(("shell", path)) if self.shell_argfiles => {
+                    shlex::split(&Self::read_file(path)?)
+                        .ok_or_else(|| Error::ShellParseError(path.to_string()))?
+                        .into_iter()
+                        .for_each(|arg| self.push(arg));
+                }
+                _ => {
+                    let contents = Self::read_file(argfile)?;
+                    contents.lines().for_each(|arg| self.push(arg.to_string()));
+                }
+            }
+        } else {
+            self.push(arg.to_string());
+        }
+
+        Ok(())
+    }
+
+    /// Adds a command line argument verbatim with no argfile expansion.
+    fn push(&mut self, arg: String) {
+        if self.next_is_unstable_option {
+            self.inspect_unstable_option(&arg);
+            self.next_is_unstable_option = false;
+        } else if let Some(unstable_option) = arg.strip_prefix("-Z") {
+            if unstable_option.is_empty() {
+                self.next_is_unstable_option = true;
+            } else {
+                self.inspect_unstable_option(unstable_option);
+            }
+        }
+
+        self.expanded.push(arg);
+    }
+
+    /// Consumes the `Expander`, returning the expanded arguments.
+    fn finish(self) -> Vec<String> {
+        self.expanded
+    }
+
+    /// Parses any relevant unstable flags specified on the command line.
+    fn inspect_unstable_option(&mut self, option: &str) {
+        match option {
+            "shell-argfiles" => self.shell_argfiles = true,
+            _ => (),
+        }
+    }
+
+    /// Reads the contents of a file as UTF-8.
+    fn read_file(path: &str) -> Result<String, Error> {
+        fs::read_to_string(path).map_err(|e| {
+            if e.kind() == io::ErrorKind::InvalidData {
+                Error::Utf8Error(Some(path.to_string()))
+            } else {
+                Error::IOError(path.to_string(), e)
             }
-            Err(err) => return Err(Error::IOError(path.to_string(), err)),
-        };
-        Ok(file.lines().map(ToString::to_string).collect())
-    } else {
-        Ok(vec![arg])
+        })
     }
 }
 
 /// **Note:** This function doesn't interpret argument 0 in any special way.
 /// If this function is intended to be used with command line arguments,
 /// `argv[0]` must be removed prior to calling it manually.
 pub fn arg_expand_all(handler: &EarlyErrorHandler, at_args: &[String]) -> Vec<String> {
-    let mut args = Vec::new();
+    let mut expander = Expander::default();
     for arg in at_args {
-        match arg_expand(arg.clone()) {
-            Ok(arg) => args.extend(arg),
-            Err(err) => handler.early_error(format!("Failed to load argument file: {err}")),
+        if let Err(err) = expander.arg(arg) {
+            handler.early_error(format!("Failed to load argument file: {err}"));
         }
     }
-    args
+    expander.finish()
 }
 
 #[derive(Debug)]
 pub enum Error {
     Utf8Error(Option<String>),
     IOError(String, io::Error),
+    ShellParseError(String),
 }
 
 impl fmt::Display for Error {
@@ -46,6 +104,7 @@ impl fmt::Display for Error {
             Error::Utf8Error(None) => write!(fmt, "Utf8 error"),
             Error::Utf8Error(Some(path)) => write!(fmt, "Utf8 error in {path}"),
             Error::IOError(path, err) => write!(fmt, "IO Error: {path}: {err}"),
+            Error::ShellParseError(path) => write!(fmt, "Invalid shell-style arguments in {path}"),
         }
     }
 }

compiler/rustc_interface/src/tests.rs

@@ -702,6 +702,7 @@ fn test_unstable_options_tracking_hash() {
     untracked!(query_dep_graph, true);
     untracked!(self_profile, SwitchWithOptPath::Enabled(None));
     untracked!(self_profile_events, Some(vec![String::new()]));
+    untracked!(shell_argfiles, true);
     untracked!(span_debug, true);
     untracked!(span_free_formats, true);
     untracked!(temps_dir, Some(String::from("abc")));

compiler/rustc_session/src/options.rs

@@ -1832,6 +1832,8 @@ written to standard error output)"),
                      query-blocked, incr-cache-load, incr-result-hashing, query-keys, function-args, args, llvm, artifact-sizes"),
     share_generics: Option<bool> = (None, parse_opt_bool, [TRACKED],
         "make the current crate share its generic instantiations"),
+        shell_argfiles: bool = (false, parse_bool, [UNTRACKED],
+        "allow argument files to be specified with POSIX \"shell-style\" argument quoting"),
     show_span: Option<String> = (None, parse_opt_string, [TRACKED],
         "show spans for compiler debugging (expr|pat|ty)"),
     simulate_remapped_rust_src_base: Option<PathBuf> = (None, parse_opt_pathbuf, [TRACKED],

src/doc/unstable-book/src/compiler-flags/shell-argfiles.md

@@ -0,0 +1,11 @@
+# `shell-argfiles`
+
+--------------------
+
+The `-Zshell-argfiles` compiler flag allows argfiles to be parsed using POSIX
+"shell-style" quoting. When enabled, the compiler will use `shlex` to parse the
+arguments from argfiles specified with `@shell:<path>`.
+
+Because this feature controls the parsing of input arguments, the
+`-Zshell-argfiles` flag must be present before the argument specifying the
+shell-style arguemnt file.

src/tools/tidy/src/deps.rs

@@ -327,6 +327,7 @@ const PERMITTED_RUSTC_DEPENDENCIES: &[&str] = &[
     "sha1",
     "sha2",
     "sharded-slab",
+    "shlex",
     "smallvec",
     "snap",
     "stable_deref_trait",

src/tools/tidy/src/ui_tests.rs

@@ -27,6 +27,10 @@ const EXTENSION_EXCEPTION_PATHS: &[&str] = &[
     "tests/ui/check-cfg/my-awesome-platform.json", // testing custom targets with cfgs
     "tests/ui/commandline-argfile-badutf8.args", // passing args via a file
     "tests/ui/commandline-argfile.args", // passing args via a file
+    "tests/ui/commandline-argfile-shell.args", // passing args via a file
+    "tests/ui/commandline-argfile-shell-badquotes.args", // passing args via a file
+    "tests/ui/commandline-argfile-shell-via-argfile-shell.args", // passing args via a file
+    "tests/ui/commandline-argfile-shell-via-argfile.args", // passing args via a file
     "tests/ui/crate-loading/auxiliary/libfoo.rlib", // testing loading a manually created rlib
     "tests/ui/include-macros/data.bin", // testing including data with the include macros
     "tests/ui/include-macros/file.txt", // testing including data with the include macros

tests/ui/commandline-argfile-shell-badquotes.args

@@ -0,0 +1 @@
+"--cfg" "unquoted_set

tests/ui/commandline-argfile-shell-badquotes.rs

@@ -0,0 +1,6 @@
+// Check to see if we can get parameters from an @argsfile file
+//
+// compile-flags: --cfg cmdline_set -Z shell-argfiles @shell:{{src-base}}/commandline-argfile-shell-badquotes.args
+
+fn main() {
+}