[Driver] Refactor to use llvm Option's new Visibility flags

This is a big refactor of the clang driver's option handling to use
the Visibility flags introduced in https://reviews.llvm.org/D157149.
There are a few distinct parts, but they can't really be split into
separate commits and still be made to compile.

1. We split out some of the flags in ClangFlags to ClangVisibility.
   Note that this does not include any subtractive flags.

2. We update the Flag definitions and OptIn/OptOut constructs in
   Options.td by hand.

3. We introduce and use a script, update_options_td_flags, to ease
   migration of flag definitions in Options.td, and we run that on
   Options.td. I intend to remove this later, but I'm committing it so
   that downstream forks can use the script to simplify merging.

4. We update calls to OptTable in the clang driver, cc1as, flang, and
   clangd to use the visibility APIs instead of Include/Exclude flags.

5. We deprecate the Include/Exclude APIs and add a release note.

*if you are running into conflicts with this change:*

Note that https://reviews.llvm.org/D157150 may also be the culprit and
if so it should be handled first.

The script in `clang/utils/update_options_td_flags.py` can help. Take
the downstream side of all conflicts and then run the following:

```
% cd clang/include/clang/Driver
% ../../../utils/update_options_td_flags.py Options.td > Options.td.new
% mv Options.td.new Options.td
```

This will hopefully be sufficient, please take a look at the diff.

Differential Revision: https://reviews.llvm.org/D157151

Author: bogner

clang-tools-extra/clangd/CompileCommands.cpp

@@ -237,14 +237,8 @@ void CommandMangler::operator()(tooling::CompileCommand &Command,
   llvm::opt::InputArgList ArgList;
   ArgList = OptTable.ParseArgs(
       llvm::ArrayRef(OriginalArgs).drop_front(), IgnoredCount, IgnoredCount,
-      /*FlagsToInclude=*/
-      IsCLMode ? (driver::options::CLOption | driver::options::CoreOption |
-                  driver::options::CLDXCOption)
-               : /*everything*/ 0,
-      /*FlagsToExclude=*/driver::options::NoDriverOption |
-          (IsCLMode
-               ? 0
-               : (driver::options::CLOption | driver::options::CLDXCOption)));
+      llvm::opt::Visibility(IsCLMode ? driver::options::CLOption
+                                     : driver::options::ClangOption));
 
   llvm::SmallVector<unsigned, 1> IndicesToDrop;
   // Having multiple architecture options (e.g. when building fat binaries)
@@ -441,23 +435,13 @@ DriverMode getDriverMode(const std::vector<std::string> &Args) {
 
 // Returns the set of DriverModes where an option may be used.
 unsigned char getModes(const llvm::opt::Option &Opt) {
-  // Why is this so complicated?!
-  // Reference is clang::driver::Driver::getIncludeExcludeOptionFlagMasks()
   unsigned char Result = DM_None;
-  if (Opt.hasFlag(driver::options::CC1Option))
+  if (Opt.hasVisibilityFlag(driver::options::ClangOption))
+    Result |= DM_GCC;
+  if (Opt.hasVisibilityFlag(driver::options::CC1Option))
     Result |= DM_CC1;
-  if (!Opt.hasFlag(driver::options::NoDriverOption)) {
-    if (Opt.hasFlag(driver::options::CLOption)) {
-      Result |= DM_CL;
-    } else if (Opt.hasFlag(driver::options::CLDXCOption)) {
-      Result |= DM_CL;
-    } else {
-      Result |= DM_GCC;
-      if (Opt.hasFlag(driver::options::CoreOption)) {
-        Result |= DM_CL;
-      }
-    }
-  }
+  if (Opt.hasVisibilityFlag(driver::options::CLOption))
+    Result |= DM_CL;
   return Result;
 }
 

clang-tools-extra/modularize/Modularize.cpp

@@ -336,13 +336,13 @@ std::string CommandLine;
 
 // Helper function for finding the input file in an arguments list.
 static std::string findInputFile(const CommandLineArguments &CLArgs) {
-  const unsigned IncludedFlagsBitmask = options::CC1Option;
+  llvm::opt::Visibility VisibilityMask(options::CC1Option);
   unsigned MissingArgIndex, MissingArgCount;
   SmallVector<const char *, 256> Argv;
   for (auto I = CLArgs.begin(), E = CLArgs.end(); I != E; ++I)
     Argv.push_back(I->c_str());
   InputArgList Args = getDriverOptTable().ParseArgs(
-      Argv, MissingArgIndex, MissingArgCount, IncludedFlagsBitmask);
+      Argv, MissingArgIndex, MissingArgCount, VisibilityMask);
   std::vector<std::string> Inputs = Args.getAllArgValues(OPT_INPUT);
   return ModularizeUtilities::getCanonicalPath(Inputs.back());
 }

clang/docs/InternalsManual.rst

@@ -663,9 +663,11 @@ Then, specify additional attributes via mix-ins:
 * ``HelpText`` holds the text that will be printed besides the option name when
   the user requests help (e.g. via ``clang --help``).
 * ``Group`` specifies the "category" of options this option belongs to. This is
-  used by various tools to filter certain options of interest.
-* ``Flags`` may contain a number of "tags" associated with the option. This
-  enables more granular filtering than the ``Group`` attribute.
+  used by various tools to categorize and sometimes filter options.
+* ``Flags`` may contain "tags" associated with the option. These may affect how
+  the option is rendered, or if it's hidden in some contexts.
+* ``Visibility`` should be used to specify the drivers in which a particular
+  option would be available. This attribute will impact tool --help
 * ``Alias`` denotes that the option is an alias of another option. This may be
   combined with ``AliasArgs`` that holds the implied value.
 
@@ -674,12 +676,14 @@ Then, specify additional attributes via mix-ins:
     // Options.td
 
     def fpass_plugin_EQ : Joined<["-"], "fpass-plugin=">,
-  +   Group<f_Group>, Flags<[CC1Option]>,
+  +   Group<f_Group>, Visibility<[ClangOption, CC1Option]>,
   +   HelpText<"Load pass plugin from a dynamic shared object file.">;
 
-New options are recognized by the Clang driver unless marked with the
-``NoDriverOption`` flag. On the other hand, options intended for the ``-cc1``
-frontend must be explicitly marked with the ``CC1Option`` flag.
+New options are recognized by the ``clang`` driver mode if ``Visibility`` is
+not specified or contains ``ClangOption``. Options intended for ``clang -cc1``
+must be explicitly marked with the ``CC1Option`` flag. Flags that specify
+``CC1Option`` but not ``ClangOption`` will only be accessible via ``-cc1``.
+This is similar for other driver modes, such as ``clang-cl`` or ``flang``.
 
 Next, parse (or manufacture) the command line arguments in the Clang driver and
 use them to construct the ``-cc1`` job:
@@ -874,7 +878,8 @@ present on command line.
 
 .. code-block:: text
 
-  def fignore_exceptions : Flag<["-"], "fignore-exceptions">, Flags<[CC1Option]>,
+  def fignore_exceptions : Flag<["-"], "fignore-exceptions">,
+    Visibility<[ClangOption, CC1Option]>,
     MarshallingInfoFlag<LangOpts<"IgnoreExceptions">>;
 
 **Negative Flag**
@@ -884,7 +889,8 @@ present on command line.
 
 .. code-block:: text
 
-  def fno_verbose_asm : Flag<["-"], "fno-verbose-asm">, Flags<[CC1Option]>,
+  def fno_verbose_asm : Flag<["-"], "fno-verbose-asm">,
+    Visibility<[ClangOption, CC1Option]>,
     MarshallingInfoNegativeFlag<CodeGenOpts<"AsmVerbose">>;
 
 **Negative and Positive Flag**
@@ -898,9 +904,9 @@ line.
 
   defm legacy_pass_manager : BoolOption<"f", "legacy-pass-manager",
     CodeGenOpts<"LegacyPassManager">, DefaultFalse,
-    PosFlag<SetTrue, [], "Use the legacy pass manager in LLVM">,
-    NegFlag<SetFalse, [], "Use the new pass manager in LLVM">,
-    BothFlags<[CC1Option]>>;
+    PosFlag<SetTrue, [], [], "Use the legacy pass manager in LLVM">,
+    NegFlag<SetFalse, [], [], "Use the new pass manager in LLVM">,
+    BothFlags<[], [ClangOption, CC1Option]>>;
 
 With most such pair of flags, the ``-cc1`` frontend accepts only the flag that
 changes the default key path value. The Clang driver is responsible for
@@ -912,10 +918,11 @@ full names of both flags. The positive flag would then be named
 ``flegacy-pass-manager`` and the negative ``fno-legacy-pass-manager``.
 ``BoolOption`` also implies the ``-`` prefix for both flags. It's also possible
 to use ``BoolFOption`` that implies the ``"f"`` prefix and ``Group<f_Group>``.
-The ``PosFlag`` and ``NegFlag`` classes hold the associated boolean value, an
-array of elements passed to the ``Flag`` class and the help text. The optional
-``BothFlags`` class holds an array of ``Flag`` elements that are common for both
-the positive and negative flag and their common help text suffix.
+The ``PosFlag`` and ``NegFlag`` classes hold the associated boolean value,
+arrays of elements passed to the ``Flag`` and ``Visibility`` classes and the
+help text. The optional ``BothFlags`` class holds arrays of ``Flag`` and
+``Visibility`` elements that are common for both the positive and negative flag
+and their common help text suffix.
 
 **String**
 
@@ -924,7 +931,8 @@ the option appears on the command line, the argument value is simply copied.
 
 .. code-block:: text
 
-  def isysroot : JoinedOrSeparate<["-"], "isysroot">, Flags<[CC1Option]>,
+  def isysroot : JoinedOrSeparate<["-"], "isysroot">,
+    Visibility<[ClangOption, CC1Option]>,
     MarshallingInfoString<HeaderSearchOpts<"Sysroot">, [{"/"}]>;
 
 **List of Strings**
@@ -935,7 +943,8 @@ vector.
 
 .. code-block:: text
 
-  def frewrite_map_file : Separate<["-"], "frewrite-map-file">, Flags<[CC1Option]>,
+  def frewrite_map_file : Separate<["-"], "frewrite-map-file">,
+    Visibility<[ClangOption, CC1Option]>,
     MarshallingInfoStringVector<CodeGenOpts<"RewriteMapFiles">>;
 
 **Integer**
@@ -946,7 +955,8 @@ and the result is assigned to the key path on success.
 
 .. code-block:: text
 
-  def mstack_probe_size : Joined<["-"], "mstack-probe-size=">, Flags<[CC1Option]>,
+  def mstack_probe_size : Joined<["-"], "mstack-probe-size=">,
+    Visibility<[ClangOption, CC1Option]>,
     MarshallingInfoInt<CodeGenOpts<"StackProbeSize">, "4096">;
 
 **Enumeration**
@@ -963,7 +973,8 @@ comma-separated string values and elements of the array within
 
 .. code-block:: text
 
-  def mthread_model : Separate<["-"], "mthread-model">, Flags<[CC1Option]>,
+  def mthread_model : Separate<["-"], "mthread-model">,
+    Visibility<[ClangOption, CC1Option]>,
     Values<"posix,single">, NormalizedValues<["POSIX", "Single"]>,
     NormalizedValuesScope<"LangOptions::ThreadModelKind">,
     MarshallingInfoEnum<LangOpts<"ThreadModel">, "POSIX">;
@@ -983,7 +994,8 @@ Finally, the command line is parsed according to the primary annotation.
 
 .. code-block:: text
 
-  def fms_extensions : Flag<["-"], "fms-extensions">, Flags<[CC1Option]>,
+  def fms_extensions : Flag<["-"], "fms-extensions">,
+    Visibility<[ClangOption, CC1Option]>,
     MarshallingInfoFlag<LangOpts<"MicrosoftExt">>,
     ImpliedByAnyOf<[fms_compatibility.KeyPath], "true">;
 
@@ -994,7 +1006,8 @@ true.
 
 .. code-block:: text
 
-  def fopenmp_enable_irbuilder : Flag<["-"], "fopenmp-enable-irbuilder">, Flags<[CC1Option]>,
+  def fopenmp_enable_irbuilder : Flag<["-"], "fopenmp-enable-irbuilder">,
+    Visibility<[ClangOption, CC1Option]>,
     MarshallingInfoFlag<LangOpts<"OpenMPIRBuilder">>,
     ShouldParseIf<fopenmp.KeyPath>;
 

clang/include/clang/Driver/ClangOptionDocs.td

@@ -28,8 +28,10 @@ GCC-compatible ``clang`` and ``clang++`` drivers.
 }];
 
   string Program = "clang";
-  list<string> ExcludedFlags = ["HelpHidden", "NoDriverOption",
-                                "CLOption", "Unsupported", "Ignored", "FlangOnlyOption"];
+  // Note: We *must* use DefaultVis and not ClangOption, since that's
+  // the name of the actual TableGen record. The alias will not work.
+  list<string> VisibilityMask = ["DefaultVis"];
+  list<string> IgnoreFlags = ["HelpHidden", "Unsupported", "Ignored"];
 }
 
 include "Options.td"

clang/include/clang/Driver/Driver.h

@@ -455,7 +455,7 @@ class Driver {
   /// ParseArgStrings - Parse the given list of strings into an
   /// ArgList.
   llvm::opt::InputArgList ParseArgStrings(ArrayRef<const char *> Args,
-                                          bool IsClCompatMode,
+                                          bool UseDriverMode,
                                           bool &ContainsError);
 
   /// BuildInputs - Construct the list of inputs and their types from
@@ -759,7 +759,8 @@ class Driver {
 
   /// Get bitmasks for which option flags to include and exclude based on
   /// the driver mode.
-  std::pair<unsigned, unsigned> getIncludeExcludeOptionFlagMasks(bool IsClCompatMode) const;
+  llvm::opt::Visibility
+  getOptionVisibilityMask(bool UseDriverMode = true) const;
 
   /// Helper used in BuildJobsForAction.  Doesn't use the cache when building
   /// jobs specifically for the given action, but will use the cache when

clang/include/clang/Driver/Options.h

@@ -23,26 +23,22 @@ enum ClangFlags {
   LinkerInput = (1 << 5),
   NoArgumentUnused = (1 << 6),
   Unsupported = (1 << 7),
-  CoreOption = (1 << 8),
-  CLOption = (1 << 9),
-  CC1Option = (1 << 10),
-  CC1AsOption = (1 << 11),
-  NoDriverOption = (1 << 12),
-  LinkOption = (1 << 13),
-  FlangOption = (1 << 14),
-  FC1Option = (1 << 15),
-  FlangOnlyOption = (1 << 16),
-  DXCOption = (1 << 17),
-  CLDXCOption = (1 << 18),
-  Ignored = (1 << 19),
-  TargetSpecific = (1 << 20),
+  LinkOption = (1 << 8),
+  Ignored = (1 << 9),
+  TargetSpecific = (1 << 10),
 };
 
 // Flags specifically for clang option visibility. We alias DefaultVis to
 // ClangOption, because "DefaultVis" is confusing in Options.td, which is used
 // for multiple drivers (clang, cl, flang, etc).
 enum ClangVisibility {
   ClangOption = llvm::opt::DefaultVis,
+  CLOption = (1 << 1),
+  CC1Option = (1 << 2),
+  CC1AsOption = (1 << 3),
+  FlangOption = (1 << 4),
+  FC1Option = (1 << 5),
+  DXCOption = (1 << 6),
 };
 
 enum ID {